feat: new db structure; platform labels

.gitignore

@@ -26,12 +26,6 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
-# local env files
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
 # vercel
 .vercel
 
@@ -41,5 +35,4 @@ package-lock.json ## use yarn.lock instead
 # Local Netlify folder
 .netlify
 
-/.env
+/.env*
-/.env.production

src/backend/database/pg-wrapper.ts

@@ -1,13 +1,11 @@
 import { Pool, PoolClient } from "pg";
 
-import { Forecast, platforms } from "../platforms";
+import { Forecast } from "../platforms";
 import { hash } from "../utils/hash";
 import { measureTime } from "../utils/measureTime";
 import { roughSizeOfObject } from "../utils/roughSize";
 
-const platformTableNames = platforms.map((platform) => platform.name);
+const forecastTableNames = ["questions", "history"];
-
-const forecastTableNames = [...platformTableNames, "combined", "history"];
 
 const allTableNames = [...forecastTableNames, "dashboards", "frontpage"];
 
@@ -111,28 +109,27 @@ let buildMetaforecastTable = (table: string) => `CREATE TABLE ${table} (
     extra json
   );`;
 
-async function pgInitializeLatest() {
+async function pgInitializeQuestions() {
   let YOLO = false;
   if (YOLO) {
     console.log("Create tables & their indexes");
-    for (const table of platformTableNames) {
+    const table = "questions";
-      await runPgCommand({
+    await runPgCommand({
-        command: dropTable(table),
+      command: dropTable(table),
-        pool: readWritePool,
+      pool: readWritePool,
-      });
+    });
-      await runPgCommand({
+    await runPgCommand({
-        command: buildMetaforecastTable(table),
+      command: buildMetaforecastTable(table),
-        pool: readWritePool,
+      pool: readWritePool,
-      });
+    });
-      await runPgCommand({
+    await runPgCommand({
-        command: createUniqueIndex(table),
+      command: createUniqueIndex(table),
-        pool: readWritePool,
+      pool: readWritePool,
-      });
+    });
-    }
     console.log("");
   } else {
     console.log(
-      "pgInitializeLatest: This command is dangerous, set YOLO to true in the code to invoke it"
+      "pgInitializeQuestions: This command is dangerous, set YOLO to true in the code to invoke it"
     );
   }
 }
@@ -234,7 +231,7 @@ async function pgInitializeFrontpage() {
 
 export async function pgInitialize() {
   await pgInitializeScaffolding();
-  await pgInitializeLatest();
+  await pgInitializeQuestions();
   await pgInitializeHistories();
   await pgInitializeDashboards();
   await pgInitializeFrontpage();
@@ -416,11 +413,11 @@ pgInsertIntoDashboard({
 export async function pgUpsert({
   contents,
   tableName,
-  replace,
+  replacePlatform,
 }: {
   contents: Forecast[];
   tableName: string;
-  replace: boolean;
+  replacePlatform?: string;
 }) {
   if (!forecastTableNames.includes(tableName)) {
     throw Error(
@@ -432,8 +429,10 @@ export async function pgUpsert({
     const client = await readWritePool.connect();
     try {
       await client.query("BEGIN");
-      if (replace) {
+      if (replacePlatform) {
-        client.query(`DELETE FROM ${tableName}`);
+        await client.query(`DELETE FROM ${tableName} WHERE platform = $1`, [
+          replacePlatform,
+        ]);
       }
       console.log(
         `Upserting ${contents.length} rows into postgres table ${tableName}.`

src/backend/flow/history/updateHistory.ts

@@ -1,10 +1,9 @@
 import { pgReadWithReadCredentials, pgUpsert } from "../../database/pg-wrapper";
 
 export async function updateHistory() {
-  let latest = await pgReadWithReadCredentials({ tableName: "combined" });
+  let latest = await pgReadWithReadCredentials({ tableName: "questions" });
   await pgUpsert({
     contents: latest,
     tableName: "history",
-    replace: false,
   });
 }

src/backend/flow/jobs.ts

@@ -1,7 +1,6 @@
 import { pgInitialize } from "../database/pg-wrapper";
 import { doEverything } from "../flow/doEverything";
 import { updateHistory } from "../flow/history/updateHistory";
-import { mergeEverything } from "../flow/mergeEverything";
 import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData";
 import { rebuildFrontpage } from "../frontpage";
 import { platforms, processPlatform } from "../platforms";
@@ -20,13 +19,6 @@ export const jobs: Job[] = [
     message: `Download predictions from ${platform.name}`,
     run: () => processPlatform(platform),
   })),
-  {
-    name: "merge",
-    message:
-      "Merge tables into one big table (and push the result to a pg database)",
-    run: mergeEverything,
-    separate: true,
-  },
   {
     name: "algolia",
     message: 'Rebuild algolia database ("index")',

src/backend/flow/mergeEverything.ts

@@ -1,28 +0,0 @@
-import { pgRead, pgUpsert } from "../database/pg-wrapper";
-import { platforms } from "../platforms";
-
-/* Merge everything */
-
-export async function mergeEverythingInner() {
-  let merged = [];
-  for (let platform of platforms) {
-    const platformName = platform.name;
-    let json = await pgRead({ tableName: platformName });
-    console.log(`${platformName} has ${json.length} questions\n`);
-    merged = merged.concat(json);
-  }
-  let mergedprocessed = merged.map((element) => ({
-    ...element,
-    optionsstringforsearch: element.options
-      .map((option) => option.name)
-      .join(", "),
-  }));
-  console.log(`In total, there are ${mergedprocessed.length} questions`);
-  return mergedprocessed;
-}
-
-export async function mergeEverything() {
-  let merged = await mergeEverythingInner();
-  await pgUpsert({ contents: merged, tableName: "combined", replace: true });
-  console.log("Done");
-}

src/backend/frontpage.ts

@@ -1,6 +1,7 @@
 import { pgRead, readWritePool } from "./database/pg-wrapper";
+import { Forecast } from "./platforms";
 
-export async function getFrontpageRaw() {
+export async function getFrontpage(): Promise<Forecast[]> {
   const client = await readWritePool.connect();
   const res = await client.query(
     "SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
@@ -10,7 +11,7 @@ export async function getFrontpageRaw() {
   return res.rows[0].frontpage_sliced;
 }
 
-export async function getFrontpageFullRaw() {
+export async function getFrontpageFull(): Promise<Forecast[]> {
   const client = await readWritePool.connect();
   const res = await client.query(
     "SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
@@ -20,31 +21,15 @@ export async function getFrontpageFullRaw() {
   return res.rows[0].frontpage_full;
 }
 
-export async function getFrontpage() {
-  let frontPageForecastsCompatibleWithFuse = [];
-  try {
-    let data = await getFrontpageRaw();
-    frontPageForecastsCompatibleWithFuse = data.map((result) => ({
-      item: result,
-      score: 0,
-    }));
-    return frontPageForecastsCompatibleWithFuse;
-  } catch (error) {
-    console.log(error);
-  } finally {
-    return frontPageForecastsCompatibleWithFuse;
-  }
-}
-
 export async function rebuildFrontpage() {
   const frontpageFull = await pgRead({
-    tableName: "combined",
+    tableName: "questions",
   });
 
   const client = await readWritePool.connect();
   const frontpageSliced = (
     await client.query(`
-    SELECT * FROM combined
+    SELECT * FROM questions
     WHERE
       (qualityindicators->>'stars')::int >= 3
       AND description != ''

src/backend/manual/manualDownload.ts

@@ -5,7 +5,7 @@ import fs from "fs";
 import { pgReadWithReadCredentials } from "../database/pg-wrapper";
 
 let main = async () => {
-  let json = await pgReadWithReadCredentials({ tableName: "combined" });
+  let json = await pgReadWithReadCredentials({ tableName: "questions" });
   let string = JSON.stringify(json, null, 2);
   let filename = "metaforecasts.json";
   fs.writeFileSync(filename, string);

src/backend/manual/noSchemaMigrate.ts

@@ -1,7 +1,6 @@
 import "dotenv/config";
 
 import { readWritePool } from "../database/pg-wrapper";
-import { platforms } from "../platforms";
 
 const migrate = async () => {
   const client = await readWritePool.connect();
@@ -11,6 +10,25 @@ const migrate = async () => {
     await client.query(q);
   };
 
+  const platformTitleToName = {
+    Betfair: "betfair",
+    FantasySCOTUS: "fantasyscotus",
+    Foretold: "foretold",
+    "GiveWell/OpenPhilanthropy": "givewellopenphil",
+    "Good Judgment": "goodjudgement",
+    "Good Judgment Open": "goodjudgmentopen",
+    Infer: "infer",
+    Kalshi: "kalshi",
+    "Manifold Markets": "manifold",
+    Metaculus: "metaculus",
+    "Peter Wildeford": "wildeford",
+    PolyMarket: "polymarket",
+    PredictIt: "predictit",
+    Rootclaim: "rootclaim",
+    Smarkets: "smarkets",
+    "X-risk estimates": "xrisk",
+  };
+
   try {
     await client.query("BEGIN");
     const copyTable = async (from: string, to: string) => {
@@ -19,13 +37,49 @@ const migrate = async () => {
       await execQuery(`INSERT INTO ${to} SELECT * FROM ${from}`);
     };
 
-    for (const platform of platforms) {
-      await copyTable(`latest.${platform.name}`, platform.name);
-    }
     await copyTable("latest.dashboards", "dashboards");
-    await copyTable("latest.combined", "combined");
+    await copyTable("latest.combined", "questions");
     await copyTable("latest.frontpage", "frontpage");
     await copyTable("history.h2022", "history");
+
+    for (const [title, name] of Object.entries(platformTitleToName)) {
+      console.log(`Updating ${title} -> ${name}`);
+      for (const table of ["questions", "history"]) {
+        await client.query(
+          `UPDATE ${table} SET platform=$1 WHERE platform=$2`,
+          [name, title]
+        );
+      }
+    }
+
+    console.log("Fixing GJOpen ids in questions and history");
+    for (const table of ["questions", "history"]) {
+      await client.query(
+        `UPDATE ${table} SET id=REPLACE(id, 'goodjudmentopen-', 'goodjudgmentopen-') WHERE id LIKE 'goodjudmentopen-%'`
+      );
+    }
+
+    const fixId = (id: string) =>
+      id.replace("goodjudmentopen-", "goodjudgmentopen-");
+
+    console.log(
+      "Please rebuild frontpage manually - current version includes invalid GJOpen and xrisk ids"
+    );
+
+    const updateDashboards = async () => {
+      const res = await client.query("SELECT id, contents FROM dashboards");
+      for (const row of res.rows) {
+        let { id, contents } = row;
+        contents = contents.map(fixId);
+        await client.query(
+          "UPDATE dashboards SET contents = $1 WHERE id = $2",
+          [JSON.stringify(contents), id]
+        );
+      }
+    };
+    console.log("Updating dashboards");
+    await updateDashboards();
+
     await client.query("COMMIT");
   } catch (e) {
     await client.query("ROLLBACK");

src/backend/platforms/betfair.ts

@@ -5,6 +5,8 @@ import https from "https";
 import { calculateStars } from "../utils/stars";
 import { Forecast, Platform } from "./";
 
+const platformName = "betfair";
+
 /* Definitions */
 let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
 
@@ -121,12 +123,14 @@ async function processPredictions(data) {
       id: id,
       title: title,
       url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
-      platform: "Betfair",
+      platform: platformName,
       description: description,
       options: options,
       timestamp: new Date().toISOString(),
       qualityindicators: {
-        stars: calculateStars("Betfair", { volume: prediction.totalMatched }),
+        stars: calculateStars(platformName, {
+          volume: prediction.totalMatched,
+        }),
         volume: prediction.totalMatched,
       },
     };
@@ -136,7 +140,9 @@ async function processPredictions(data) {
 }
 
 export const betfair: Platform = {
-  name: "betfair",
+  name: platformName,
+  label: "Betfair",
+  color: "#3d674a",
   async fetcher() {
     const data = await fetchPredictions();
     const results = await processPredictions(data); // somehow needed

src/backend/platforms/example-fetch.ts

@@ -23,7 +23,7 @@ async function fetchData() {
 
 async function processPredictions(predictions) {
   let results = await predictions.map((prediction) => {
-    let id = `platform-${prediction.id}`;
+    let id = `example-${prediction.id}`;
     let probability = prediction.probability;
     let options = [
       {
@@ -61,6 +61,8 @@ async function processPredictions(predictions) {
 
 export const example: Platform = {
   name: "example",
+  label: "Example platform",
+  color: "#ff0000",
   async fetcher() {
     let data = await fetchData();
     let results = await processPredictions(data); // somehow needed

src/backend/platforms/fantasyscotus.ts

@@ -2,7 +2,9 @@
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
-import { Platform } from "./";
+import { Forecast, Platform } from "./";
+
+const platformName = "fantasyscotus";
 
 /* Definitions */
 let unixtime = new Date().getTime();
@@ -65,19 +67,19 @@ async function processData(data) {
   let historicalPercentageCorrect = data.stats.pcnt_correct;
   let historicalProbabilityCorrect =
     Number(historicalPercentageCorrect.replace("%", "")) / 100;
-  let results = [];
+  let results: Forecast[] = [];
   for (let event of events) {
     if (event.accuracy == "") {
-      let id = `fantasyscotus-${event.id}`;
+      let id = `${platformName}-${event.id}`;
       // if the thing hasn't already resolved
       let predictionData = await getPredictionsData(event.docket_url);
       let pAffirm = predictionData.proportionAffirm;
       //let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
-      let eventObject = {
+      let eventObject: Forecast = {
         id: id,
         title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
         url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,
-        platform: "FantasySCOTUS",
+        platform: platformName,
         description: `${(pAffirm * 100).toFixed(2)}% (${
           predictionData.numAffirm
         } out of ${
@@ -100,7 +102,7 @@ async function processData(data) {
         timestamp: new Date().toISOString(),
         qualityindicators: {
           numforecasts: Number(predictionData.numForecasts),
-          stars: calculateStars("FantasySCOTUS", {}),
+          stars: calculateStars(platformName, {}),
         },
       };
       results.push(eventObject);
@@ -112,7 +114,9 @@ async function processData(data) {
 
 /* Body */
 export const fantasyscotus: Platform = {
-  name: "fantasyscotus",
+  name: platformName,
+  label: "FantasySCOTUS",
+  color: "#231149",
   async fetcher() {
     let rawData = await fetchData();
     let results = await processData(rawData);

src/backend/platforms/foretold.ts

@@ -5,6 +5,9 @@ import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
 /* Definitions */
+
+const platformName = "foretold";
+
 let graphQLendpoint = "https://api.foretold.io/graphql";
 let highQualityCommunities = [
   "0104d8e8-07e4-464b-8b32-74ef22b49f21",
@@ -54,7 +57,9 @@ async function fetchAllCommunityQuestions(communityId) {
 }
 
 export const foretold: Platform = {
-  name: "foretold",
+  name: platformName,
+  label: "Foretold",
+  color: "#62520b",
   async fetcher() {
     let results = [];
     for (let community of highQualityCommunities) {
@@ -62,7 +67,7 @@ export const foretold: Platform = {
       questions = questions.map((question) => question.node);
       questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
       questions.forEach((question) => {
-        let id = `foretold-${question.id}`;
+        let id = `${platformName}-${question.id}`;
         let options = [];
         if (question.valueType == "PERCENTAGE") {
           let probability = question.previousAggregate.value.percentage;
@@ -83,13 +88,13 @@ export const foretold: Platform = {
           id: id,
           title: question.name,
           url: `https://www.foretold.io/c/${community}/m/${question.id}`,
-          platform: "Foretold",
+          platform: platformName,
           description: "",
           options: options,
           timestamp: new Date().toISOString(),
           qualityindicators: {
             numforecasts: Math.floor(Number(question.measurementCount) / 2),
-            stars: calculateStars("Foretold", {}),
+            stars: calculateStars(platformName, {}),
           },
           /*liquidity: liquidity.toFixed(2),
           tradevolume: tradevolume.toFixed(2),

src/backend/platforms/givewellopenphil.ts

@@ -5,6 +5,8 @@ import fs from "fs";
 import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
+const platformName = "givewellopenphil";
+
 /* Support functions */
 async function fetchPage(url: string) {
   let response = await axios({
@@ -48,11 +50,11 @@ async function main1() {
     let result = {
       title: title,
       url: url,
-      platform: "GiveWell",
+      platform: platformName,
       description: description,
       timestamp: new Date().toISOString(),
       qualityindicators: {
-        stars: calculateStars("GiveWell/OpenPhilanthropy", {}),
+        stars: calculateStars(platformName, {}),
       },
     }; // Note: This requires some processing afterwards
     // console.log(result)
@@ -65,7 +67,9 @@ async function main1() {
 }
 
 export const givewellopenphil: Platform = {
-  name: "givewellopenphil",
+  name: platformName,
+  label: "GiveWell/OpenPhilanthropy",
+  color: "#32407e",
   async fetcher() {
     // main1()
     return; // not necessary to refill the DB every time

src/backend/platforms/goodjudgment.ts

@@ -8,6 +8,7 @@ import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "goodjudgment";
 let endpoint = "https://goodjudgment.io/superforecasts/";
 String.prototype.replaceAll = function replaceAll(search, replace) {
   return this.split(search).join(replace);
@@ -15,7 +16,9 @@ String.prototype.replaceAll = function replaceAll(search, replace) {
 
 /* Body */
 export const goodjudgment: Platform = {
-  name: "goodjudgment",
+  name: platformName,
+  label: "Good Judgment",
+  color: "#7d4f1b",
   async fetcher() {
     // Proxy fuckery
     let proxy;
@@ -64,7 +67,7 @@ export const goodjudgment: Platform = {
       let title = table[0]["0"].split("\t\t\t").splice(3)[0];
       if (title != undefined) {
         title = title.replaceAll("</a>", "");
-        let id = `goodjudgment-${hash(title)}`;
+        let id = `${platformName}-${hash(title)}`;
         let description = table
           .filter((row) => row["0"].includes("BACKGROUND:"))
           .map((row) => row["0"])
@@ -101,12 +104,12 @@ export const goodjudgment: Platform = {
           id: id,
           title: title,
           url: endpoint,
-          platform: "Good Judgment",
+          platform: platformName,
           description: description,
           options: options,
           timestamp: new Date().toISOString(),
           qualityindicators: {
-            stars: calculateStars("Good Judgment", {}),
+            stars: calculateStars(platformName, {}),
           },
           extra: {
             superforecastercommentary: analysis || "",

src/backend/platforms/goodjudgmentopen.ts

@@ -8,6 +8,8 @@ import toMarkdown from "../utils/toMarkdown";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "goodjudgmentopen";
+
 let htmlEndPoint = "https://www.gjopen.com/questions?page=";
 let annoyingPromptUrls = [
   "https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
@@ -185,12 +187,12 @@ async function goodjudgmentopen_inner(cookie) {
           }
           let questionNumRegex = new RegExp("questions/([0-9]+)");
           let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
-          let id = `goodjudmentopen-${questionNum}`;
+          let id = `${platformName}-${questionNum}`;
           let question = {
             id: id,
             title: title,
             url: url,
-            platform: "Good Judgment Open",
+            platform: platformName,
             ...moreinfo,
           };
           if (j % 30 == 0 || DEBUG_MODE == "on") {
@@ -236,8 +238,10 @@ async function goodjudgmentopen_inner(cookie) {
   return results;
 }
 
-export const goodjudmentopen: Platform = {
+export const goodjudgmentopen: Platform = {
-  name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen`
+  name: platformName,
+  label: "Good Judgment Open",
+  color: "#002455",
   async fetcher() {
     let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
     return await applyIfSecretExists(cookie, goodjudgmentopen_inner);

src/backend/platforms/index.ts

@@ -4,10 +4,10 @@ import { fantasyscotus } from "./fantasyscotus";
 import { foretold } from "./foretold";
 import { givewellopenphil } from "./givewellopenphil";
 import { goodjudgment } from "./goodjudgment";
-import { goodjudmentopen } from "./goodjudmentopen";
+import { goodjudgmentopen } from "./goodjudgmentopen";
 import { infer } from "./infer";
 import { kalshi } from "./kalshi";
-import { manifoldmarkets } from "./manifoldmarkets";
+import { manifold } from "./manifold";
 import { metaculus } from "./metaculus";
 import { polymarket } from "./polymarket";
 import { predictit } from "./predictit";
@@ -67,7 +67,9 @@ export interface Forecast {
 export type PlatformFetcher = () => Promise<Forecast[] | null>;
 
 export interface Platform {
-  name: string;
+  name: string; // short name for ids and `platform` db column, e.g. "xrisk"
+  label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates"
+  color: string; // used on frontend
   fetcher?: PlatformFetcher;
 }
 
@@ -92,10 +94,10 @@ export const platforms: Platform[] = [
   foretold,
   givewellopenphil,
   goodjudgment,
-  goodjudmentopen,
+  goodjudgmentopen,
   infer,
   kalshi,
-  manifoldmarkets,
+  manifold,
   metaculus,
   polymarket,
   predictit,
@@ -114,8 +116,8 @@ export const processPlatform = async (platform: Platform) => {
   if (results && results.length) {
     await pgUpsert({
       contents: results,
-      tableName: platform.name,
+      tableName: "questions",
-      replace: true,
+      replacePlatform: platform.name,
     });
     console.log("Done");
   } else {

src/backend/platforms/infer.ts

@@ -8,6 +8,7 @@ import toMarkdown from "../utils/toMarkdown";
 import { Forecast, Platform } from "./";
 
 /* Definitions */
+const platformName = "infer";
 let htmlEndPoint = "https://www.infer-pub.com/questions";
 String.prototype.replaceAll = function replaceAll(search, replace) {
   return this.split(search).join(replace);
@@ -145,7 +146,7 @@ async function fetchStats(questionUrl, cookie) {
     qualityindicators: {
       numforecasts: Number(numforecasts),
       numforecasters: Number(numforecasters),
-      stars: calculateStars("Infer", { numforecasts }),
+      stars: calculateStars(platformName, { numforecasts }),
     },
   };
 
@@ -218,7 +219,7 @@ async function infer_inner(cookie) {
         let moreinfo = await fetchStats(url, cookie);
         let questionNumRegex = new RegExp("questions/([0-9]+)");
         let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
-        let id = `infer-${questionNum}`;
+        let id = `${platformName}-${questionNum}`;
         let question = {
           id: id,
           title: title,
@@ -278,7 +279,9 @@ async function infer_inner(cookie) {
 }
 
 export const infer: Platform = {
-  name: "infer",
+  name: platformName,
+  label: "Infer",
+  color: "#223900",
   async fetcher() {
     let cookie = process.env.INFER_COOKIE;
     return await applyIfSecretExists(cookie, infer_inner);

src/backend/platforms/kalshi.ts

@@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "kalshi";
 let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 
 async function fetchAllMarkets() {
@@ -34,17 +35,17 @@ async function processMarkets(markets) {
         type: "PROBABILITY",
       },
     ];
-    let id = `kalshi-${market.id}`;
+    let id = `${platformName}-${market.id}`;
     let result = {
       id: id,
       title: market.title.replaceAll("*", ""),
       url: `https://kalshi.com/markets/${market.ticker_name}`,
-      platform: "Kalshi",
+      platform: platformName,
       description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
       options: options,
       timestamp: new Date().toISOString(),
       qualityindicators: {
-        stars: calculateStars("Kalshi", {
+        stars: calculateStars(platformName, {
           shares_volume: market.volume,
           interest: market.open_interest,
         }),
@@ -70,7 +71,9 @@ async function processMarkets(markets) {
 }
 
 export const kalshi: Platform = {
-  name: "kalshi",
+  name: platformName,
+  label: "Kalshi",
+  color: "#615691",
   fetcher: async function () {
     let markets = await fetchAllMarkets();
     return await processMarkets(markets);

src/backend/platforms/manifold.ts

@@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "manifold";
 let endpoint = "https://manifold.markets/api/v0/markets";
 // See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5
 
@@ -44,7 +45,7 @@ function showStatistics(results) {
 
 async function processPredictions(predictions) {
   let results = await predictions.map((prediction) => {
-    let id = `manifold-${prediction.id}`;
+    let id = `manifold-${prediction.id}`; // oops, doesn't match platform name
     let probability = prediction.probability;
     let options = [
       {
@@ -67,7 +68,7 @@ async function processPredictions(predictions) {
       options: options,
       timestamp: new Date().toISOString(),
       qualityindicators: {
-        stars: calculateStars("Manifold Markets", {
+        stars: calculateStars(platformName, {
           volume7Days: prediction.volume7Days,
           volume24Hours: prediction.volume24Hours,
           pool: prediction.pool,
@@ -88,8 +89,10 @@ async function processPredictions(predictions) {
   return unresolvedResults; //resultsProcessed
 }
 
-export const manifoldmarkets: Platform = {
+export const manifold: Platform = {
-  name: "manifoldmarkets",
+  name: platformName,
+  label: "Manifold Markets",
+  color: "#793466",
   async fetcher() {
     let data = await fetchData();
     let results = await processPredictions(data); // somehow needed

src/backend/platforms/metaculus.ts

@@ -6,6 +6,7 @@ import toMarkdown from "../utils/toMarkdown";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "metaculus";
 let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
 let now = new Date().toISOString();
 let DEBUG_MODE = "off";
@@ -94,7 +95,9 @@ async function fetchMetaculusQuestionDescription(slug) {
 }
 
 export const metaculus: Platform = {
-  name: "metaculus",
+  name: platformName,
+  label: "Metaculus",
+  color: "#006669",
   async fetcher() {
     // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
     // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
@@ -144,18 +147,18 @@ export const metaculus: Platform = {
                 },
               ];
             }
-            let id = `metaculus-${result.id}`;
+            let id = `${platformName}-${result.id}`;
             let interestingInfo = {
               id: id,
               title: result.title,
               url: "https://www.metaculus.com" + result.page_url,
-              platform: "Metaculus",
+              platform: platformName,
               description: description,
               options: options,
               timestamp: new Date().toISOString(),
               qualityindicators: {
                 numforecasts: Number(result.number_of_predictions),
-                stars: calculateStars("Metaculus", {
+                stars: calculateStars(platformName, {
                   numforecasts: result.number_of_predictions,
                 }),
               },

src/backend/platforms/polymarket.ts

@@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
 import { Forecast, Platform } from "./";
 
 /* Definitions */
+const platformName = "polymarket";
 let graphQLendpoint =
   "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 let units = 10 ** 6;
@@ -63,7 +64,9 @@ async function fetchIndividualContractData(marketMakerAddress) {
 }
 
 export const polymarket: Platform = {
-  name: "polymarket",
+  name: platformName,
+  label: "PolyMarket",
+  color: "#00314e",
   async fetcher() {
     let results: Forecast[] = [];
     let webpageEndpointData = await fetchAllContractInfo();
@@ -79,7 +82,7 @@ export const polymarket: Platform = {
         );
         if (moreMarketAnswer.length > 0) {
           let moreMarketInfo = moreMarketAnswer[0];
-          let id = `polymarket-${addressLowerCase.slice(0, 10)}`;
+          let id = `${platformName}-${addressLowerCase.slice(0, 10)}`;
           // console.log(id);
           let numforecasts = Number(moreMarketInfo.tradesQuantity);
           let tradevolume =
@@ -103,7 +106,7 @@ export const polymarket: Platform = {
             id: id,
             title: marketInfo.question,
             url: "https://polymarket.com/market/" + marketInfo.slug,
-            platform: "PolyMarket",
+            platform: platformName,
             description: marketInfo.description,
             options: options,
             timestamp: new Date().toISOString(),
@@ -111,7 +114,7 @@ export const polymarket: Platform = {
               numforecasts: numforecasts.toFixed(0),
               liquidity: liquidity.toFixed(2),
               tradevolume: tradevolume.toFixed(2),
-              stars: calculateStars("Polymarket", {
+              stars: calculateStars(platformName, {
                 liquidity,
                 option: options[0],
                 volume: tradevolume,

src/backend/platforms/predictit.ts

@@ -1,10 +1,11 @@
-/* Imports */
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
 import { Platform } from "./";
 
+const platformName = "predictit";
+
 /* Support functions */
 async function fetchmarkets() {
   let response = await axios({
@@ -39,7 +40,9 @@ function sleep(ms: number) {
 
 /* Body */
 export const predictit: Platform = {
-  name: "predictit",
+  name: platformName,
+  label: "PredictIt",
+  color: "#460c00",
   async fetcher() {
     let markets = await fetchmarkets();
     let marketVolumes = await fetchmarketvolumes();
@@ -53,7 +56,7 @@ export const predictit: Platform = {
     let results = [];
     for (let market of markets) {
       // console.log(market.name)
-      let id = `predictit-${market.id}`;
+      let id = `${platformName}-${market.id}`;
       let isbinary = market.contracts.length == 1;
       await sleep(3000 * (1 + Math.random()));
       let descriptionraw = await fetchmarketrules(market.id);
@@ -97,12 +100,12 @@ export const predictit: Platform = {
         id: id,
         title: market["name"],
         url: market.url,
-        platform: "PredictIt",
+        platform: platformName,
         description: description,
         options: options,
         timestamp: new Date().toISOString(),
         qualityindicators: {
-          stars: calculateStars("PredictIt", {}),
+          stars: calculateStars(platformName, {}),
           shares_volume: shares_volume,
         },
       };

src/backend/platforms/rootclaim.ts

@@ -1,11 +1,11 @@
-/* Imports */
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
-import { Platform } from "./";
+import { Forecast, Platform } from "./";
 
 /* Definitions */
+const platformName = "rootclaim";
 let jsonEndpoint =
   "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 
@@ -24,12 +24,14 @@ async function fetchAllRootclaims() {
 }
 
 export const rootclaim: Platform = {
-  name: "rootclaim",
+  name: platformName,
+  label: "Rootclaim",
+  color: "#0d1624",
   async fetcher() {
     let claims = await fetchAllRootclaims();
-    let results = [];
+    let results: Forecast[] = [];
     for (let claim of claims) {
-      let id = `rootclaim-${claim.slug.toLowerCase()}`;
+      let id = `${platformName}-${claim.slug.toLowerCase()}`;
       let options = [];
       for (let scenario of claim.scenarios) {
         //console.log(scenario)
@@ -42,17 +44,17 @@ export const rootclaim: Platform = {
         });
       }
       let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
-      let obj = {
+      let obj: Forecast = {
         id: id,
         title: toMarkdown(claim.question).replace("\n", ""),
         url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
-        platform: "Rootclaim",
+        platform: platformName,
         description: toMarkdown(claim.background).replace("&#39;", "'"),
         options: options,
         timestamp: new Date().toISOString(),
         qualityindicators: {
           numforecasts: 1,
-          stars: calculateStars("Rootclaim", {}),
+          stars: calculateStars(platformName, {}),
         },
       };
       results.push(obj);

src/backend/platforms/smarkets.ts

@@ -1,13 +1,14 @@
-/* Imports */
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "smarkets";
 let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
 let VERBOSE = false;
 let empty = () => 0;
+
 /* Support functions */
 
 async function fetchEvents(url) {
@@ -60,7 +61,9 @@ async function fetchPrices(marketid) {
 }
 
 export const smarkets: Platform = {
-  name: "smarkets",
+  name: platformName,
+  label: "Smarkets",
+  color: "#6f5b41",
   async fetcher() {
     let htmlPath =
       "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
@@ -93,7 +96,7 @@ export const smarkets: Platform = {
     for (let market of markets) {
       VERBOSE ? console.log("================") : empty();
       VERBOSE ? console.log("Market: ", market) : empty();
-      let id = `smarkets-${market.id}`;
+      let id = `${platformName}-${market.id}`;
       let name = market.name;
 
       let contracts = await fetchContracts(market.id);
@@ -160,12 +163,12 @@ export const smarkets: Platform = {
         id: id,
         title: name,
         url: "https://smarkets.com/event/" + market.event_id + market.slug,
-        platform: "Smarkets",
+        platform: platformName,
         description: market.description,
         options: options,
         timestamp: new Date().toISOString(),
         qualityindicators: {
-          stars: calculateStars("Smarkets", {}),
+          stars: calculateStars(platformName, {}),
         },
       };
       VERBOSE ? console.log(result) : empty();

src/backend/platforms/wildeford.ts

@@ -7,6 +7,7 @@ import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
 /* Definitions */
+const platformName = "wildeford";
 const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
 const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`;
 // https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0
@@ -73,7 +74,7 @@ async function processPredictions(predictions) {
   );
   let results = currentPredictions.map((prediction) => {
     let title = prediction["Prediction"].replace(" [update]", "");
-    let id = `wildeford-${hash(title)}`;
+    let id = `${platformName}-${hash(title)}`;
     let probability = Number(prediction["Odds"].replace("%", "")) / 100;
     let options = [
       {
@@ -91,14 +92,14 @@ async function processPredictions(predictions) {
       id: id,
       title: title,
       url: prediction["url"],
-      platform: "Peter Wildeford",
+      platform: platformName,
       description: prediction["Notes"] || "",
       options: options,
       timestamp: new Date(
         Date.parse(prediction["Prediction Date"] + "Z")
       ).toISOString(),
       qualityindicators: {
-        stars: calculateStars("Peter Wildeford", null),
+        stars: calculateStars(platformName, null),
       },
     };
     return result;
@@ -120,7 +121,9 @@ export async function wildeford_inner(google_api_key) {
 }
 
 export const wildeford: Platform = {
-  name: "wildeford",
+  name: platformName,
+  label: "Peter Wildeford",
+  color: "#984158",
   async fetcher() {
     const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
     return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);

src/backend/platforms/xrisk.ts

@@ -1,15 +1,25 @@
 import fs from "fs";
 
+import { hash } from "../utils/hash";
 import { Platform } from "./";
 
+const platformName = "xrisk";
+
 export const xrisk: Platform = {
   name: "xrisk",
+  label: "X-risk estimates",
+  color: "#272600",
   async fetcher() {
-    return; // not necessary to refill the DB every time
+    // return; // not necessary to refill the DB every time
     let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {
       encoding: "utf-8",
     });
-    const results = JSON.parse(fileRaw);
+    let results = JSON.parse(fileRaw);
+    results = results.map((item) => ({
+      ...item,
+      id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
+      platform: platformName,
+    }));
     return results;
   },
 };

src/backend/utils/algolia.ts

@@ -1,34 +1,13 @@
 import algoliasearch from "algoliasearch";
 
 import { pgReadWithReadCredentials } from "../database/pg-wrapper";
-import { mergeEverythingInner } from "../flow/mergeEverything";
+import { platforms } from "../platforms";
 
 let cookie = process.env.ALGOLIA_MASTER_API_KEY;
 const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
 const client = algoliasearch(algoliaAppId, cookie);
 const index = client.initIndex("metaforecast");
 
-export async function rebuildAlgoliaDatabaseTheHardWay() {
-  console.log("Doing this the hard way");
-  let records = await mergeEverythingInner();
-  records = records.map((record, index: number) => ({
-    ...record,
-    has_numforecasts: record.numforecasts ? true : false,
-    objectID: index,
-  }));
-  // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
-
-  if (index.exists()) {
-    console.log("Index exists");
-    index
-      .replaceAllObjects(records, { safe: true })
-      .catch((error) => console.log(error));
-    console.log(
-      `Pushed ${records.length} records. Algolia will update asynchronously`
-    );
-  }
-}
-
 let getoptionsstringforsearch = (record: any) => {
   let result = "";
   if (!!record.options && record.options.length > 0) {
@@ -42,11 +21,16 @@ let getoptionsstringforsearch = (record: any) => {
 
 export async function rebuildAlgoliaDatabaseTheEasyWay() {
   let records: any[] = await pgReadWithReadCredentials({
-    tableName: "combined",
+    tableName: "questions",
   });
 
+  const platformNameToLabel = Object.fromEntries(
+    platforms.map((platform) => [platform.name, platform.label])
+  );
+
   records = records.map((record, index: number) => ({
     ...record,
+    platformLabel: platformNameToLabel[record.platform] || record.platform,
     has_numforecasts: record.numforecasts ? true : false,
     objectID: index,
     optionsstringforsearch: getoptionsstringforsearch(record),
@@ -62,4 +46,4 @@ export async function rebuildAlgoliaDatabaseTheEasyWay() {
   }
 }
 
-export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay
+export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay;

src/backend/utils/evaluations/pullForecastsToCSVForRating.ts

@@ -24,7 +24,7 @@ let main = async () => {
     "PredictIt",
     "Rootclaim",
   ];
-  let json = await pgReadWithReadCredentials({ tableName: "combined" });
+  let json = await pgReadWithReadCredentials({ tableName: "questions" });
   console.log(json.length);
   //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
   //console.log(uniquePlatforms)

src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts

@@ -26,7 +26,7 @@ let shuffleArray = (array) => {
 
 let main = async () => {
   let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim']
-  let json = await pgReadWithReadCredentials({ tableName: "combined" });
+  let json = await pgReadWithReadCredentials({ tableName: "questions" });
   console.log(json.length);
   //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
   //console.log(uniquePlatforms)

src/backend/utils/misc/process-forecasts-into-elicit.ts

@@ -9,7 +9,7 @@ let locationData = "./data/";
 /* Body */
 // let rawdata =  fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src
 async function main() {
-  let data = await pgReadWithReadCredentials({ tableName: "combined" }); //JSON.parse(rawdata)
+  let data = await pgReadWithReadCredentials({ tableName: "questions" }); //JSON.parse(rawdata)
   let processDescription = (description) => {
     if (description == null || description == undefined || description == "") {
       return "";

src/backend/utils/stars.ts

@@ -105,7 +105,7 @@ function calculateStarsGiveWellOpenPhil(data) {
   return starsInteger;
 }
 
-function calculateStarsGoodJudment(data) {
+function calculateStarsGoodJudgment(data) {
   let nuno = (data) => 4;
   let eli = (data) => 4;
   let misha = (data) => 3.5;
@@ -114,7 +114,7 @@ function calculateStarsGoodJudment(data) {
   return starsInteger;
 }
 
-function calculateStarsGoodJudmentOpen(data) {
+function calculateStarsGoodJudgmentOpen(data) {
   let nuno = (data) => (data.numforecasts > 100 ? 3 : 2);
   let eli = (data) => 3;
   let misha = (data) =>
@@ -173,7 +173,7 @@ function calculateStarsLadbrokes(data) {
   return starsInteger;
 }
 
-function calculateStarsManifoldMarkets(data) {
+function calculateStarsManifold(data) {
   let nuno = (data) =>
     data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100)
       ? 2
@@ -268,15 +268,56 @@ function calculateStarsWilliamHill(data) {
   return starsInteger;
 }
 
-export function calculateStars(platform, data) {
+export function calculateStars(platform: string, data) {
   let stars = 2;
   switch (platform) {
+    case "betfair":
+      stars = calculateStarsBetfair(data);
+      break;
+    case "infer":
+      stars = calculateStarsInfer(data);
+      break;
+    case "foretold":
+      stars = calculateStarsForetold(data);
+      break;
+    case "givewellopenphil":
+      stars = calculateStarsGiveWellOpenPhil(data);
+      break;
+    case "goodjudgment":
+      stars = calculateStarsGoodJudgment(data);
+      break;
+    case "goodjudgmentopen":
+      stars = calculateStarsGoodJudgmentOpen(data);
+      break;
+    case "kalshi":
+      stars = calculateStarsKalshi(data);
+      break;
+    case "manifold":
+      stars = calculateStarsManifold(data);
+      break;
+    case "metaculus":
+      stars = calculateStarsMetaculus(data);
+      break;
+    case "polymarket":
+      stars = calculateStarsPolymarket(data);
+      break;
+    case "predictit":
+      stars = calculateStarsPredictIt(data);
+      break;
+    case "rootclaim":
+      stars = calculateStarsRootclaim(data);
+      break;
+    case "smarkets":
+      stars = calculateStarsSmarkets(data);
+      break;
+    case "wildeford":
+      stars = calculateStarsWildeford(data);
+      break;
+
+    // deprecated
     case "AstralCodexTen":
       stars = calculateStarsAstralCodexTen(data);
       break;
-    case "Betfair":
-      stars = calculateStarsBetfair(data);
-      break;
     case "CoupCast":
       stars = calculateStarsCoupCast(data);
       break;
@@ -289,54 +330,15 @@ export function calculateStars(platform, data) {
     case "Estimize":
       stars = calculateStarsEstimize(data);
       break;
-    case "Foretold":
-      stars = calculateStarsForetold(data);
-      break;
-    case "GiveWell/OpenPhilanthropy":
-      stars = calculateStarsGiveWellOpenPhil(data);
-      break;
-    case "Good Judgment":
-      stars = calculateStarsGoodJudment(data);
-      break;
-    case "Good Judgment Open":
-      stars = calculateStarsGoodJudmentOpen(data);
-      break;
     case "Hypermind":
       stars = calculateStarsHypermind(data);
       break;
-    case "Infer":
-      stars = calculateStarsInfer(data);
-      break;
-    case "Kalshi":
-      stars = calculateStarsKalshi(data);
-      break;
     case "Ladbrokes":
       stars = calculateStarsLadbrokes(data);
       break;
-    case "Manifold Markets":
-      stars = calculateStarsManifoldMarkets(data);
-      break;
-    case "Metaculus":
-      stars = calculateStarsMetaculus(data);
-      break;
     case "Omen":
       stars = calculateStarsOmen(data);
       break;
-    case "Polymarket":
-      stars = calculateStarsPolymarket(data);
-      break;
-    case "PredictIt":
-      stars = calculateStarsPredictIt(data);
-      break;
-    case "Rootclaim":
-      stars = calculateStarsRootclaim(data);
-      break;
-    case "Smarkets":
-      stars = calculateStarsSmarkets(data);
-      break;
-    case "Peter Wildeford":
-      stars = calculateStarsWildeford(data);
-      break;
     case "WilliamHill":
       stars = calculateStarsWilliamHill(data);
       break;

src/pages/api/all-forecasts.ts

@@ -1,12 +1,12 @@
 import { NextApiRequest, NextApiResponse } from "next/types";
 
-import { getFrontpageFullRaw } from "../../backend/frontpage";
+import { getFrontpageFull } from "../../backend/frontpage";
 
 export default async function handler(
   req: NextApiRequest,
   res: NextApiResponse
 ) {
-  let frontpageFull = await getFrontpageFullRaw();
+  let frontpageFull = await getFrontpageFull();
   console.log(frontpageFull.map((element) => element.title).slice(0, 5));
   console.log("...");
   res.status(200).json(frontpageFull);

src/pages/api/create-dashboard-from-ids.ts

@@ -1,7 +1,7 @@
-import crypto from "crypto";
 import { NextApiRequest, NextApiResponse } from "next/types";
 
 import { pgInsertIntoDashboard } from "../../backend/database/pg-wrapper";
+import { hash } from "../../backend/utils/hash";
 
 export default async function handler(
   req: NextApiRequest,
@@ -14,8 +14,6 @@ export default async function handler(
 
   let body = req.body;
   console.log(body);
-  const hash = (s: string) =>
-    crypto.createHash("sha256").update(s).digest("hex").slice(0, 10);
   try {
     let id = hash(JSON.stringify(body.ids));
     let pgResponse = await pgInsertIntoDashboard({

src/pages/api/dashboard-by-id.ts

@@ -23,7 +23,7 @@ export default async function handler(
     console.log(dashboardItem);
     let dashboardContents = await pgGetByIds({
       ids: dashboardItem.contents,
-      table: "combined",
+      table: "questions",
     });
     res.status(200).send({
       dashboardContents,

src/pages/api/frontpage.ts

@@ -1,12 +1,12 @@
 import { NextApiRequest, NextApiResponse } from "next/types";
 
-import { getFrontpageRaw } from "../../backend/frontpage";
+import { getFrontpage } from "../../backend/frontpage";
 
 export default async function handler(
   req: NextApiRequest,
   res: NextApiResponse
 ) {
-  let frontpageElements = await getFrontpageRaw();
+  let frontpageElements = await getFrontpage();
   console.log(frontpageElements.map((element) => element.title).slice(0, 5));
   console.log("...");
   res.status(200).json(frontpageElements);

src/pages/api/questions.ts

@@ -6,7 +6,7 @@ export default async function handler(
   req: NextApiRequest,
   res: NextApiResponse
 ) {
-  let allQuestions = await pgRead({ tableName: "combined" });
+  let allQuestions = await pgRead({ tableName: "questions" });
   console.log(allQuestions.map((element) => element.title).slice(0, 5));
   console.log("...");
   res.status(200).json(allQuestions);

src/pages/capture.tsx

@@ -8,17 +8,11 @@ import CommonDisplay from "../web/search/CommonDisplay";
 
 export { getServerSideProps } from "../web/search/anySearchPage";
 
-const CapturePage: NextPage<Props> = ({
+const CapturePage: NextPage<Props> = (props) => {
-  defaultResults,
-  initialResults,
-  initialQueryParameters,
-}) => {
   return (
     <Layout page={"capture"}>
       <CommonDisplay
-        defaultResults={defaultResults}
+        {...props}
-        initialResults={initialResults}
-        initialQueryParameters={initialQueryParameters}
         hasSearchbar={true}
         hasCapture={true}
         hasAdvancedOptions={false}

src/pages/index.tsx

@@ -8,17 +8,11 @@ import CommonDisplay from "../web/search/CommonDisplay";
 
 export { getServerSideProps } from "../web/search/anySearchPage";
 
-const IndexPage: NextPage<Props> = ({
+const IndexPage: NextPage<Props> = (props) => {
-  defaultResults,
-  initialResults,
-  initialQueryParameters,
-}) => {
   return (
     <Layout page={"search"}>
       <CommonDisplay
-        defaultResults={defaultResults}
+        {...props}
-        initialResults={initialResults}
-        initialQueryParameters={initialQueryParameters}
         hasSearchbar={true}
         hasCapture={false}
         hasAdvancedOptions={true}

src/pages/secretEmbed.tsx

@@ -3,7 +3,7 @@
 import React from "react";
 
 import { displayForecast } from "../web/display/displayForecasts";
-import { platformsWithLabels } from "../web/platforms";
+import { FrontendForecast } from "../web/platforms";
 import searchAccordingToQueryData from "../web/worker/searchAccordingToQueryData";
 
 /* Helper functions */
@@ -15,18 +15,12 @@ export async function getServerSideProps(context) {
     query: "",
     starsThreshold: 2,
     forecastsThreshold: 0,
-    forecastingPlatforms: platformsWithLabels, // weird key value format,
     ...urlQuery,
   };
 
-  let results;
+  let results: FrontendForecast[] = [];
-  switch (initialQueryParameters.query != "") {
+  if (initialQueryParameters.query != "") {
-    case true:
+    results = await searchAccordingToQueryData(initialQueryParameters, 1);
-      results = await searchAccordingToQueryData(initialQueryParameters);
-      break;
-    default:
-      results = [];
-      break;
   }
 
   return {
@@ -49,8 +43,7 @@ export default function Home({ results }) {
             <div id="secretEmbed">
               {result
                 ? displayForecast({
-                    ...result.item,
+                    ...result,
-                    score: result.score,
                     showTimeStamp: true,
                     expandFooterToFullWidth: true,
                   })

src/web/display/dashboardCreator.tsx

@@ -3,7 +3,7 @@ import React, { useState } from "react";
 let exampleInput = `{
   "title": "Random example",
   "description": "Just a random description of a random example",
-  "ids": [ "metaculus-372", "goodjudmentopen-2244", "metaculus-7550", "kalshi-09d060ee-b184-4167-b86b-d773e56b4162", "wildeford-5d1a04e1a8", "metaculus-2817" ],
+  "ids": [ "metaculus-372", "goodjudgmentopen-2244", "metaculus-7550", "kalshi-09d060ee-b184-4167-b86b-d773e56b4162", "wildeford-5d1a04e1a8", "metaculus-2817" ],
   "creator": "Peter Parker"
 }`;
 

src/web/display/displayForecasts.tsx

@@ -3,6 +3,8 @@ import React from "react";
 import { FaRegClipboard } from "react-icons/fa";
 import ReactMarkdown from "react-markdown";
 
+import { FrontendForecast } from "../platforms";
+
 /* Definitions */
 
 /* Support functions */
@@ -350,9 +352,15 @@ let checkIfDisplayTimeStampAtBottom = (qualityIndicators) => {
   }
 };
 
-let getCurrencySymbolIfNeeded = ({ indicator, platform }) => {
+let getCurrencySymbolIfNeeded = ({
+  indicator,
+  platform,
+}: {
+  indicator: any;
+  platform: string;
+}) => {
   let indicatorsWhichNeedCurrencySymbol = ["Volume", "Interest", "Liquidity"];
-  let dollarPlatforms = ["PredictIt", "Kalshi", "PolyMarket"];
+  let dollarPlatforms = ["predictit", "kalshi", "polymarket"];
   if (indicatorsWhichNeedCurrencySymbol.includes(indicator)) {
     if (dollarPlatforms.includes(platform)) {
       return "$";
@@ -461,7 +469,13 @@ let showFirstQualityIndicator = ({
   }
 };
 
-let displayQualityIndicators = ({
+const displayQualityIndicators: React.FC<{
+  numforecasts: number;
+  timestamp: number;
+  showTimeStamp: boolean;
+  qualityindicators: any;
+  platform: string; // id string - e.g. "goodjudgment", not "Good Judgment"
+}> = ({
   numforecasts,
   timestamp,
   showTimeStamp,
@@ -504,6 +518,7 @@ let displayQualityIndicators = ({
 let forecastFooter = ({
   stars,
   platform,
+  platformLabel,
   numforecasts,
   qualityindicators,
   timestamp,
@@ -532,7 +547,7 @@ let forecastFooter = ({
           expandFooterToFullWidth ? "place-self-center" : "self-center"
         }  col-span-1 font-bold ${debuggingWithBackground ? "bg-red-100" : ""}`}
       >
-        {platform
+        {platformLabel
           .replace("Good Judgment Open", "GJOpen")
           .replace("OpenPhilanthropy", "OpenPhil")
           .replace("AstralCodexTen", "ACX")
@@ -559,22 +574,30 @@ let forecastFooter = ({
 
 /* Body */
 
-export function displayForecast({
+interface SingleProps {
-  id,
+  forecast: FrontendForecast;
-  title,
+  showTimeStamp: boolean;
-  url,
+  expandFooterToFullWidth: boolean;
-  platform,
+  showIdToggle?: boolean;
-  author,
+}
-  description,
+
-  options,
+export const displayForecast: React.FC<SingleProps> = ({
-  qualityindicators,
+  forecast: {
-  timestamp,
+    id,
-  visualization,
+    title,
-  score,
+    url,
+    platform,
+    platformLabel,
+    description,
+    options,
+    qualityindicators,
+    timestamp,
+    visualization,
+  },
   showTimeStamp,
   expandFooterToFullWidth,
   showIdToggle,
-}) {
+}) => {
   // const [isJustCopiedSignalVisible, setIsJustCopiedSignalVisible] = useState(false)
   const isJustCopiedSignalVisible = false;
 
@@ -588,7 +611,7 @@ export function displayForecast({
       <div className="flex-grow">
         <div
           className={`text-gray-800 ${opacityFromScore(
-            score
+            0
           )} text-lg mb-2 font-medium justify-self-start`}
         >
           <div
@@ -641,7 +664,7 @@ export function displayForecast({
                       ? "flex"
                       : "hidden"
                   } ${opacityFromScore(
-                    score
+                    0
                   )} row-end-2 col-start-2 col-end-2 row-start-1 row-end-1 col-span-1 items-center justify-center text-gray-600 ml-3 mr-2 `}
                 >
                   <svg className="mt-1" height="10" width="16">
@@ -657,7 +680,7 @@ export function displayForecast({
         {(options.length != 2 ||
           (options[0].name != "Yes" && options[0].name != "No")) && (
           <>
-            <div className={`mb-2 mt-2 ${opacityFromScore(score)}`}>
+            <div className={`mb-2 mt-2 ${opacityFromScore(0)}`}>
               {formatForecastOptions(options)}
             </div>
             <div
@@ -667,7 +690,7 @@ export function displayForecast({
                   ? "flex"
                   : "hidden"
               } ${opacityFromScore(
-                score
+                0
               )} col-start-2 col-end-2 row-start-1 row-end-1 text-gray-600 mt-3 mb-3`}
             >
               <svg className="ml-6 mr-1 mt-2" height="10" width="16">
@@ -680,13 +703,13 @@ export function displayForecast({
           </>
         )}
 
-        {platform !== "Guesstimate" && options.length < 3 && (
+        {platform !== "guesstimate" && options.length < 3 && (
-          <div className={`text-gray-500 ${opacityFromScore(score)} mt-4`}>
+          <div className={`text-gray-500 ${opacityFromScore(0)} mt-4`}>
             {displayMarkdown(description)}
           </div>
         )}
 
-        {platform === "Guesstimate" && (
+        {platform === "guesstimate" && (
           <img
             className="rounded-sm mb-1"
             src={visualization}
@@ -705,10 +728,11 @@ export function displayForecast({
         </svg>
         {`Last updated: ${timestamp ? timestamp.slice(0, 10) : "unknown"}`}
       </div>
-      <div className={`${opacityFromScore(score)} w-full`}>
+      <div className={`${opacityFromScore(0)} w-full`}>
         {forecastFooter({
           stars: qualityindicators.stars,
-          platform: author || platform,
+          platform: platform,
+          platformLabel: platformLabel || platform, // author || platformLabel,
           numforecasts: qualityindicators.numforecasts,
           qualityindicators,
           timestamp,
@@ -718,30 +742,40 @@ export function displayForecast({
       </div>
     </a>
   );
+};
+
+interface Props {
+  results: FrontendForecast[];
+  numDisplay: number;
+  showIdToggle: boolean;
 }
 
-export default function displayForecasts({
+const DisplayForecasts: React.FC<Props> = ({
   results,
   numDisplay,
   showIdToggle,
-}) {
+}) => {
-  return !!results && !!results.slice ? (
+  if (!results) {
-    results.slice(0, numDisplay).map((fuseSearchResult) => {
+    return <></>;
-      /*let displayWithMetaculusCapture =
+  }
+  return (
+    <>
+      {results.slice(0, numDisplay).map((result) => {
+        /*let displayWithMetaculusCapture =
           fuseSearchResult.item.platform == "Metaculus"
             ? metaculusEmbed(fuseSearchResult.item)
             : displayForecast({ ...fuseSearchResult.item });
         */
-      let display = displayForecast({
+        const display = displayForecast({
-        ...fuseSearchResult.item,
+          forecast: result,
-        score: fuseSearchResult.score,
+          showTimeStamp: false,
-        showTimeStamp: false,
+          expandFooterToFullWidth: false,
-        expandFooterToFullWidth: false,
+          showIdToggle,
-        showIdToggle,
+        });
-      });
+        return display;
-      return display;
+      })}
-    })
+    </>
-  ) : (
-    <></>
   );
-}
+};
+
+export default DisplayForecasts;

src/web/display/displayOneForecastForCapture.tsx

@@ -2,16 +2,16 @@ import domtoimage from "dom-to-image"; // https://github.com/tsayen/dom-to-image
 import { useEffect, useRef, useState } from "react";
 import { CopyToClipboard } from "react-copy-to-clipboard";
 
+import { FrontendForecast } from "../platforms";
 import { uploadToImgur } from "../worker/uploadToImgur";
 import { displayForecast } from "./displayForecasts";
 
-function displayOneForecastInner(result, containerRef) {
+function displayOneForecastInner(result: FrontendForecast, containerRef) {
   return (
     <div ref={containerRef}>
       {result
         ? displayForecast({
-            ...result.item,
+            forecast: result,
-            score: result.score,
             showTimeStamp: true,
             expandFooterToFullWidth: true,
           })
@@ -170,7 +170,7 @@ let generateMetaculusSource = (result, hasDisplayBeenCaptured) => {
 };
 
 interface Props {
-  result: any;
+  result: FrontendForecast;
 }
 
 const DisplayOneForecast: React.FC<Props> = ({ result }) => {

src/web/display/multiSelectPlatforms.tsx

@@ -2,7 +2,7 @@ import chroma from "chroma-js";
 import React from "react";
 import Select from "react-select";
 
-import { platformsWithLabels } from "../platforms";
+import { PlatformConfig } from "../platforms";
 
 const colourStyles = {
   control: (styles) => ({ ...styles, backgroundColor: "white" }),
@@ -59,17 +59,48 @@ const colourStyles = {
   }),
 };
 
-export default function MultiSelectPlatform({ onChange, value }) {
+interface Props {
+  onChange: (platforms: string[]) => void;
+  value: string[];
+  platformsConfig: PlatformConfig[];
+}
+
+export const MultiSelectPlatform: React.FC<Props> = ({
+  onChange,
+  value,
+  platformsConfig,
+}) => {
+  type Option = {
+    value: string;
+    label: string;
+    color: string;
+  };
+
+  const options: Option[] = platformsConfig.map((platform) => ({
+    value: platform.name,
+    label: platform.label,
+    color: platform.color,
+  }));
+
+  const id2option: { [k: string]: Option } = {};
+  for (const option of options) id2option[option.value] = option;
+
+  const selectValue = value.map((v) => id2option[v]).filter((v) => v);
+
+  const onSelectChange = (newValue: Option[]) => {
+    onChange(newValue.map((o) => o.value));
+  };
+
   return (
     <Select
-      defaultValue={platformsWithLabels}
+      defaultValue={options}
       isMulti
       className="basic-multi-select w-full text-gray-700"
-      onChange={onChange}
+      onChange={onSelectChange}
       closeMenuOnSelect={false}
-      options={platformsWithLabels}
+      options={options}
-      value={value}
+      value={selectValue}
       styles={colourStyles}
     />
   );
-}
+};

src/web/platforms.ts

@@ -1,61 +1,12 @@
-export const distinctColors = [
+import { Forecast } from "../backend/platforms";
-  "#3d674a",
-  "#231149",
-  "#62520b",
-  "#32407e",
-  "#7d4f1b",
-  "#002455",
-  "#223900",
-  "#615691",
-  "#003419",
-  "#793466",
-  "#006669",
-  "#984158",
-  "#00314e",
-  "#460c00",
-  "#0d1624",
-  "#6f5b41",
-  "#240d23",
-  "#272600",
-  "#755469",
-  "#2a323d",
-];
 
-// https://medialab.github.io/iwanthue/ fancy light background
+export interface PlatformConfig {
-export const platformNames = [
+  name: string;
-  "Betfair",
-  "FantasySCOTUS",
-  "Foretold",
-  "GiveWell/OpenPhilanthropy",
-  "Good Judgment",
-  "Good Judgment Open",
-  "Guesstimate",
-  "Infer",
-  "Kalshi",
-  "Manifold Markets",
-  "Metaculus",
-  "Peter Wildeford",
-  "PolyMarket",
-  "PredictIt",
-  "Rootclaim",
-  "Smarkets",
-  "X-risk estimates",
-];
-
-export interface PlatformWithLabel {
-  value: string;
   label: string;
   color: string;
 }
 
-export const platformsWithLabels: PlatformWithLabel[] = platformNames.map(
+export type FrontendForecast = Forecast & {
-  (name, i) => ({
+  platformLabel: string;
-    value: name,
+  visualization?: any;
-    label: name,
+};
-    color: distinctColors[i],
-  })
-);
-
-export const platforms = platformsWithLabels;
-
-// Deprecated: AstralCodexTen, CoupCast, CSET-foretell, Estimize, Elicit, Hypermind, Omen, WilliamHill

src/web/search/CommonDisplay.tsx

@@ -1,54 +1,13 @@
 import { useRouter } from "next/router";
-import React, {
+import React, { DependencyList, EffectCallback, Fragment, useEffect, useState } from "react";
-  DependencyList,
-  EffectCallback,
-  Fragment,
-  useEffect,
-  useState,
-} from "react";
 
 import ButtonsForStars from "../display/buttonsForStars";
 import Form from "../display/form";
-import MultiSelectPlatform from "../display/multiSelectPlatforms";
+import { MultiSelectPlatform } from "../display/multiSelectPlatforms";
 import { SliderElement } from "../display/slider";
-import { platformsWithLabels, PlatformWithLabel } from "../platforms";
+import { FrontendForecast } from "../platforms";
 import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
-
+import { Props as AnySearchPageProps, QueryParameters } from "./anySearchPage";
-interface QueryParametersWithoutNum {
-  query: string;
-  starsThreshold: number;
-  forecastsThreshold: number;
-  forecastingPlatforms: PlatformWithLabel[];
-}
-
-export interface QueryParameters extends QueryParametersWithoutNum {
-  numDisplay: number;
-}
-
-interface Props {
-  defaultResults: any;
-  initialResults: any;
-  initialQueryParameters: QueryParameters;
-  hasSearchbar: boolean;
-  hasCapture: boolean;
-  hasAdvancedOptions: boolean;
-  placeholder: string;
-  displaySeeMoreHint: boolean;
-  displayForecastsWrapper: (opts: {
-    results: any;
-    numDisplay: number;
-    whichResultToDisplayAndCapture: number;
-    showIdToggle: boolean;
-  }) => React.ReactNode;
-}
-
-export const defaultQueryParameters: QueryParametersWithoutNum = {
-  query: "",
-  starsThreshold: 2,
-  forecastsThreshold: 0,
-  forecastingPlatforms: platformsWithLabels, // weird key value format,
-};
-export const defaultNumDisplay = 21;
 
 const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => {
   const initial = React.useRef(true);
@@ -61,11 +20,29 @@ const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => {
   }, deps);
 };
 
+interface Props extends AnySearchPageProps {
+  hasSearchbar: boolean;
+  hasCapture: boolean;
+  hasAdvancedOptions: boolean;
+  placeholder: string;
+  displaySeeMoreHint: boolean;
+  displayForecastsWrapper: (opts: {
+    results: FrontendForecast[];
+    numDisplay: number;
+    whichResultToDisplayAndCapture: number;
+    showIdToggle: boolean;
+  }) => React.ReactNode;
+}
+
 /* Body */
 const CommonDisplay: React.FC<Props> = ({
   defaultResults,
   initialResults,
   initialQueryParameters,
+  defaultQueryParameters,
+  initialNumDisplay,
+  defaultNumDisplay,
+  platformsConfig,
   hasSearchbar,
   hasCapture,
   hasAdvancedOptions,
@@ -76,13 +53,12 @@ const CommonDisplay: React.FC<Props> = ({
   const router = useRouter();
   /* States */
 
-  const [queryParameters, setQueryParameters] =
+  const [queryParameters, setQueryParameters] = useState<QueryParameters>(
-    useState<QueryParametersWithoutNum>(initialQueryParameters);
+    initialQueryParameters
-
-  const [numDisplay, setNumDisplay] = useState(
-    initialQueryParameters.numDisplay ?? defaultNumDisplay
   );
 
+  const [numDisplay, setNumDisplay] = useState(initialNumDisplay);
+
   // used to distinguish numDisplay updates which force search and don't force search, see effects below
   const [forceSearch, setForceSearch] = useState(0);
 
@@ -100,25 +76,25 @@ const CommonDisplay: React.FC<Props> = ({
       numDisplay,
     };
 
-    let filterManually = (queryData: QueryParameters, results) => {
+    let filterManually = (
+      queryData: QueryParameters,
+      results: FrontendForecast[]
+    ) => {
       if (
         queryData.forecastingPlatforms &&
         queryData.forecastingPlatforms.length > 0
       ) {
-        let forecastingPlatforms = queryData.forecastingPlatforms.map(
-          (platformObj) => platformObj.value
-        );
         results = results.filter((result) =>
-          forecastingPlatforms.includes(result.item.platform)
+          queryData.forecastingPlatforms.includes(result.platform)
         );
       }
       if (queryData.starsThreshold === 4) {
         results = results.filter(
-          (result) => result.item.qualityindicators.stars >= 4
+          (result) => result.qualityindicators.stars >= 4
         );
       }
       if (queryData.forecastsThreshold) {
-        // results = results.filter(result => (result.qualityindicators && result.item.qualityindicators.numforecasts > forecastsThreshold))
+        // results = results.filter(result => (result.qualityindicators && result.qualityindicators.numforecasts > forecastsThreshold))
       }
       return results;
     };
@@ -128,17 +104,13 @@ const CommonDisplay: React.FC<Props> = ({
 
     let results = queryIsEmpty
       ? filterManually(queryData, defaultResults)
-      : await searchAccordingToQueryData(queryData);
+      : await searchAccordingToQueryData(queryData, numDisplay);
 
     setResults(results);
   }
 
   // I don't want the function which display forecasts (displayForecasts) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home.
-  let getInfoToDisplayForecastsFunction = ({
+  let getInfoToDisplayForecastsFunction = () => {
-    results,
-    whichResultToDisplayAndCapture,
-    showIdToggle,
-  }) => {
     let numDisplayRounded =
       numDisplay % 3 != 0
         ? numDisplay + (3 - (Math.round(numDisplay) % 3))
@@ -154,7 +126,7 @@ const CommonDisplay: React.FC<Props> = ({
   const updateRoute = () => {
     const stringify = (key: string, value: any) => {
       if (key === "forecastingPlatforms") {
-        return value.map((x) => x.value).join("|");
+        return value.join("|");
       } else {
         return String(value);
       }
@@ -330,6 +302,7 @@ const CommonDisplay: React.FC<Props> = ({
             </div>
             <div className="flex col-span-3 items-center justify-center">
               <MultiSelectPlatform
+                platformsConfig={platformsConfig}
                 value={queryParameters.forecastingPlatforms}
                 onChange={onChangeSelectedPlatforms}
               />
@@ -344,13 +317,7 @@ const CommonDisplay: React.FC<Props> = ({
         </div>
       ) : null}
 
-      <div>
+      <div>{getInfoToDisplayForecastsFunction()}</div>
-        {getInfoToDisplayForecastsFunction({
-          results,
-          whichResultToDisplayAndCapture,
-          showIdToggle,
-        })}
-      </div>
 
       {displaySeeMoreHint &&
       (!results || (results.length != 0 && numDisplay < results.length)) ? (

src/web/search/anySearchPage.tsx

@@ -1,46 +1,103 @@
 import { GetServerSideProps } from "next";
 
 import { getFrontpage } from "../../backend/frontpage";
+import { platforms } from "../../backend/platforms";
+import { FrontendForecast, PlatformConfig } from "../platforms";
 import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
-import {
-  defaultNumDisplay,
-  defaultQueryParameters,
-  QueryParameters,
-} from "./CommonDisplay";
 
 /* Common code for / and /capture */
 
+export interface QueryParameters {
+  query: string;
+  starsThreshold: number;
+  forecastsThreshold: number;
+  forecastingPlatforms: string[]; // platform names
+}
+
 export interface Props {
-  defaultResults: any;
+  defaultResults: FrontendForecast[];
-  initialResults: any;
+  initialResults: FrontendForecast[];
   initialQueryParameters: QueryParameters;
+  defaultQueryParameters: QueryParameters;
+  initialNumDisplay: number;
+  defaultNumDisplay: number;
+  platformsConfig: PlatformConfig[];
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
-  let urlQuery = context.query;
+  const urlQuery = context.query;
 
-  let initialQueryParameters: QueryParameters = {
+  const platformsConfig = platforms.map((platform) => ({
-    ...defaultQueryParameters,
+    name: platform.name,
-    numDisplay: defaultNumDisplay,
+    label: platform.label,
-    ...urlQuery, // FIXME - parse numerical fields
+    color: platform.color,
+  }));
+  platformsConfig.push({
+    name: "guesstimate",
+    label: "Guesstimate",
+    color: "223900",
+  });
+
+  const defaultQueryParameters: QueryParameters = {
+    query: "",
+    starsThreshold: 2,
+    forecastsThreshold: 0,
+    forecastingPlatforms: platforms.map((platform) => platform.name),
   };
 
-  let defaultResults = await getFrontpage();
+  const initialQueryParameters: QueryParameters = {
+    ...defaultQueryParameters,
+  };
+  if (urlQuery.query) {
+    initialQueryParameters.query = String(urlQuery.query);
+  }
+  if (urlQuery.starsThreshold) {
+    initialQueryParameters.starsThreshold = Number(urlQuery.starsThreshold);
+  }
+  if (urlQuery.forecastsThreshold !== undefined) {
+    initialQueryParameters.forecastsThreshold = Number(
+      urlQuery.forecastsThreshold
+    );
+  }
+  if (urlQuery.forecastingPlatforms !== undefined) {
+    initialQueryParameters.forecastingPlatforms = String(
+      urlQuery.forecastingPlatforms
+    ).split("|");
+  }
+
+  const platformNameToLabel = Object.fromEntries(
+    platforms.map((platform) => [platform.name, platform.label])
+  );
+
+  const defaultNumDisplay = 21;
+  const initialNumDisplay = Number(urlQuery.numDisplay) || defaultNumDisplay;
+
+  const defaultResults = (await getFrontpage()).map((result) => ({
+    ...result,
+    platformLabel: platformNameToLabel[result.platform] || result.platform,
+  }));
 
   const initialResults =
     !!initialQueryParameters &&
     initialQueryParameters.query != "" &&
     initialQueryParameters.query != undefined
-      ? await searchAccordingToQueryData(initialQueryParameters)
+      ? await searchAccordingToQueryData(
+          initialQueryParameters,
+          initialNumDisplay
+        )
       : defaultResults;
 
   return {
     props: {
       initialQueryParameters,
+      defaultQueryParameters,
+      initialNumDisplay,
+      defaultNumDisplay,
       initialResults,
       defaultResults,
+      platformsConfig,
     },
   };
 };

src/web/worker/searchAccordingToQueryData.ts

@@ -1,56 +1,48 @@
+import { FrontendForecast } from "../platforms";
+import { QueryParameters } from "../search/anySearchPage";
 import searchGuesstimate from "./searchGuesstimate";
 import searchWithAlgolia from "./searchWithAlgolia";
 
-export default async function searchAccordingToQueryData(queryData) {
+export default async function searchAccordingToQueryData(
-  let results = [];
+  queryData: QueryParameters,
+  limit: number
+): Promise<FrontendForecast[]> {
+  let results: FrontendForecast[] = [];
 
   try {
     // defs
     let query = queryData.query == undefined ? "" : queryData.query;
-    if (query == "") return -1;
+    if (query == "") return [];
     let forecastsThreshold = queryData.forecastsThreshold;
     let starsThreshold = queryData.starsThreshold;
-    let forecastingPlatforms = queryData.forecastingPlatforms.map(
-      (x) => x.value
-    );
     let platformsIncludeGuesstimate =
-      forecastingPlatforms.includes("Guesstimate") && starsThreshold <= 1;
+      queryData.forecastingPlatforms.includes("guesstimate") &&
+      starsThreshold <= 1;
 
     // preparation
     let unawaitedAlgoliaResponse = searchWithAlgolia({
       queryString: query,
-      hitsPerPage: queryData.numDisplay + 50,
+      hitsPerPage: limit + 50,
       starsThreshold,
-      filterByPlatforms: forecastingPlatforms,
+      filterByPlatforms: queryData.forecastingPlatforms,
       forecastsThreshold,
     });
 
     // consider the guesstimate and the non-guesstimate cases separately.
-    switch (platformsIncludeGuesstimate) {
+    if (platformsIncludeGuesstimate) {
-      case false:
+      let responses = await Promise.all([
-        results = await unawaitedAlgoliaResponse;
+        unawaitedAlgoliaResponse,
-        break;
+        searchGuesstimate(query),
-      case true:
+      ]); // faster than two separate requests
-        let responses = await Promise.all([
+      let responsesNotGuesstimate = responses[0];
-          unawaitedAlgoliaResponse,
+      let responsesGuesstimate = responses[1];
-          searchGuesstimate(query),
+      results = [...responsesNotGuesstimate, ...responsesGuesstimate];
-        ]); // faster than two separate requests
+      //results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
-        let responsesNotGuesstimate = responses[0];
+    } else {
-        let responsesGuesstimate = responses[1];
+      results = await unawaitedAlgoliaResponse;
-        results = [...responsesNotGuesstimate, ...responsesGuesstimate];
-        //results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
-        break;
-      default:
-        return -1;
     }
-    // Maintain compatibility with fuse
-    let makeCompatibleWithFuse = (results) =>
-      results.map((result, index) => ({
-        item: result,
-        score: 0, // 0.4 - 0.4 / (index + 1),
-      }));
 
-    results = makeCompatibleWithFuse(results);
+    return results;
   } catch (error) {
     console.log(error);
   } finally {

src/web/worker/searchGuesstimate.ts

@@ -1,14 +1,18 @@
 /* Imports */
 import axios from "axios";
 
+import { FrontendForecast } from "../platforms";
+
 /* Definitions */
 let urlEndPoint =
   "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c";
 
 /* Body */
 
-export default function searchGuesstimate(query) {
+export default async function searchGuesstimate(
-  let response = axios({
+  query
+): Promise<FrontendForecast[]> {
+  let response = await axios({
     url: urlEndPoint,
     // credentials: "omit",
     headers: {
@@ -24,48 +28,45 @@ export default function searchGuesstimate(query) {
       "%20"
     )}&hitsPerPage=20&page=0&getRankingInfo=true\"}`,
     method: "POST",
-  })
+  });
-    .then((res) => res.data.hits)
-    .then((models) =>
-      models.map((model, index) => {
-        let description = model.description
-          ? model.description.replace(/\n/g, " ").replace(/  /g, " ")
-          : "";
-        let stars = description.length > 250 ? 2 : 1;
-        return {
-          title: model.name,
-          url: `https://www.getguesstimate.com/models/${model.id}`,
-          platform: "Guesstimate",
-          description: description,
-          options: [],
-          qualityindicators: {
-            stars: stars,
-            numforecasts: 1,
-            numforecasters: 1,
-          },
-          visualization: model.big_screenshot,
-          ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
-        };
-      })
-    )
-    .then((models) => {
-      // filter for duplicates. Surprisingly common.
-      let uniqueTitles = [];
-      let uniqueModels = [];
-      for (let model of models) {
-        if (
-          !uniqueTitles.includes(model.title) &&
-          !model.title.includes("copy")
-        ) {
-          uniqueModels.push(model);
-          uniqueTitles.push(model.title);
-        }
-      }
-      return uniqueModels;
-    });
 
-  console.log(response);
+  const models: any[] = response.data.hits;
-  return response; // This is a promise. Usable with either async/await (a mess in React) or with .then(guesstimateModels => doSomething(guesstimateModels))
+  const mappedModels: FrontendForecast[] = models.map((model, index) => {
+    let description = model.description
+      ? model.description.replace(/\n/g, " ").replace(/  /g, " ")
+      : "";
+    let stars = description.length > 250 ? 2 : 1;
+    return {
+      id: `guesstimate-${model.id}`,
+      title: model.name,
+      url: `https://www.getguesstimate.com/models/${model.id}`,
+      timestamp: model.created_at, // TODO - check that format matches
+      platform: "guesstimate",
+      platformLabel: "Guesstimate",
+      description: description,
+      options: [],
+      qualityindicators: {
+        stars: stars,
+        numforecasts: 1,
+        numforecasters: 1,
+      },
+      visualization: model.big_screenshot,
+      ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
+    };
+  });
+
+  // filter for duplicates. Surprisingly common.
+  let uniqueTitles = [];
+  let uniqueModels: FrontendForecast[] = [];
+  for (let model of mappedModels) {
+    if (!uniqueTitles.includes(model.title) && !model.title.includes("copy")) {
+      uniqueModels.push(model);
+      uniqueTitles.push(model.title);
+    }
+  }
+
+  console.log(uniqueModels);
+  return uniqueModels;
 }
 
 // searchGuesstimate("COVID-19").then(guesstimateModels => console.log(guesstimateModels))

src/web/worker/searchWithAlgolia.ts

@@ -1,5 +1,7 @@
 import algoliasearch from "algoliasearch";
 
+import { FrontendForecast } from "../platforms";
+
 const client = algoliasearch(
   process.env.NEXT_PUBLIC_ALGOLIA_APP_ID,
   process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_KEY
@@ -76,13 +78,12 @@ let noExactMatch = (queryString, result) => {
 // only query string
 export default async function searchWithAlgolia({
   queryString,
-  hitsPerPage,
+  hitsPerPage = 5,
   starsThreshold,
   filterByPlatforms,
   forecastsThreshold,
-}) {
+}): Promise<FrontendForecast[]> {
-  hitsPerPage = hitsPerPage || 5;
+  let response = await index.search<FrontendForecast>(queryString, {
-  let response = await index.search(queryString, {
     hitsPerPage,
     filters: buildFilter({
       starsThreshold,
@@ -92,12 +93,7 @@ export default async function searchWithAlgolia({
     //facetFilters: buildFacetFilter({filterByPlatforms}),
     getRankingInfo: true,
   });
-  let results: any[] = response.hits;
+  let results: FrontendForecast[] = response.hits;
-  console.log(
-    "searchWithAlgolia.js/searchWithAlgolia/queryString",
-    queryString
-  );
-  console.log("searchWithAlgolia.js/searchWithAlgolia/results", results);
 
   let recursionError = ["metaforecast", "metaforecasts", "metaforecasting"];
   if (
@@ -106,9 +102,11 @@ export default async function searchWithAlgolia({
   ) {
     results = [
       {
+        id: "not-found",
         title: "No search results match your query",
         url: "https://metaforecast.org",
-        platform: "Metaforecast",
+        platform: "metaforecast",
+        platformLabel: "Metaforecast",
         description: "Maybe try a broader query?",
         options: [
           {
@@ -128,17 +126,19 @@ export default async function searchWithAlgolia({
           numforecasters: 1,
           stars: 5,
         },
-        noExactSearchResults: true,
+        // noExactSearchResults: true,
-        optionsstringforsearch: "Yes, No",
+        // optionsstringforsearch: "Yes, No",
-        has_numforecasts: true,
+        // has_numforecasts: true,
       },
     ];
   } else if (recursionError.includes(queryString.toLowerCase())) {
     results = [
       {
+        id: "recursion-error",
         title: `Did you mean: ${queryString}?`,
         url: "https://metaforecast.org/recursion?bypassEasterEgg=true",
-        platform: "Metaforecast",
+        platform: "metaforecast",
+        platformLabel: "Metaforecast",
         description:
           "Fatal error: Too much recursion. Click to proceed anyways",
         options: [
@@ -159,9 +159,9 @@ export default async function searchWithAlgolia({
           numforecasters: 1,
           stars: 5,
         },
-        noExactSearchResults: true,
+        // noExactSearchResults: true,
-        optionsstringforsearch: "Yes, No",
+        // optionsstringforsearch: "Yes, No",
-        has_numforecasts: true,
+        // has_numforecasts: true,
       },
       ...results,
     ];
@@ -171,9 +171,11 @@ export default async function searchWithAlgolia({
     noExactMatch(queryString, results[0])
   ) {
     results.unshift({
+      id: "not-found-2",
       title: "No search results appear to match your query",
       url: "https://metaforecast.org",
-      platform: "Metaforecast",
+      platform: "metaforecast",
+      platformLabel: "Metaforecast",
       description: "Maybe try a broader query? That said, we could be wrong.",
       options: [
         {
@@ -193,12 +195,12 @@ export default async function searchWithAlgolia({
         numforecasters: 1,
         stars: 1,
       },
-      noExactSearchResults: true,
+      // noExactSearchResults: true,
-      optionsstringforsearch: "Yes, No",
+      // optionsstringforsearch: "Yes, No",
-      has_numforecasts: true,
+      // has_numforecasts: true,
     });
   } else {
-    results[0].noExactSearchResults = false;
+    // results[0].noExactSearchResults = false;
   }
 
   return results;