tweak: Add license, some small tweaks.

Unnecessary to expose the vercel url.

.gitignore

@@ -31,7 +31,7 @@ yarn-error.log*
 
 # yarn vs npm conflict
 package-lock.json ## use yarn.lock instead
-yarn.lock
+# yarn.lock
 
 # Local Netlify folder
 .netlify

LICENSE.md

@@ -0,0 +1,7 @@
+Copyright 2023 Quantified Uncertainty Research Institute.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -89,5 +89,22 @@ Overall, the services which we use are:
 
 ## Various notes
 
+- This repository is released under the [MIT license](https://opensource.org/licenses/MIT). See `LICENSE.md`
 - Commits follow [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/#summary)
 - For elicit and metaculus, this library currently filters out questions with <10 predictions.
+- The database is updated once a day, at 3:00 AM UTC, with the command `ts-node -T src/backend/flow/doEverythingForScheduler.ts`. The frontpage is updated after that, at 6:00 AM UTC with the command `ts-node -T src/backend/index.ts frontpage`. It's possible that either of these two operations makes the webpage briefly go down.
+
+## To do
+
+- [x] Update Metaculus and Manifold Markets fetchers
+- [x] Add markets from [Insight Prediction](https://insightprediction.com/).
+- [ ] Use <https://news.manifold.markets/p/above-the-fold-midterms-special> to update stars calculation for Manifold.
+- [ ] Add a few more snippets, with fetching individual questions, questions with histories, questions added within the last 24h to the /contrib folder (good first issue)
+- [ ] Refactor code so that users can capture and push the question history chart to imgur (good first issue)
+- [ ] Upgrade to [React 18](https://reactjs.org/blog/2022/03/08/react-18-upgrade-guide.html). This will require dealing with the workaround we used for [this issue](https://github.com/vercel/next.js/issues/36019#issuecomment-1103266481)
+- [ ] Add database of resolutions
+- [ ] Allow users to embed predictions in the EA Forum/LessWrong (in progress)
+- [ ] Find a long-term mantainer for this project
+- [ ] Allow users to record their own predictions
+- [ ] Release snapshots (I think @niplav is working on this)
+- [ ] ...

package.json

@@ -24,95 +24,101 @@
     "build": "prisma generate && next build",
     "next-start": "next start",
     "next-export": "next export",
-    "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES"
+    "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES",
+    "upgrade-interactive": "yarn upgrade-interactive --latest"
   },
   "dependencies": {
-    "@floating-ui/react-dom": "^0.7.0",
-    "@graphql-yoga/node": "^2.1.0",
-    "@pothos/core": "^3.5.1",
-    "@pothos/plugin-prisma": "^3.4.0",
-    "@pothos/plugin-relay": "^3.10.0",
-    "@prisma/client": "^3.11.1",
-    "@quri/squiggle-lang": "^0.2.11",
-    "@tailwindcss/forms": "^0.4.0",
-    "@tailwindcss/typography": "^0.5.1",
-    "@types/chroma-js": "^2.1.3",
+    "@floating-ui/react-dom": "^0.7.2",
+    "@graphql-yoga/plugin-response-cache": "^1.1.0",
+    "@pothos/core": "^3.22.8",
+    "@pothos/plugin-prisma": "^3.35.6",
+    "@pothos/plugin-relay": "^3.28.6",
+    "@prisma/client": "^3.15.2",
+    "@quri/squiggle-lang": "^0.5.1",
+    "@tailwindcss/forms": "^0.4.1",
+    "@tailwindcss/typography": "^0.5.7",
+    "@types/chroma-js": "^2.1.4",
     "@types/dom-to-image": "^2.6.4",
-    "@types/google-spreadsheet": "^3.2.1",
-    "@types/jsdom": "^16.2.14",
+    "@types/google-spreadsheet": "^3.3.0",
+    "@types/jsdom": "^16.2.15",
     "@types/nprogress": "^0.2.0",
-    "@types/react": "^17.0.39",
-    "@types/react-copy-to-clipboard": "^5.0.2",
+    "@types/react": "<18.0.0",
+    "@types/react-copy-to-clipboard": "^5.0.4",
     "@types/textversionjs": "^1.1.1",
     "@types/tunnel": "^0.0.3",
-    "airtable": "^0.11.1",
+    "airtable": "^0.11.5",
     "ajv": "^8.11.0",
-    "algoliasearch": "^4.10.3",
-    "autoprefixer": "^10.1.0",
-    "axios": "^0.25.0",
+    "algoliasearch": "^4.14.2",
+    "autoprefixer": "10.4.5",
+    "axios": "^1.2.0",
     "chroma-js": "^2.4.2",
     "critters": "^0.0.16",
-    "date-fns": "^2.28.0",
+    "date-fns": "^2.29.3",
     "dom-to-image": "^2.6.0",
-    "dotenv": "^16.0.0",
+    "dotenv": "^16.0.3",
     "fetch": "^1.1.0",
     "fs": "^0.0.1-security",
-    "fuse.js": "^6.4.6",
-    "google-spreadsheet": "^3.1.15",
-    "graphql": "^16.3.0",
-    "graphql-request": "^4.0.0",
-    "html-to-image": "^1.7.0",
+    "fuse.js": "^6.6.2",
+    "google-spreadsheet": "^3.3.0",
+    "graphql": "^16.6.0",
+    "graphql-request": "^5.0.0",
+    "graphql-yoga": "^3.0.0-next.10",
+    "html-to-image": "^1.10.8",
     "https": "^1.0.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^19.0.0",
-    "json2csv": "^5.0.5",
-    "multiselect-react-dropdown": "^2.0.17",
-    "next": "12",
-    "next-plausible": "^3.1.6",
-    "next-urql": "^3.3.2",
+    "json2csv": "^5.0.7",
+    "multiselect-react-dropdown": "^2.0.25",
+    "next": "^12.3.1",
+    "next-plausible": "^3.6.3",
+    "next-urql": "^3.3.3",
     "nprogress": "^0.2.0",
-    "open": "^7.3.1",
-    "papaparse": "^5.3.0",
-    "pg": "^8.7.3",
-    "postcss": "^8.2.1",
+    "open": "^7.4.2",
+    "papaparse": "^5.3.2",
+    "pg": "^8.8.0",
+    "postcss": "^8.4.18",
     "postcss-flexbugs-fixes": "^5.0.2",
-    "postcss-preset-env": "^7.3.2",
-    "prisma": "^3.11.1",
+    "postcss-preset-env": "^7.8.2",
+    "prisma": "^3.15.2",
     "query-string": "^7.1.1",
     "re-resizable": "^6.9.9",
     "react": "^17.0.2",
     "react-component-export-image": "^1.0.6",
-    "react-compound-slider": "^3.3.1",
-    "react-copy-to-clipboard": "^5.0.3",
+    "react-compound-slider": "^3.4.0",
+    "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^17.0.2",
-    "react-dropdown": "^1.9.2",
-    "react-hook-form": "^7.27.0",
-    "react-icons": "^4.2.0",
-    "react-is": "^18.0.0",
-    "react-markdown": "^8.0.0",
+    "react-dropdown": "^1.11.0",
+    "react-hook-form": "^7.38.0",
+    "react-icons": "^4.6.0",
+    "react-is": "^18.2.0",
+    "react-markdown": "^8.0.3",
     "react-safe": "^1.3.0",
-    "react-select": "^5.2.2",
+    "react-select": "^5.5.4",
     "remark-gfm": "^3.0.1",
-    "tabletojson": "^2.0.4",
-    "tailwindcss": "^3.0.22",
+    "tabletojson": "^2.0.7",
+    "tailwindcss": "^3.2.0",
     "textversionjs": "^1.1.3",
-    "ts-node": "^10.7.0",
+    "ts-node": "^10.9.1",
     "tunnel": "^0.0.6",
-    "urql": "^2.2.0",
-    "urql-custom-scalars-exchange": "^0.1.5",
-    "victory": "^36.3.2"
+    "urql": "^2.2.3",
+    "urql-custom-scalars-exchange": "^0.1.6",
+    "victory": "^36.6.8"
+  },
+  "resolutions": {
+    "@types/react": "<18.0.0"
   },
   "devDependencies": {
-    "@graphql-codegen/cli": "^2.6.2",
-    "@graphql-codegen/introspection": "^2.1.1",
-    "@graphql-codegen/near-operation-file-preset": "^2.2.9",
-    "@graphql-codegen/schema-ast": "^2.4.1",
-    "@graphql-codegen/typed-document-node": "^2.2.8",
-    "@graphql-codegen/typescript": "^2.4.8",
-    "@graphql-codegen/typescript-operations": "^2.3.5",
-    "@netlify/plugin-nextjs": "^4.2.4",
-    "@svgr/cli": "^6.2.1",
+    "@graphql-codegen/cli": "^2.13.7",
+    "@graphql-codegen/introspection": "^2.2.1",
+    "@graphql-codegen/near-operation-file-preset": "^2.4.3",
+    "@graphql-codegen/schema-ast": "^2.5.1",
+    "@graphql-codegen/typed-document-node": "^2.3.5",
+    "@graphql-codegen/typescript": "^2.7.5",
+    "@graphql-codegen/typescript-operations": "^2.5.5",
+    "@svgr/cli": "^6.5.0",
     "@types/pg": "^8.6.5",
-    "netlify-cli": "^9.13.6"
+    "eslint": "^8.25.0",
+    "eslint-config-next": "^12.3.1",
+    "typescript": "4.9.3"
   }
 }

src/Global.d.ts

@@ -0,0 +1,8 @@
+// Workaround related to: https://github.com/vercel/next.js/issues/29788
+// https://github.com/vercel/next.js/issues/29788#issuecomment-1000595524
+declare type StaticImageData = {
+  src: string;
+  height: number;
+  width: number;
+  placeholder?: string;
+};

src/backend/platforms/goodjudgment.ts

@@ -1,10 +1,11 @@
 /* Imports */
 import axios from "axios";
-import { Tabletojson } from "tabletojson";
+import {Tabletojson} from "tabletojson";
 
-import { average } from "../../utils";
-import { hash } from "../utils/hash";
-import { FetchedQuestion, Platform } from "./";
+import {average} from "../../utils";
+import {hash} from "../utils/hash";
+import {FetchedQuestion, Platform} from "./";
+import {FullQuestionOption} from "../../common/types";
 
 /* Definitions */
 const platformName = "goodjudgment";
@@ -30,32 +31,30 @@ export const goodjudgment: Platform = {
     // hard-coded backup proxy
 		*/
     // proxy = {
-    //   ip: process.env.BACKUP_PROXY_IP,
-    //   port: process.env.BACKUP_PROXY_PORT,
+    // ip: process.env.BACKUP_PROXY_IP,
+    // port: process.env.BACKUP_PROXY_PORT,
     // };
     // // }
     // let agent = tunnel.httpsOverHttp({
-    //   proxy: {
+    // proxy: {
     //     host: proxy.ip,
     //     port: proxy.port,
-    //   },
+    // },
     // });
 
-    const content = await axios
-      .request({
-        url: "https://goodjudgment.io/superforecasts/",
-        method: "get",
-        headers: {
-          "User-Agent": "Chrome",
-        },
-        // agent,
-        // port: 80,
-      })
-      .then((query) => query.data);
+    const content = await axios.request({
+      url: "https://goodjudgment.io/superforecasts/",
+      method: "get",
+      headers: {
+        "User-Agent": "Chrome"
+      },
+      // agent,
+      // port: 80,
+    }).then((query) => query.data);
 
     // Processing
     let results: FetchedQuestion[] = [];
-    let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false });
+    let jsonTable = Tabletojson.convert(content, {stripHtmlFromCells: false});
     jsonTable.shift(); // deletes first element
     jsonTable.pop(); // deletes last element
 
@@ -63,38 +62,21 @@ export const goodjudgment: Platform = {
       let title = table[0]["0"].split("\t\t\t").splice(3)[0];
       if (title != undefined) {
         title = title.replaceAll("</a>", "");
-        const id = `${platformName}-${hash(title)}`;
-        const description = table
-          .filter((row: any) => row["0"].includes("BACKGROUND:"))
-          .map((row: any) => row["0"])
-          .map((text: any) =>
-            text
-              .split("BACKGROUND:")[1]
-              .split("Examples of Superforecaster")[0]
-              .split("AT A GLANCE")[0]
-              .replaceAll("\n\n", "\n")
-              .split("\n")
-              .slice(3)
-              .join(" ")
-              .replaceAll("      ", "")
-              .replaceAll("<br> ", "")
-          )[0];
-        const options = table
-          .filter((row: any) => "4" in row)
-          .map((row: any) => ({
-            name: row["2"]
-              .split('<span class="qTitle">')[1]
-              .replace("</span>", ""),
-            probability: Number(row["3"].split("%")[0]) / 100,
-            type: "PROBABILITY",
-          }));
-        let analysis = table.filter((row: any) =>
-          row[0] ? row[0].toLowerCase().includes("commentary") : false
-        );
+        const id = `${platformName}-${
+          hash(title)
+        }`;
+        const description = table.filter((row : any) => row["0"].includes("BACKGROUND:")).map((row : any) => row["0"]).map((text : any) => text.split("BACKGROUND:")[1].split("Examples of Superforecaster")[0].split("AT A GLANCE")[0].replaceAll("\n\n", "\n").split("\n").slice(3).join(" ").replaceAll("      ", "").replaceAll("<br> ", ""))[0];
+        const options = table.filter((row : any) => "4" in row).map((row : any) => ({
+          name: row["2"].split('<span class="qTitle">')[1].replace("</span>", ""),
+          probability: Number(row["3"].split("%")[0]) / 100,
+          type: "PROBABILITY"
+        }));
+        let analysis = table.filter((row : any) => row[0] ? row[0].toLowerCase().includes("commentary") : false);
         // "Examples of Superforecaster Commentary" / Analysis
         // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list.
         analysis = analysis ? analysis[0] : "";
-        analysis = analysis ? analysis[0] : ""; // not a duplicate
+        analysis = analysis ? analysis[0] : "";
+        // not a duplicate
         // console.log(analysis)
         let standardObj: FetchedQuestion = {
           id,
@@ -104,16 +86,14 @@ export const goodjudgment: Platform = {
           options,
           qualityindicators: {},
           extra: {
-            superforecastercommentary: analysis || "",
-          },
+            superforecastercommentary: analysis || ""
+          }
         };
         results.push(standardObj);
       }
     }
 
-    console.log(
-      "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js"
-    );
+    console.log("Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js");
 
     return results;
   },
@@ -121,8 +101,8 @@ export const goodjudgment: Platform = {
     let nuno = () => 4;
     let eli = () => 4;
     let misha = () => 3.5;
-    let starsDecimal = average([nuno()]); //, eli(), misha()])
+    let starsDecimal = average([nuno()]); // , eli(), misha()])
     let starsInteger = Math.round(starsDecimal);
     return starsInteger;
-  },
+  }
 };

src/backend/platforms/goodjudgmentopen.ts

@@ -1,12 +1,13 @@
 /* Imports */
 import axios from "axios";
-import { Tabletojson } from "tabletojson";
+import {Tabletojson} from "tabletojson";
 
-import { average } from "../../utils";
-import { applyIfSecretExists } from "../utils/getSecrets";
-import { sleep } from "../utils/sleep";
+import {average} from "../../utils";
+import {applyIfSecretExists} from "../utils/getSecrets";
+import {sleep} from "../utils/sleep";
 import toMarkdown from "../utils/toMarkdown";
-import { FetchedQuestion, Platform } from "./";
+import {FetchedQuestion, Platform} from "./";
+import {FullQuestionOption} from "../../common/types";
 
 /* Definitions */
 const platformName = "goodjudgmentopen";
@@ -24,33 +25,33 @@ const id = () => 0;
 
 /* Support functions */
 
-function cleanDescription(text: string) {
+function cleanDescription(text : string) {
   let md = toMarkdown(text);
   let result = md.replaceAll("---", "-").replaceAll("  ", " ");
   return result;
 }
 
-async function fetchPage(page: number, cookie: string) {
+async function fetchPage(page : number, cookie : string) {
   const response: string = await axios({
     url: htmlEndPoint + page,
     method: "GET",
     headers: {
-      Cookie: cookie,
-    },
+      Cookie: cookie
+    }
   }).then((res) => res.data);
-  //console.log(response)
+  // console.log(response)
   return response;
 }
 
-async function fetchStats(questionUrl: string, cookie: string) {
+async function fetchStats(questionUrl : string, cookie : string) {
   let response: string = await axios({
     url: questionUrl + "/stats",
     method: "GET",
     headers: {
       "Content-Type": "text/html",
       Cookie: cookie,
-      Referer: questionUrl,
-    },
+      Referer: questionUrl
+    }
   }).then((res) => res.data);
 
   if (response.includes("Sign up or sign in to forecast")) {
@@ -61,9 +62,7 @@ async function fetchStats(questionUrl: string, cookie: string) {
 
   // Parse the embedded json
   let htmlElements = response.split("\n");
-  let jsonLines = htmlElements.filter((element) =>
-    element.includes("data-react-props")
-  );
+  let jsonLines = htmlElements.filter((element) => element.includes("data-react-props"));
   let embeddedJsons = jsonLines.map((jsonLine, i) => {
     let innerJSONasHTML = jsonLine.split('data-react-props="')[1].split('"')[0];
     let json = JSON.parse(innerJSONasHTML.replaceAll(""", '"'));
@@ -76,27 +75,11 @@ async function fetchStats(questionUrl: string, cookie: string) {
   let numforecasters = firstEmbeddedJson.question.predictors_count;
   let numforecasts = firstEmbeddedJson.question.prediction_sets_count;
   let questionType = firstEmbeddedJson.question.type;
-  if (
-    questionType.includes("Binary") ||
-    questionType.includes("NonExclusiveOpinionPoolQuestion") ||
-    questionType.includes("Forecast::Question") ||
-    !questionType.includes("Forecast::MultiTimePeriodQuestion")
-  ) {
-    options = firstEmbeddedJson.question.answers.map((answer: any) => ({
-      name: answer.name,
-      probability: answer.normalized_probability,
-      type: "PROBABILITY",
-    }));
+  if (questionType.includes("Binary") || questionType.includes("NonExclusiveOpinionPoolQuestion") || questionType.includes("Forecast::Question") || ! questionType.includes("Forecast::MultiTimePeriodQuestion")) {
+    options = firstEmbeddedJson.question.answers.map((answer : any) => ({name: answer.name, probability: answer.normalized_probability, type: "PROBABILITY"}));
     if (options.length == 1 && options[0].name == "Yes") {
-      let probabilityNo =
-        options[0].probability > 1
-          ? 1 - options[0].probability / 100
-          : 1 - options[0].probability;
-      options.push({
-        name: "No",
-        probability: probabilityNo,
-        type: "PROBABILITY",
-      });
+      let probabilityNo = options[0].probability > 1 ? 1 - options[0].probability / 100 : 1 - options[0].probability;
+      options.push({name: "No", probability: probabilityNo, type: "PROBABILITY"});
     }
   }
   let result = {
@@ -105,30 +88,28 @@ async function fetchStats(questionUrl: string, cookie: string) {
     qualityindicators: {
       numforecasts: Number(numforecasts),
       numforecasters: Number(numforecasters),
-      comments_count: Number(comments_count),
-    },
+      comments_count: Number(comments_count)
+    }
   };
   // console.log(JSON.stringify(result, null, 4));
   return result;
 }
 
-function isSignedIn(html: string) {
-  let isSignedInBool = !(
-    html.includes("You need to sign in or sign up before continuing") ||
-    html.includes("Sign up")
-  );
+function isSignedIn(html : string) {
+  let isSignedInBool = !(html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up"));
   // console.log(html)
-  if (!isSignedInBool) {
+  if (! isSignedInBool) {
     console.log("Error: Not signed in.");
   }
-  console.log(`is signed in? ${isSignedInBool ? "yes" : "no"}`);
+  console.log(`is signed in? ${
+    isSignedInBool ? "yes" : "no"
+  }`);
   return isSignedInBool;
 }
 
-function reachedEnd(html: string) {
+function reachedEnd(html : string) {
   let reachedEndBool = html.includes("No questions match your filter");
-  if (reachedEndBool) {
-    //console.log(html)
+  if (reachedEndBool) { // console.log(html)
   }
   console.log(`Reached end? ${reachedEndBool}`);
   return reachedEndBool;
@@ -136,7 +117,7 @@ function reachedEnd(html: string) {
 
 /* Body */
 
-async function goodjudgmentopen_inner(cookie: string) {
+async function goodjudgmentopen_inner(cookie : string) {
   let i = 1;
   let response = await fetchPage(i, cookie);
 
@@ -144,7 +125,7 @@ async function goodjudgmentopen_inner(cookie: string) {
   let init = Date.now();
   // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   console.log("Page #1")
-  while (!reachedEnd(response) && isSignedIn(response)) {
+  while (! reachedEnd(response) && isSignedIn(response)) {
     let htmlLines = response.split("\n");
     DEBUG_MODE == "on" ? htmlLines.forEach((line) => console.log(line)) : id();
     let h5elements = htmlLines.filter((str) => str.includes("<h5> <a href="));
@@ -153,20 +134,19 @@ async function goodjudgmentopen_inner(cookie: string) {
     for (let h5element of h5elements) {
       let h5elementSplit = h5element.split('"><span>');
       let url = h5elementSplit[0].split('<a href="')[1];
-      if (!annoyingPromptUrls.includes(url)) {
+      if (! annoyingPromptUrls.includes(url)) {
         let title = h5elementSplit[1].replace("</span></a></h5>", "");
         await sleep(1000 + Math.random() * 1000); // don't be as noticeable
         try {
           let moreinfo = await fetchStats(url, cookie);
-          if (moreinfo.isbinary) {
-            if (!moreinfo.crowdpercentage) {
-              // then request again.
+          /*if (moreinfo.isbinary) {
+            if (! moreinfo.crowdpercentage) { // then request again.
               moreinfo = await fetchStats(url, cookie);
             }
-          }
+          }*/
           let questionNumRegex = new RegExp("questions/([0-9]+)");
           const questionNumMatch = url.match(questionNumRegex);
-          if (!questionNumMatch) {
+          if (! questionNumMatch) {
             throw new Error(`Couldn't find question num in ${url}`);
           }
           let questionNum = questionNumMatch[1];
@@ -176,21 +156,19 @@ async function goodjudgmentopen_inner(cookie: string) {
             title: title,
             url: url,
             platform: platformName,
-            ...moreinfo,
+            ... moreinfo
           };
           if (j % 30 == 0 || DEBUG_MODE == "on") {
             console.log(`Page #${i}`);
             console.log(question);
-          }else{
+          } else {
             console.log(question.title)
           }
           // console.log(question)
           results.push(question);
         } catch (error) {
           console.log(error);
-          console.log(
-            `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
-          );
+          console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`);
         }
       }
       j = j + 1;
@@ -203,9 +181,7 @@ async function goodjudgmentopen_inner(cookie: string) {
       response = await fetchPage(i, cookie);
     } catch (error) {
       console.log(error);
-      console.log(
-        `We encountered some error when fetching page #${i}, so it won't appear on the final json`
-      );
+      console.log(`We encountered some error when fetching page #${i}, so it won't appear on the final json`);
     }
   }
 
@@ -216,9 +192,11 @@ async function goodjudgmentopen_inner(cookie: string) {
 
   let end = Date.now();
   let difference = end - init;
-  console.log(
-    `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
-  );
+  console.log(`Took ${
+    difference / 1000
+  } seconds, or ${
+    difference / (1000 * 60)
+  } minutes.`);
 
   return results;
 }
@@ -230,23 +208,18 @@ export const goodjudgmentopen: Platform = {
   version: "v1",
   async fetcher() {
     let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
-    return (await applyIfSecretExists(cookie, goodjudgmentopen_inner)) || null;
+    return(await applyIfSecretExists(cookie, goodjudgmentopen_inner)) || null;
   },
   calculateStars(data) {
-    let minProbability = Math.min(
-      ...data.options.map((option) => option.probability || 0)
-    );
-    let maxProbability = Math.max(
-      ...data.options.map((option) => option.probability || 0)
-    );
+    let minProbability = Math.min(...data.options.map((option) => option.probability || 0));
+    let maxProbability = Math.max(...data.options.map((option) => option.probability || 0));
 
     let nuno = () => ((data.qualityindicators.numforecasts || 0) > 100 ? 3 : 2);
     let eli = () => 3;
-    let misha = () =>
-      minProbability > 0.1 || maxProbability < 0.9 ? 3.1 : 2.5;
+    let misha = () => minProbability > 0.1 || maxProbability < 0.9 ? 3.1 : 2.5;
 
     let starsDecimal = average([nuno(), eli(), misha()]);
     let starsInteger = Math.round(starsDecimal);
     return starsInteger;
-  },
+  }
 };

src/backend/platforms/insight.ts

@@ -1,126 +1,339 @@
 /* Imports */
+import {or} from "ajv/dist/compile/codegen";
 import axios from "axios";
 
 import {FetchedQuestion, Platform} from ".";
+import {QuestionOption} from "../../common/types";
+import toMarkdown from "../utils/toMarkdown";
+import { average } from "../../utils";
 
 /* Definitions */
 const platformName = "insight";
-const marketsEnpoint = "https://insightprediction.com/api/markets";
+const marketsEnpoint = "https://insightprediction.com/api/markets?orderBy=is_resolved&sortedBy=asc";
 const getMarketEndpoint = (id : number) => `https://insightprediction.com/api/markets/${id}`;
+const SPORTS_CATEGORIES = [
+  'World Cup',
+  'MLB',
+  'Futures',
+  'Sports',
+  'EPL',
+  'Golf',
+  'NHL',
+  'College Football'
+]
 
 /* Support functions */
 
-async function fetchQuestionStats(bearer: string, marketId: number) {
-    const response = await axios({
-        url: getMarketEndpoint(marketId),
-        method: "GET",
-        headers: {
-            "Content-Type": "application/json",
-            Accept: "application/json",
-            Authorization: `Bearer ${bearer}`
-        }
-    }).then((res) => res.data);
-    // console.log(response)
-    return response;
+// Stubs
+const excludeMarketFromTitle = (title : any) => {
+  if (!!title) {
+    return title.includes(" vs ") || title.includes(" Over: ") || title.includes("NFL") || title.includes("Will there be a first time winner") || title.includes("Premier League")
+  } else {
+    return true
+  }
 }
 
+const hasActiveYesNoOrderBook = (orderbook : any) => {
+  if (!!orderbook) {
+    let yes = !!orderbook.yes && !!orderbook.yes.buy && Array.isArray(orderbook.yes.buy) && orderbook.yes.buy.length != 0 && !!orderbook.yes.buy[0].price && !!orderbook.yes.sell && Array.isArray(orderbook.yes.sell) && orderbook.yes.sell.length != 0 && !!orderbook.yes.sell[0].price
+    let no = !!orderbook.no && !!orderbook.no.buy && Array.isArray(orderbook.no.buy) && orderbook.no.buy.length != 0 && !!orderbook.no.buy[0].price && !!orderbook.no.sell && Array.isArray(orderbook.no.sell) && orderbook.no.sell.length != 0 && !!orderbook.no.sell[0].price
+    return yes && no
+  } else {
+    return false
+  }
+}
+
+const isBinaryQuestion = (data : any) => Array.isArray(data) && data.length == 1
+
+const geomMean = (a : number, b : number) => Math.sqrt(a * b)
+
+const processRelativeUrls = (a : string) => a.replaceAll("] (/", "](http://insightprediction.com/").replaceAll("](/", "](http://insightprediction.com/")
+
+const processDescriptionText = (text : any) => {
+  if (typeof text === 'string') {
+    return processRelativeUrls(toMarkdown(text))
+  } else {
+    return ""
+  }
+}
+
+const getOrderbookPrize = (orderbook : any) => {
+  let yes_min_cents = orderbook.yes.buy[0].price
+  let yes_max_cents = orderbook.yes.sell[0].price
+  let yes_min = Number(yes_min_cents.slice(0, -1))
+  let yes_max = Number(yes_max_cents.slice(0, -1))
+  let yes_price_orderbook = geomMean(yes_min, yes_max)
+  return yes_price_orderbook
+}
+
+const getAnswerProbability = (answer : any) => {
+  let orderbook = answer.orderbook
+  let latest_yes_price = answer.latest_yes_price
+  if (!! orderbook && hasActiveYesNoOrderBook(orderbook)) {
+    let yes_price_orderbook = getOrderbookPrize(orderbook)
+    let yes_probability = (latest_yes_price ? geomMean(latest_yes_price, yes_price_orderbook) : yes_price_orderbook) / 100
+    return yes_probability
+  } else if (!! latest_yes_price) {
+    return latest_yes_price / 100
+  } else {
+    return -1
+  }
+}
+
+// Fetching
 async function fetchPage(bearer: string, pageNum: number) {
-    const response = await axios({
-        url: `${marketsEnpoint}?page=${pageNum}`, // &orderBy=is_resolved&sortedBy=desc`,
-        method: "GET",
-        headers: {
-            "Content-Type": "application/json",
-            Accept: "application/json",
-            Authorization: `Bearer ${bearer}`
-        }
-    }).then((res) => res.data);
-    // console.log(response);
-    return response;
-}
-
-async function fetchData(bearer: string) {
-    let pageNum = 1;
-    let reachedEnd = false;
-    let results = [];
-    while (! reachedEnd) {
-        let newPage = await fetchPage(bearer, pageNum);
-        let newPageData = newPage.data;
-        let marketsFromPage = []
-        for (let market of newPageData) {
-            let response = await fetchQuestionStats(bearer, market.id);
-            let marketData = response.data
-            let marketAnswer = marketData.answer.data
-            delete marketData.answer
-            // These are the options and their prices.
-            let marketOptions = marketAnswer.map(answer => {
-                return({name: answer.title, probability: answer.latest_yes_price, type: "PROBABILITY"})
-            })
-            marketsFromPage.push({
-                ... marketData,
-                options: marketOptions
-            });
-        }
-
-        let finalObject = marketsFromPage
-
-        console.log(`Page = #${pageNum}`);
-        // console.log(newPageData)
-        // console.dir(finalObject, {depth: null});
-        results.push(... finalObject);
-
-        let newPagination = newPage.meta.pagination;
-        if (newPagination.total_pages == pageNum) {
-            reachedEnd = true;
-        } else {
-            pageNum = pageNum + 1;
-        }
+  let pageUrl = `${marketsEnpoint}&page=${pageNum}`
+  const response = await axios({
+    url: pageUrl, // &orderBy=is_resolved&sortedBy=desc`,
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Bearer ${bearer}`
     }
-    return results
+  }).then((res) => res.data);
+  // console.log(response);
+  return response;
 }
 
-async function processPredictions(predictions: any[]) {
-    let filteredPredictions = predictions.filter(prediction => !prediction.is_resolved && prediction.category != "Sports")
-    let results = filteredPredictions.map((prediction) => {
-        const id = `${platformName}-${
-            prediction.id
-        }`;
-        const options: FetchedQuestion["options"] = prediction.options
-        const result: FetchedQuestion = {
-            id,
-            title: prediction.title,
-            url: `https:${
-                prediction.url
-            }`,
-            description: prediction.rules,
-            options,
-            qualityindicators: {
-                volume: prediction.volume,
-                createdTime: prediction.created_at
-                // other: prediction.otherx,
-                // indicators: prediction.indicatorx,
-            }
-        };
-        return result;
-    });
-    // Filter results
-    return results; // resultsProcessed
+async function fetchMarket(bearer: string, marketId: number) {
+  const response = await axios({
+    url: getMarketEndpoint(marketId),
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Bearer ${bearer}`
+    }
+  }).then((res) => res.data);
+  // console.log(response)
+  return response;
+
 }
 
+const processMarket = (market : any) => {
+  let options: FetchedQuestion["options"] = []
+
+  if (!!market && !!market.answer && !!market.answer.data) {
+    let data = market.answer.data
+    if (isBinaryQuestion(data)) { // Binary questions
+      let answer = data[0]
+      let probability = getAnswerProbability(answer)
+      if (probability != -1) {
+        options = [
+          {
+            name: "Yes",
+            probability: probability,
+            type: "PROBABILITY"
+          }, {
+            name: "No",
+            probability: 1 - probability,
+            type: "PROBABILITY"
+          },
+        ];
+      } 
+    } else { // non binary question
+      for (let answer of data) {
+        let probability = getAnswerProbability(answer)
+        if (probability != -1) {
+          let newOption: QuestionOption = ({
+            name: String(answer.title),
+            probability: probability,
+            type: "PROBABILITY"
+          });
+          options.push(newOption)
+        }
+      }
+    }
+    if (!! options && Array.isArray(options) && options.length > 0) {
+      const id = `${platformName}-${
+        market.id
+      }`
+      const result: FetchedQuestion = {
+        id: id,
+        title: market.title,
+        url: market.url,
+        description: processDescriptionText(market.rules),
+        options,
+        qualityindicators: market.coin_id == "USD" ? (
+          {volume: market.volume}
+        ) : ({})
+      };
+      return result;
+    }
+  } 
+  return null
+}
+
+async function fetchAllMarkets(bearer: string) {
+  let pageNum = 1
+  let markets = []
+  let categories = []
+  let isEnd = false
+  while (! isEnd) {
+    if(pageNum % 20 == 0){
+      console.log(`Fetching page #${pageNum}`) // : ${pageUrl}
+    }
+    let page = await fetchPage(bearer, pageNum)
+    // console.log(JSON.stringify(page, null, 2))
+    let data = page.data
+    if (!! data && Array.isArray(data) && data.length > 0) {
+      let lastMarket = data[data.length - 1]
+      let isLastMarketResolved = lastMarket.is_resolved
+      if (isLastMarketResolved == true) {
+        isEnd = true
+      }
+      let newMarkets = data.filter(market => !market.is_resolved && !market.is_expired && ! excludeMarketFromTitle(market.title))
+      for (let initMarketData of newMarkets) {
+        let fullMarketDataResponse = await fetchMarket(bearer, initMarketData.id)
+        let fullMarketData = fullMarketDataResponse.data
+        let processedMarketData = processMarket(fullMarketData)
+
+        if (processedMarketData != null && ! SPORTS_CATEGORIES.includes(fullMarketData.category)) {
+          console.log(`- Adding: ${
+            fullMarketData.title
+          }`)
+          console.group()
+          console.log(fullMarketData)
+          console.log(JSON.stringify(processedMarketData, null, 2))
+          console.groupEnd()
+
+          markets.push(processedMarketData)
+        }
+
+        let category = fullMarketData.category
+        categories.push(category)
+
+      }
+    } else {
+      isEnd = true
+    } pageNum = pageNum + 1
+  }
+  console.log(markets)
+  console.log(categories)
+  return markets
+}
+/*
+async function fetchQuestionStats(bearer : string, marketId : number) {
+  const response = await axios({
+    url: getMarketEndpoint(marketId),
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Bearer ${bearer}`
+    }
+  }).then((res) => res.data);
+  // console.log(response)
+  return response;
+}
+
+
+
+async function fetchData(bearer : string) {
+  let pageNum = 1;
+  let reachedEnd = false;
+  let results = [];
+  while (! reachedEnd) {
+    let newPage = await fetchPage(bearer, pageNum);
+    let newPageData = newPage.data;
+    let marketsFromPage = []
+    for (let market of newPageData) {
+      let response = await fetchQuestionStats(bearer, market.id);
+      let marketData = response.data
+      let marketAnswer = marketData.answer.data
+      delete marketData.answer
+      // These are the options and their prices.
+      let marketOptions = marketAnswer.map(answer => {
+        return({name: answer.title, probability: answer.latest_yes_price, type: "PROBABILITY"})
+      })
+      marketsFromPage.push({
+        ... marketData,
+        options: marketOptions
+      });
+    }
+
+    let finalObject = marketsFromPage
+
+    console.log(`Page = #${pageNum}`);
+    // console.log(newPageData)
+    console.dir(finalObject, {depth: null});
+    results.push(... finalObject);
+
+    let newPagination = newPage.meta.pagination;
+    if (newPagination.total_pages == pageNum) {
+      reachedEnd = true;
+    } else {
+      pageNum = pageNum + 1;
+    }
+  }
+  return results
+}
+
+async function processPredictions(predictions : any[]) {
+  let results = await predictions.map((prediction) => {
+    const id = `${platformName}-${
+      prediction.id
+    }`;
+    const probability = prediction.probability;
+    const options: FetchedQuestion["options"] = [
+      {
+        name: "Yes",
+        probability: probability,
+        type: "PROBABILITY"
+      }, {
+        name: "No",
+        probability: 1 - probability,
+        type: "PROBABILITY"
+      },
+    ];
+    const result: FetchedQuestion = {
+      id,
+      title: prediction.title,
+      url: "https://example.com",
+      description: prediction.description,
+      options,
+      qualityindicators: {
+        // other: prediction.otherx,
+        // indicators: prediction.indicatorx,
+      }
+    };
+    return result;
+  });
+  return results; // resultsProcessed
+}
+*/
 /* Body */
-
 export const insight: Platform = {
-    name: platformName,
-    label: "Insight Prediction",
-    color: "#ff0000",
-    version: "v1",
-    async fetcher() {
-        let bearer = process.env.INSIGHT_BEARER;
-        let data = await fetchData(bearer);
-        let results = await processPredictions(data);
-        console.log(results);
-        return results;
-    },
-    calculateStars(data) {
-        return 2;
+  name: platformName,
+  label: "Insight Prediction",
+  color: "#ff0000",
+  version: "v1",
+  async fetcher() {
+    let bearer = process.env.INSIGHT_BEARER;
+    if (!! bearer) {
+      let data = await fetchAllMarkets(bearer);
+      return data
+    } else {
+      throw Error("No INSIGHT_BEARER available in environment")
     }
+    // let results: FetchedQuestion[] = []; // await processPredictions(data); // somehow needed
+    // return results;
+  },
+  calculateStars(data) {
+    let nuno = () => {
+      if((data.qualityindicators.volume || 0) > 10000){
+        return 4
+      } else if((data.qualityindicators.volume || 0) > 1000){
+        return 3
+      } else{
+        return 2
+      }
+    }
+    let eli = () => null;
+    let misha = () => null;
+    let starsDecimal = average([nuno()]); //, eli(data), misha(data)])
+    let starsInteger = Math.round(starsDecimal);
+    return starsInteger;
+  }
 };

src/backend/platforms/manifold.ts

@@ -6,12 +6,12 @@ import { FetchedQuestion, Platform } from "./";
 
 /* Definitions */
 const platformName = "manifold";
-const endpoint = "https://manifold.markets/api/v0/markets";
+const ENDPOINT = "https://manifold.markets/api/v0/markets";
 // See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5
 
 /* Support functions */
 
-async function fetchData() {
+async function fetchPage(endpoint: string) {
   let response = await axios({
     url: endpoint,
     method: "GET",
@@ -23,6 +23,31 @@ async function fetchData() {
   return response;
 }
 
+async function fetchAllData(){
+  let endpoint = ENDPOINT
+  let end = false
+  let allData = []
+  let counter = 1
+  while(!end){
+    console.log(`Query #${counter}: ${endpoint}`)
+    let newData = await fetchPage(endpoint)
+    if(Array.isArray(newData)){
+      allData.push(...newData)
+      let hasReachedEnd = (newData.length == 0) || (newData[newData.length -1] == undefined) || (newData[newData.length -1].id == undefined)
+      if(!hasReachedEnd){
+        let lastId = newData[newData.length -1].id
+        endpoint = `${ENDPOINT}?before=${lastId}`
+      }else{
+        end = true
+      }
+    }else{
+      end = true
+    }
+    counter = counter +1
+  }
+  return allData
+}
+
 function showStatistics(results: FetchedQuestion[]) {
   console.log(`Num unresolved markets: ${results.length}`);
   let sum = (arr: number[]) => arr.reduce((tally, a) => tally + a, 0);
@@ -63,12 +88,12 @@ function processPredictions(predictions: any[]): FetchedQuestion[] {
       id: id,
       title: prediction.question,
       url: prediction.url,
-      description: prediction.description,
+      description: prediction.description || "",
       options,
       qualityindicators: {
         createdTime: prediction.createdTime,
-        volume7Days: prediction.volume7Days,
-        volume24Hours: prediction.volume24Hours,
+        // volume7Days: prediction.volume7Days, // deprecated.
+        volume24Hours: prediction.volume24Hours, 
         pool: prediction.pool, // normally liquidity, but I don't actually want to show it.
       },
       extra: {
@@ -90,16 +115,16 @@ export const manifold: Platform = {
   color: "#793466",
   version: "v1",
   async fetcher() {
-    let data = await fetchData();
+    let data = await fetchAllData();
     let results = processPredictions(data); // somehow needed
     showStatistics(results);
     return results;
   },
   calculateStars(data) {
     let nuno = () =>
-      (data.qualityindicators.volume7Days || 0) > 250 ||
+      (data.qualityindicators.volume24Hours || 0) > 100 ||
       ((data.qualityindicators.pool || 0) > 500 &&
-        (data.qualityindicators.volume7Days || 0) > 100)
+        (data.qualityindicators.volume24Hours || 0) > 50)
         ? 2
         : 1;
     let eli = () => null;

src/backend/platforms/metaculus/api.ts

@@ -64,6 +64,7 @@ const predictableProps = {
         additionalProperties: true,
       },
     },
+    nullable: true,
     additionalProperties: true,
   },
 } as const;
@@ -211,15 +212,17 @@ const fetchAndValidate = async <T = unknown>(
   url: string,
   validator: ValidateFunction<T>
 ): Promise<T> => {
-  console.log(url);
+  // console.log(url);
   const data = await fetchWithRetries<object>(url);
   if (validator(data)) {
     return data;
+  }else{
+    console.log(data)
+    throw new Error(
+      `Response validation for url ${url} failed: ` +
+        JSON.stringify(validator.errors, null, 4)
+    );
   }
-  throw new Error(
-    `Response validation for url ${url} failed: ` +
-      JSON.stringify(validator.errors)
-  );
 };
 
 export async function fetchApiQuestions(

src/backend/platforms/metaculus/index.ts

@@ -1,28 +1,27 @@
-import { FetchedQuestion, Platform } from "..";
-import { average } from "../../../utils";
-import { sleep } from "../../utils/sleep";
+import Error from "next/error";
+import {FetchedQuestion, Platform} from "..";
+import {average} from "../../../utils";
+import {sleep} from "../../utils/sleep";
 import {
   ApiCommon,
   ApiMultipleQuestions,
   ApiPredictable,
   ApiQuestion,
   fetchApiQuestions,
-  fetchSingleApiQuestion,
+  fetchSingleApiQuestion
 } from "./api";
 
 const platformName = "metaculus";
 const now = new Date().toISOString();
 const SLEEP_TIME = 1000;
 
-async function apiQuestionToFetchedQuestions(
-  apiQuestion: ApiQuestion
-): Promise<FetchedQuestion[]> {
+async function apiQuestionToFetchedQuestions(apiQuestion: ApiQuestion): Promise<FetchedQuestion[]> {
   // one item can expand:
   // - to 0 questions if we don't want it;
   // - to 1 question if it's a simple forecast
   // - to multiple questions if it's a group (see https://github.com/quantified-uncertainty/metaforecast/pull/84 for details)
 
-  const skip = (q: ApiPredictable): boolean => {
+  const skip = (q : ApiPredictable) : boolean => {
     if (q.publish_time > now || now > q.resolve_time) {
       return true;
     }
@@ -32,89 +31,119 @@ async function apiQuestionToFetchedQuestions(
     return false;
   };
 
-  const buildFetchedQuestion = (
-    q: ApiPredictable & ApiCommon
-  ): Omit<FetchedQuestion, "url" | "description" | "title"> => {
-    const isBinary = q.possibilities.type === "binary";
-    let options: FetchedQuestion["options"] = [];
-    if (isBinary) {
-      const probability = q.community_prediction.full.q2;
-      if (probability !== undefined) {
-        options = [
-          {
-            name: "Yes",
-            probability: probability,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - probability,
-            type: "PROBABILITY",
-          },
-        ];
+  const buildFetchedQuestion = (q : ApiPredictable & ApiCommon) : Omit < FetchedQuestion,
+    "url" | "description" | "title" > => {
+      const isBinary = q.possibilities.type === "binary";
+      let options: FetchedQuestion["options"] = [];
+      if (isBinary) {
+        const probability = q.community_prediction?.full.q2;
+        if (probability !== undefined) {
+          options = [
+            {
+              name: "Yes",
+              probability: probability,
+              type: "PROBABILITY"
+            }, {
+              name: "No",
+              probability: 1 - probability,
+              type: "PROBABILITY"
+            },
+          ];
+        }
       }
-    }
-    return {
-      id: `${platformName}-${q.id}`,
-      options,
-      qualityindicators: {
-        numforecasts: q.number_of_predictions,
-      },
-      extra: {
-        resolution_data: {
-          publish_time: apiQuestion.publish_time,
-          resolution: apiQuestion.resolution,
-          close_time: apiQuestion.close_time,
-          resolve_time: apiQuestion.resolve_time,
+      return {
+          id: `${platformName}-${
+          q.id
+        }`,
+        options,
+        qualityindicators: {
+          numforecasts: q.number_of_predictions
         },
-      },
+        extra: {
+          resolution_data: {
+            publish_time: apiQuestion.publish_time,
+            resolution: apiQuestion.resolution,
+            close_time: apiQuestion.close_time,
+            resolve_time: apiQuestion.resolve_time
+          }
+        }
+      };
     };
-  };
 
   if (apiQuestion.type === "group") {
     await sleep(SLEEP_TIME);
-    const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
-    if (apiQuestionDetails.type !== "group") {
-      throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript
+    let apiQuestionDetailsTemp
+    try{
+      apiQuestionDetailsTemp = await fetchSingleApiQuestion(apiQuestion.id);
+    }catch(error){
+      console.log(error)
+      return []
     }
-    return (apiQuestionDetails.sub_questions || [])
-      .filter((q) => !skip(q))
-      .map((sq) => {
-        const tmp = buildFetchedQuestion(sq);
-        return {
-          ...tmp,
-          title: `${apiQuestion.title} (${sq.title})`,
-          description: apiQuestionDetails.description || "",
-          url: `https://www.metaculus.com${apiQuestion.page_url}?sub-question=${sq.id}`,
-        };
-      });
+    const apiQuestionDetails = apiQuestionDetailsTemp
+    if (apiQuestionDetails.type !== "group") {
+      console.log("Error: expected `group` type")
+      return [] //throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript
+    }else{
+      try{
+        let result = (apiQuestionDetails.sub_questions || []).filter((q) => ! skip(q)).map((sq) => {
+          const tmp = buildFetchedQuestion(sq);
+          return {
+            ... tmp,
+            title: `${
+              apiQuestion.title
+            } (${
+              sq.title
+            })`,
+            description: apiQuestionDetails.description || "",
+            url: `https://www.metaculus.com${
+              apiQuestion.page_url
+            }?sub-question=${
+              sq.id
+            }`
+          };
+        });
+        return result
+      }catch(error){
+        console.log(error)
+        return []
+      }
+    }
+      
   } else if (apiQuestion.type === "forecast") {
     if (apiQuestion.group) {
       return []; // sub-question, should be handled on the group level
     }
     if (skip(apiQuestion)) {
+      console.log(`- [Skipping]: ${
+        apiQuestion.title
+      }`)
+      /*console.log(`Close time: ${
+        apiQuestion.close_time
+      }, resolve time: ${
+        apiQuestion.resolve_time
+      }`)*/
       return [];
     }
 
     await sleep(SLEEP_TIME);
-    const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
-    const tmp = buildFetchedQuestion(apiQuestion);
-    return [
-      {
-        ...tmp,
-        title: apiQuestion.title,
-        description: apiQuestionDetails.description || "",
-        url: "https://www.metaculus.com" + apiQuestion.page_url,
-      },
-    ];
+    try{
+      const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
+      const tmp = buildFetchedQuestion(apiQuestion);
+      return [{
+          ... tmp,
+          title: apiQuestion.title,
+          description: apiQuestionDetails.description || "",
+          url: "https://www.metaculus.com" + apiQuestion.page_url
+        },];
+    }catch(error){
+      console.log(error)
+      return []
+    }
   } else {
-    if (apiQuestion.type !== "claim") {
-      // should never happen, since `discriminator` in JTD schema causes a strict runtime check
-      console.log(
-        `Unknown metaculus question type: ${
-          (apiQuestion as any).type
-        }, skipping`
-      );
+    if (apiQuestion.type !== "claim") { // should never happen, since `discriminator` in JTD schema causes a strict runtime check
+      console.log(`Unknown metaculus question type: ${
+        (apiQuestion as any).type
+      }, skipping`);
     }
     return [];
   }
@@ -125,19 +154,25 @@ export const metaculus: Platform<"id" | "debug"> = {
   label: "Metaculus",
   color: "#006669",
   version: "v2",
-  fetcherArgs: ["id", "debug"],
+  fetcherArgs: [
+    "id", "debug"
+  ],
   async fetcher(opts) {
     let allQuestions: FetchedQuestion[] = [];
 
-    if (opts.args?.id) {
-      const id = Number(opts.args.id);
-      const apiQuestion = await fetchSingleApiQuestion(id);
-      const questions = await apiQuestionToFetchedQuestions(apiQuestion);
-      console.log(questions);
-      return {
-        questions,
-        partial: true,
-      };
+    if (opts.args ?. id) {
+      try{
+        console.log("Using optional id arg.")
+        const id = Number(opts.args.id);
+        const apiQuestion = await fetchSingleApiQuestion(id);
+        const questions = await apiQuestionToFetchedQuestions(apiQuestion);
+        console.log(questions);
+        return {questions, partial: true};
+  
+      }catch(error){
+        console.log(error)
+        return {questions: [], partial: true};
+      }
     }
 
     let next: string | null = "https://www.metaculus.com/api2/questions/";
@@ -148,14 +183,17 @@ export const metaculus: Platform<"id" | "debug"> = {
       await sleep(SLEEP_TIME);
       const apiQuestions: ApiMultipleQuestions = await fetchApiQuestions(next);
       const results = apiQuestions.results;
-
+      // console.log(results)
       let j = false;
 
       for (const result of results) {
         const questions = await apiQuestionToFetchedQuestions(result);
+        // console.log(questions)
         for (const question of questions) {
-          console.log(`- ${question.title}`);
-          if ((!j && i % 20 === 0) || opts.args?.debug) {
+          console.log(`- ${
+            question.title
+          }`);
+          if ((! j && i % 20 === 0) || opts.args ?. debug) {
             console.log(question);
             j = true;
           }
@@ -167,20 +205,16 @@ export const metaculus: Platform<"id" | "debug"> = {
       i += 1;
     }
 
-    return {
-      questions: allQuestions,
-      partial: false,
-    };
+    return {questions: allQuestions, partial: false};
   },
 
   calculateStars(data) {
-    const { numforecasts } = data.qualityindicators;
-    const nuno = () =>
-      (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
+    const {numforecasts} = data.qualityindicators;
+    const nuno = () => (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
     const eli = () => 3;
     const misha = () => 3;
     const starsDecimal = average([nuno(), eli(), misha()]);
     const starsInteger = Math.round(starsDecimal);
     return starsInteger;
-  },
+  }
 };

src/pages/_app.tsx

@@ -17,12 +17,14 @@ Router.events.on("routeChangeStart", (as, { shallow }) => {
 Router.events.on("routeChangeComplete", () => NProgress.done());
 Router.events.on("routeChangeError", () => NProgress.done());
 
+
 function MyApp({ Component, pageProps }: AppProps) {
   return (
     <PlausibleProvider domain="metaforecast.org">
       <Component {...pageProps} />
     </PlausibleProvider>
   );
+  // Workaround in package.json for: https://github.com/vercel/next.js/issues/36019#issuecomment-1103266481
 }
 
 export default withUrqlClient((ssr) => getUrqlClientOptions(ssr), {

src/pages/api/graphql.ts

@@ -1,13 +1,24 @@
-import { NextApiRequest, NextApiResponse } from "next";
+import {NextApiRequest, NextApiResponse} from "next";
 
 // apollo-server-micro is problematic since v3, see https://github.com/apollographql/apollo-server/issues/5547, so we use graphql-yoga instead
-import { createServer } from "@graphql-yoga/node";
+import {createYoga} from "graphql-yoga";
+import {useResponseCache} from '@graphql-yoga/plugin-response-cache'
 
-import { schema } from "../../graphql/schema";
+import {schema} from "../../graphql/schema";
 
-const server = createServer<{
+const server = createYoga < {
   req: NextApiRequest;
   res: NextApiResponse;
-}>({ schema });
+} > ({
+  schema,
+  graphqlEndpoint: '/api/graphql',
+  plugins: [useResponseCache(
+      { // global cache
+        session: () => null,
+        ttl: 2 * 60 * 60 * 1000,
+        // ^ 2h * 60 mins per hour, 60 seconds per min 1000 miliseconds per second
+      }
+    )]
+});
 
 export default server;

src/web/common/BoxedLink.tsx

@@ -11,13 +11,13 @@ export const BoxedLink: React.FC<Props> = ({
   children,
 }) => (
   <a
-    className={`px-2 py-1 border-2 border-gray-400 rounded-lg text-black no-underline text-normal hover:bg-gray-100 inline-flex flex-nowrap space-x-1 items-center ${
+    className={`px-2 py-1 border-2 border-gray-400 rounded-lg text-black no-underline  hover:bg-gray-100 inline-flex flex-nowrap space-x-1 items-center text-xs md:text-lg  ${
       size === "small" ? "text-sm" : ""
     }`}
     href={url}
     target="_blank"
   >
     <span>{children}</span>
-    <FaExternalLinkAlt className="text-gray-400 inline" />
+    <FaExternalLinkAlt className="text-gray-400 inline " />
   </a>
 );

src/web/common/Spinner.tsx

@@ -11,7 +11,7 @@ export const Spinner: React.FC = () => (
       cy="12"
       r="10"
       stroke="currentColor"
-      stroke-width="4"
+      strokeWidth="4"
     ></circle>
     <path
       className="opacity-75"

src/web/questions/components/HistoryChart/InnerChart.tsx

@@ -13,9 +13,11 @@ import {
   VictoryVoronoiContainer,
 } from "victory";
 
-import { chartColors, ChartData, ChartSeries, height, width } from "./utils";
+import { chartColors, ChartData, ChartSeries, goldenRatio } from "./utils";
 
-let dateFormat = "MMM do y"; // "yyyy-MM-dd"
+const height = 200
+const width = 200 * goldenRatio
+let dateFormat = "dd/MM/yy"; // "yyyy-MM-dd" // "MMM do yy"
 
 // can't be replaced with React component, VictoryChart requires VictoryGroup elements to be immediate children
 const getVictoryGroup = ({
@@ -37,7 +39,7 @@ const getVictoryGroup = ({
           data: {
             // strokeOpacity: highlight ?  1 : 0.5,
             strokeOpacity: highlight && !isBinary ? 0.8 : 0.6,
-            strokeWidth: highlight && !isBinary ? 4 : 3,
+            strokeWidth: highlight && !isBinary ? 2.5 : 1.5,
           },
         }}
       />
@@ -71,9 +73,9 @@ export const InnerChart: React.FC<Props> = ({
   const domainMax =
     maxProbability < 0.5 ? Math.round(10 * (maxProbability + 0.05)) / 10 : 1;
   const padding = {
-    top: 20,
-    bottom: 75,
-    left: 70,
+    top: 12,
+    bottom: 33,
+    left: 30,
     right: 17,
   };
 
@@ -99,12 +101,12 @@ export const InnerChart: React.FC<Props> = ({
                 <VictoryLabel
                   style={[
                     {
-                      fontSize: 16,
+                      fontSize: 10,
                       fill: "black",
                       strokeWidth: 0.05,
                     },
                     {
-                      fontSize: 16,
+                      fontSize: 10,
                       fill: "#777",
                       strokeWidth: 0.05,
                     },
@@ -118,7 +120,7 @@ export const InnerChart: React.FC<Props> = ({
                 )}`
               }
               style={{
-                fontSize: 17, // needs to be set here and not just in labelComponent for text size calculations
+                fontSize: 10, // needs to be set here and not just in labelComponent for text size calculations
                 fontFamily:
                   '"Gill Sans", "Gill Sans MT", "Ser­avek", "Trebuchet MS", sans-serif',
                 // default font family from Victory, need to be specified explicitly for some reason, otherwise text size gets miscalculated
@@ -128,10 +130,10 @@ export const InnerChart: React.FC<Props> = ({
                 fill: "white",
               }}
               cornerRadius={4}
-              flyoutPadding={{ top: 4, bottom: 4, left: 16, right: 16 }}
+              flyoutPadding={{ top: 4, bottom: 4, left: 10, right: 10 }}
             />
           }
-          radius={50}
+          radius={20}
           voronoiBlacklist={
             [...Array(seriesList.length).keys()].map((i) => `line-${i}`)
             // see: https://github.com/FormidableLabs/victory/issues/545
@@ -159,10 +161,10 @@ export const InnerChart: React.FC<Props> = ({
         }}
         tickLabelComponent={
           <VictoryLabel
-            dx={-40}
+            dx={-10}
             dy={0}
             angle={-30}
-            style={{ fontSize: 15, fill: "#777" }}
+            style={{ fontSize: 9, fill: "#777" }}
           />
         }
         scale={{ x: "time" }}
@@ -174,7 +176,7 @@ export const InnerChart: React.FC<Props> = ({
           grid: { stroke: "#D3D3D3", strokeWidth: 0.5 },
         }}
         tickLabelComponent={
-          <VictoryLabel dy={0} style={{ fontSize: 18, fill: "#777" }} />
+          <VictoryLabel dy={0} dx={5} style={{ fontSize: 9, fill: "#777" }} />
         }
         // tickFormat specifies how ticks should be displayed
         tickFormat={(x) => `${x * 100}%`}
@@ -205,6 +207,7 @@ export const InnerChart: React.FC<Props> = ({
             })
             */
       }
+      
     </VictoryChart>
   );
 };

src/web/questions/components/HistoryChart/index.tsx

@@ -23,7 +23,7 @@ export const HistoryChart: React.FC<Props> = ({ question }) => {
   const data = useMemo(() => buildChartData(question), [question]);
 
   return (
-    <div className="flex items-center flex-col space-y-4 sm:flex-row sm:space-y-0">
+    <div className="flex items-center space-y-4 sm:flex-row sm:space-y-0 ">
       <InnerChart data={data} highlight={highlight} />
       <Legend
         items={data.seriesNames.map((name, i) => ({

src/web/questions/components/HistoryChart/utils.ts

@@ -18,7 +18,7 @@ export const chartColors = [
   "#F59E0B", // amber-500
 ];
 
-const goldenRatio = (1 + Math.sqrt(5)) / 2;
+export const goldenRatio = (1 + Math.sqrt(5)) / 2;
 // used both for chart and for ssr placeholder
 export const width = 750;
 export const height = width / goldenRatio;

src/web/questions/components/QuestionCard/QuestionFooter.tsx

@@ -172,6 +172,7 @@ export const QuestionFooter: React.FC<Props> = ({
       >
         {question.platform.label
           .replace("Good Judgment Open", "GJOpen")
+          .replace("Insight Prediction", "Insight")
           .replace(/ /g, "\u00a0")}
       </div>
       <div

src/web/questions/components/QuestionCard/index.tsx

@@ -17,8 +17,8 @@ const truncateText = (length: number, text: string): string => {
     return text;
   }
   const breakpoints = " .!?";
-  let lastLetter: string | undefined = undefined;
-  let lastIndex: number | undefined = undefined;
+  let lastLetter
+  let lastIndex
   for (let index = length; index > 0; index--) {
     const letter = text[index];
     if (breakpoints.includes(letter)) {

src/web/questions/components/QuestionOptions.tsx

@@ -101,7 +101,7 @@ const OptionRow: React.FC<OptionProps> = ({ option, mode, textMode }) => {
       <div
         className={`flex-none rounded-md text-center ${
           mode === "primary"
-            ? "text-normal text-white px-2 py-0.5 font-bold"
+            ? "text-sm md:text-lg text-normal text-white px-2 py-0.5 font-bold"
             : "text-sm w-14 py-0.5"
         } ${
           mode === "primary"
@@ -113,7 +113,7 @@ const OptionRow: React.FC<OptionProps> = ({ option, mode, textMode }) => {
       </div>
       <div
         className={`leading-snug ${
-          mode === "primary" ? "text-normal" : "text-sm"
+          mode === "primary" ? "text-sm md:text-lg text-normal" : "text-sm"
         } ${
           mode === "primary" ? textColor(option.probability) : "text-gray-700"
         }`}

src/web/questions/components/QuestionTitle.tsx

@@ -10,7 +10,7 @@ export const QuestionTitle: React.FC<Props> = ({
   question,
   linkToMetaforecast,
 }) => (
-  <h1 className="sm:text-3xl text-xl">
+  <h1 className="sm:text-3xl text-lg">
     <a
       className="text-black no-underline hover:text-gray-700"
       href={

src/web/questions/components/Stars.tsx

@@ -54,5 +54,5 @@ function getStarsColor(numstars: number) {
 }
 
 export const Stars: React.FC<{ num: number }> = ({ num }) => {
-  return <div className={getStarsColor(num)}>{getstars(num)}</div>;
+  return <div className={getStarsColor(num) + " text-xs md:text-lg"}>{getstars(num)}</div>;
 };

src/web/questions/pages/EmbedQuestionPage.tsx

@@ -30,24 +30,25 @@ export const getServerSideProps: GetServerSideProps<Props> = async (
     props: {
       urqlState: ssrCache.extractData(),
       id,
+      question
     },
   };
 };
 
 const EmbedQuestionPage: NextPage<Props> = ({ id }) => {
   return (
-    <div className="bg-white min-h-screen">
+    <div className="block bg-white min-h-screen">
       <Query document={QuestionPageDocument} variables={{ id }}>
         {({ data: { result: question } }) =>
           question ? (
-            <div className="p-4">
-              <QuestionTitle question={question} linkToMetaforecast={true} />
+            <div className="flex flex-col p-2 w-full h-12/12">
+              {/*<QuestionTitle question={question} linkToMetaforecast={true} /> */}
 
-              <div className="mb-5 mt-5">
+              <div className="mb-1 mt-1">
                 <QuestionInfoRow question={question} />
               </div>
 
-              <div className="mb-10">
+              <div className="mb-0">
                 <QuestionChartOrVisualization question={question} />
               </div>
             </div>

src/web/questions/pages/QuestionPage.tsx

@@ -72,7 +72,7 @@ const Section: React.FC<{ title: string; id?: string }> = ({
 const EmbedSection: React.FC<{ question: QuestionWithHistoryFragment }> = ({
   question,
 }) => {
-  const url = getBasePath() + `/questions/embed/${question.id}`;
+  const url = `https://${getBasePath()}/questions/embed/${question.id}`;
   return (
     <Section title="Embed" id="embed">
       <CopyParagraph

src/web/utils.ts

@@ -2,7 +2,7 @@ import { QuestionFragment } from "./fragments.generated";
 
 export const getBasePath = () => {
   if (process.env.NEXT_PUBLIC_VERCEL_URL) {
-    return `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`;
+    return `https://metaforecast.org`;//`https://${process.env.NEXT_PUBLIC_VERCEL_URL}`;
   }
 
   // can be used for local development if you prefer non-default port

src/web/worker/searchWithAlgolia.ts

@@ -97,7 +97,7 @@ export default async function searchWithAlgolia({
         url: "https://metaforecast.org",
         platform: "metaforecast",
         platformLabel: "metaforecast",
-        description: "Maybe try a broader query?",
+        description: "Maybe try a broader query, e.g., reduce the number of 'stars' by clicking in 'Advanced options'?",
         options: [
           {
             name: "Yes",
@@ -166,7 +166,7 @@ export default async function searchWithAlgolia({
       url: "https://metaforecast.org",
       platform: "metaforecast",
       platformLabel: "metaforecast",
-      description: "Maybe try a broader query? That said, we could be wrong.",
+      description: "Maybe try a broader query? Maybe try a broader query, e.g., reduce the number of 'stars' by clicking in 'Advanced options'? That said, we could be wrong.",
       options: [
         {
           name: "Yes",

src/web/worker/uploadToImgur.ts

@@ -5,7 +5,7 @@ export async function uploadToImgur(dataURL: string): Promise<string> {
     method: "post",
     url: "https://api.imgur.com/3/image",
     headers: {
-      Authorization: "Bearer 8e9666fb889318515a62208560d4e8393dac26d8",
+      Authorization: `Bearer ${process.env.IMGUR_BEARER}`,
     },
     data: {
       type: "base64",