tweak: Insight name display

So far restricted to:
- Binary markets
- Denominated in USD

.gitignore

@@ -31,7 +31,7 @@ yarn-error.log*
 
 # yarn vs npm conflict
 package-lock.json ## use yarn.lock instead
-yarn.lock
+# yarn.lock
 
 # Local Netlify folder
 .netlify

README.md

@@ -91,3 +91,16 @@ Overall, the services which we use are:
 
 - Commits follow [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/#summary)
 - For elicit and metaculus, this library currently filters out questions with <10 predictions.
+- The database is updated once a day, at 3:00 AM UTC, with the command `ts-node -T src/backend/flow/doEverythingForScheduler.ts`. The frontpage is updated after that, at 6:00 AM UTC with the command `ts-node -T src/backend/index.ts frontpage`. It's possible that either of these two operations makes the webpage briefly go down.
+
+## To do
+
+- [x] Update Metaculus and Manifold Markets fetchers
+- [ ] Upgrade to [React 18](https://reactjs.org/blog/2022/03/08/react-18-upgrade-guide.html). This will require dealing with the workaround we used for [this issue](https://github.com/vercel/next.js/issues/36019#issuecomment-1103266481)
+- [ ] Add markets from [Insight Prediction](https://insightprediction.com/).
+- [ ] Add database of resolutions
+- [ ] Allow users to embed predictions in the EA Forum/LessWrong (in progress)
+- [ ] Find a long-term mantainer for this project
+- [ ] Allow users to record their own predictions
+- [ ] Release snapshots (I think @niplav is working on this)
+- [ ] ...

package.json

@@ -7,8 +7,7 @@
     "url": "git+https://github.com/QURIresearch/metaforecast.git"
   },
   "keywords": [
-    "forecasts",
-    "predictions"
+    "forecasts", "predictions"
   ],
   "author": "Nuño Sempere",
   "license": "MIT",
@@ -27,92 +26,96 @@
     "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES"
   },
   "dependencies": {
-    "@floating-ui/react-dom": "^0.7.0",
-    "@graphql-yoga/node": "^2.1.0",
-    "@pothos/core": "^3.5.1",
-    "@pothos/plugin-prisma": "^3.4.0",
-    "@pothos/plugin-relay": "^3.10.0",
-    "@prisma/client": "^3.11.1",
-    "@quri/squiggle-lang": "^0.2.11",
-    "@tailwindcss/forms": "^0.4.0",
-    "@tailwindcss/typography": "^0.5.1",
-    "@types/chroma-js": "^2.1.3",
+    "@floating-ui/react-dom": "^0.7.2",
+    "@graphql-yoga/node": "^2.13.13",
+    "@pothos/core": "^3.22.8",
+    "@pothos/plugin-prisma": "^3.35.6",
+    "@pothos/plugin-relay": "^3.28.6",
+    "@prisma/client": "^3.15.2",
+    "@quri/squiggle-lang": "^0.2.12",
+    "@tailwindcss/forms": "^0.4.1",
+    "@tailwindcss/typography": "^0.5.7",
+    "@types/chroma-js": "^2.1.4",
     "@types/dom-to-image": "^2.6.4",
-    "@types/google-spreadsheet": "^3.2.1",
-    "@types/jsdom": "^16.2.14",
+    "@types/google-spreadsheet": "^3.3.0",
+    "@types/jsdom": "^16.2.15",
     "@types/nprogress": "^0.2.0",
-    "@types/react": "^17.0.39",
-    "@types/react-copy-to-clipboard": "^5.0.2",
+    "@types/react": "^17.0.50",
+    "@types/react-copy-to-clipboard": "^5.0.4",
     "@types/textversionjs": "^1.1.1",
     "@types/tunnel": "^0.0.3",
-    "airtable": "^0.11.1",
+    "airtable": "^0.11.5",
     "ajv": "^8.11.0",
-    "algoliasearch": "^4.10.3",
-    "autoprefixer": "^10.1.0",
+    "algoliasearch": "^4.14.2",
+    "autoprefixer": "10.4.5",
     "axios": "^0.25.0",
     "chroma-js": "^2.4.2",
     "critters": "^0.0.16",
-    "date-fns": "^2.28.0",
+    "date-fns": "^2.29.3",
     "dom-to-image": "^2.6.0",
-    "dotenv": "^16.0.0",
+    "dotenv": "^16.0.3",
     "fetch": "^1.1.0",
     "fs": "^0.0.1-security",
-    "fuse.js": "^6.4.6",
-    "google-spreadsheet": "^3.1.15",
-    "graphql": "^16.3.0",
-    "graphql-request": "^4.0.0",
-    "html-to-image": "^1.7.0",
+    "fuse.js": "^6.6.2",
+    "google-spreadsheet": "^3.3.0",
+    "graphql": "^16.6.0",
+    "graphql-request": "^4.3.0",
+    "html-to-image": "^1.10.8",
     "https": "^1.0.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^19.0.0",
-    "json2csv": "^5.0.5",
-    "multiselect-react-dropdown": "^2.0.17",
-    "next": "12",
-    "next-plausible": "^3.1.6",
-    "next-urql": "^3.3.2",
+    "json2csv": "^5.0.7",
+    "multiselect-react-dropdown": "^2.0.25",
+    "next": "^12.3.1",
+    "next-plausible": "^3.6.3",
+    "next-urql": "^3.3.3",
     "nprogress": "^0.2.0",
-    "open": "^7.3.1",
-    "papaparse": "^5.3.0",
-    "pg": "^8.7.3",
-    "postcss": "^8.2.1",
+    "open": "^7.4.2",
+    "papaparse": "^5.3.2",
+    "pg": "^8.8.0",
+    "postcss": "^8.4.18",
     "postcss-flexbugs-fixes": "^5.0.2",
-    "postcss-preset-env": "^7.3.2",
-    "prisma": "^3.11.1",
+    "postcss-preset-env": "^7.8.2",
+    "prisma": "^3.15.2",
     "query-string": "^7.1.1",
     "re-resizable": "^6.9.9",
     "react": "^17.0.2",
     "react-component-export-image": "^1.0.6",
-    "react-compound-slider": "^3.3.1",
-    "react-copy-to-clipboard": "^5.0.3",
+    "react-compound-slider": "^3.4.0",
+    "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^17.0.2",
-    "react-dropdown": "^1.9.2",
-    "react-hook-form": "^7.27.0",
-    "react-icons": "^4.2.0",
-    "react-is": "^18.0.0",
-    "react-markdown": "^8.0.0",
+    "react-dropdown": "^1.11.0",
+    "react-hook-form": "^7.38.0",
+    "react-icons": "^4.6.0",
+    "react-is": "^18.2.0",
+    "react-markdown": "^8.0.3",
     "react-safe": "^1.3.0",
-    "react-select": "^5.2.2",
+    "react-select": "^5.5.4",
     "remark-gfm": "^3.0.1",
-    "tabletojson": "^2.0.4",
-    "tailwindcss": "^3.0.22",
+    "tabletojson": "^2.0.7",
+    "tailwindcss": "^3.2.0",
     "textversionjs": "^1.1.3",
-    "ts-node": "^10.7.0",
+    "ts-node": "^10.9.1",
     "tunnel": "^0.0.6",
-    "urql": "^2.2.0",
-    "urql-custom-scalars-exchange": "^0.1.5",
-    "victory": "^36.3.2"
+    "urql": "^2.2.3",
+    "urql-custom-scalars-exchange": "^0.1.6",
+    "victory": "^36.6.8"
+  },
+  "resolutions": {
+    "@types/react": "<18.0.0"
   },
   "devDependencies": {
-    "@graphql-codegen/cli": "^2.6.2",
-    "@graphql-codegen/introspection": "^2.1.1",
-    "@graphql-codegen/near-operation-file-preset": "^2.2.9",
-    "@graphql-codegen/schema-ast": "^2.4.1",
-    "@graphql-codegen/typed-document-node": "^2.2.8",
-    "@graphql-codegen/typescript": "^2.4.8",
-    "@graphql-codegen/typescript-operations": "^2.3.5",
-    "@netlify/plugin-nextjs": "^4.2.4",
-    "@svgr/cli": "^6.2.1",
+    "@graphql-codegen/cli": "^2.13.7",
+    "@graphql-codegen/introspection": "^2.2.1",
+    "@graphql-codegen/near-operation-file-preset": "^2.4.3",
+    "@graphql-codegen/schema-ast": "^2.5.1",
+    "@graphql-codegen/typed-document-node": "^2.3.5",
+    "@graphql-codegen/typescript": "^2.7.5",
+    "@graphql-codegen/typescript-operations": "^2.5.5",
+    "@svgr/cli": "^6.5.0",
     "@types/pg": "^8.6.5",
-    "netlify-cli": "^9.13.6"
+    "eslint": "^8.25.0",
+    "eslint-config-next": "^12.3.1",
+    "typescript": "4.8.4"
   }
 }

src/Global.d.ts

@@ -0,0 +1,8 @@
+// Workaround related to: https://github.com/vercel/next.js/issues/29788
+// https://github.com/vercel/next.js/issues/29788#issuecomment-1000595524
+declare type StaticImageData = {
+  src: string;
+  height: number;
+  width: number;
+  placeholder?: string;
+};

src/backend/platforms/insight.ts

@@ -1,15 +1,218 @@
 /* Imports */
+import {or} from "ajv/dist/compile/codegen";
 import axios from "axios";
 
 import {FetchedQuestion, Platform} from ".";
+import {QuestionOption} from "../../common/types";
+import toMarkdown from "../utils/toMarkdown";
+import { average } from "../../utils";
 
 /* Definitions */
 const platformName = "insight";
-const marketsEnpoint = "https://insightprediction.com/api/markets";
+const marketsEnpoint = "https://insightprediction.com/api/markets?orderBy=is_resolved&sortedBy=asc";
 const getMarketEndpoint = (id : number) => `https://insightprediction.com/api/markets/${id}`;
+const SPORTS_CATEGORIES = [
+  'World Cup',
+  'MLB',
+  'Futures',
+  'Sports',
+  'EPL',
+  'Golf',
+  'NHL',
+  'College Football'
+]
 
 /* Support functions */
 
+// Stubs
+const excludeMarketFromTitle = (title : any) => {
+  if (!!title) {
+    return title.includes(" vs ") || title.includes(" Over: ") || title.includes("NFL") || title.includes("Will there be a first time winner") || title.includes("Premier League")
+  } else {
+    return true
+  }
+}
+
+const hasActiveYesNoOrderBook = (orderbook : any) => {
+  if (!!orderbook) {
+    let yes = !!orderbook.yes && !!orderbook.yes.buy && Array.isArray(orderbook.yes.buy) && orderbook.yes.buy.length != 0 && !!orderbook.yes.buy[0].price && !!orderbook.yes.sell && Array.isArray(orderbook.yes.sell) && orderbook.yes.sell.length != 0 && !!orderbook.yes.sell[0].price
+    let no = !!orderbook.no && !!orderbook.no.buy && Array.isArray(orderbook.no.buy) && orderbook.no.buy.length != 0 && !!orderbook.no.buy[0].price && !!orderbook.no.sell && Array.isArray(orderbook.no.sell) && orderbook.no.sell.length != 0 && !!orderbook.no.sell[0].price
+    return yes && no
+  } else {
+    return false
+  }
+}
+
+const isBinaryQuestion = (data : any) => Array.isArray(data) && data.length == 1
+
+const geomMean = (a : number, b : number) => Math.sqrt(a * b)
+
+const processRelativeUrls = (a : string) => a.replaceAll("] (/", "](http://insightprediction.com/").replaceAll("](/", "](http://insightprediction.com/")
+
+const processDescriptionText = (text : any) => {
+  if (typeof text === 'string') {
+    return processRelativeUrls(toMarkdown(text))
+  } else {
+    return ""
+  }
+}
+
+const getOrderbookPrize = (orderbook : any) => {
+  let yes_min_cents = orderbook.yes.buy[0].price
+  let yes_max_cents = orderbook.yes.sell[0].price
+  let yes_min = Number(yes_min_cents.slice(0, -1))
+  let yes_max = Number(yes_max_cents.slice(0, -1))
+  let yes_price_orderbook = geomMean(yes_min, yes_max)
+  return yes_price_orderbook
+}
+
+const getAnswerProbability = (answer : any) => {
+  let orderbook = answer.orderbook
+  let latest_yes_price = answer.latest_yes_price
+  if (!! orderbook && hasActiveYesNoOrderBook(orderbook)) {
+    let yes_price_orderbook = getOrderbookPrize(orderbook)
+    let yes_probability = (latest_yes_price ? geomMean(latest_yes_price, yes_price_orderbook) : yes_price_orderbook) / 100
+    return yes_probability
+  } else if (!! latest_yes_price) {
+    return latest_yes_price / 100
+  } else {
+    return -1
+  }
+}
+
+// Fetching
+async function fetchPage(bearer: string, pageNum: number) {
+  let pageUrl = `${marketsEnpoint}&page=${pageNum}`
+  const response = await axios({
+    url: pageUrl, // &orderBy=is_resolved&sortedBy=desc`,
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Bearer ${bearer}`
+    }
+  }).then((res) => res.data);
+  // console.log(response);
+  return response;
+}
+
+async function fetchMarket(bearer: string, marketId: number) {
+  const response = await axios({
+    url: getMarketEndpoint(marketId),
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Bearer ${bearer}`
+    }
+  }).then((res) => res.data);
+  // console.log(response)
+  return response;
+
+}
+
+const processMarket = (market : any) => {
+  let options: FetchedQuestion["options"] = []
+
+  if (!!market && !!market.answer && !!market.answer.data) {
+    let data = market.answer.data
+    if (isBinaryQuestion(data)) { // Binary questions
+      let answer = data[0]
+      let probability = getAnswerProbability(answer)
+      if (probability != -1) {
+        options = [
+          {
+            name: "Yes",
+            probability: probability,
+            type: "PROBABILITY"
+          }, {
+            name: "No",
+            probability: 1 - probability,
+            type: "PROBABILITY"
+          },
+        ];
+      } 
+    } else { // non binary question
+      for (let answer of data) {
+        let probability = getAnswerProbability(answer)
+        if (probability != -1) {
+          let newOption: QuestionOption = ({
+            name: String(answer.title),
+            probability: probability,
+            type: "PROBABILITY"
+          });
+          options.push(newOption)
+        }
+      }
+    }
+    if (!! options && Array.isArray(options) && options.length > 0) {
+      const id = `${platformName}-${
+        market.id
+      }`
+      const result: FetchedQuestion = {
+        id: id,
+        title: market.title,
+        url: market.url,
+        description: processDescriptionText(market.rules),
+        options,
+        qualityindicators: market.coin_id == "USD" ? (
+          {volume: market.volume}
+        ) : ({})
+      };
+      return result;
+    }
+  } 
+  return null
+}
+
+async function fetchAllMarkets(bearer: string) {
+  let pageNum = 1
+  let markets = []
+  let categories = []
+  let isEnd = false
+  while (! isEnd) {
+    if(pageNum % 20 == 0){
+      console.log(`Fetching page #${pageNum}`) // : ${pageUrl}
+    }
+    let page = await fetchPage(bearer, pageNum)
+    // console.log(JSON.stringify(page, null, 2))
+    let data = page.data
+    if (!! data && Array.isArray(data) && data.length > 0) {
+      let lastMarket = data[data.length - 1]
+      let isLastMarketResolved = lastMarket.is_resolved
+      if (isLastMarketResolved == true) {
+        isEnd = true
+      }
+      let newMarkets = data.filter(market => !market.is_resolved && !market.is_expired && ! excludeMarketFromTitle(market.title))
+      for (let initMarketData of newMarkets) {
+        let fullMarketDataResponse = await fetchMarket(bearer, initMarketData.id)
+        let fullMarketData = fullMarketDataResponse.data
+        let processedMarketData = processMarket(fullMarketData)
+
+        if (processedMarketData != null && ! SPORTS_CATEGORIES.includes(fullMarketData.category)) {
+          console.log(`- Adding: ${
+            fullMarketData.title
+          }`)
+          console.group()
+          console.log(fullMarketData)
+          console.log(JSON.stringify(processedMarketData, null, 2))
+          console.groupEnd()
+
+          markets.push(processedMarketData)
+        }
+
+        let category = fullMarketData.category
+        categories.push(category)
+
+      }
+    } else {
+      isEnd = true
+    } pageNum = pageNum + 1
+  }
+  console.log(markets)
+  console.log(categories)
+  return markets
+}
 /*
 async function fetchQuestionStats(bearer : string, marketId : number) {
   const response = await axios({
@@ -25,19 +228,7 @@ async function fetchQuestionStats(bearer : string, marketId : number) {
   return response;
 }
 
-async function fetchPage(bearer : string, pageNum : number) {
-  const response = await axios({
-    url: `${marketsEnpoint}?page=${pageNum}`, // &orderBy=is_resolved&sortedBy=desc`,
-    method: "GET",
-    headers: {
-      "Content-Type": "application/json",
-      Accept: "application/json",
-      Authorization: `Bearer ${bearer}`
-    }
-  }).then((res) => res.data);
-  // console.log(response);
-  return response;
-}
+
 
 async function fetchData(bearer : string) {
   let pageNum = 1;
@@ -112,7 +303,6 @@ async function processPredictions(predictions : any[]) {
   return results; // resultsProcessed
 }
 */
-
 /* Body */
 export const insight: Platform = {
   name: platformName,
@@ -121,12 +311,29 @@ export const insight: Platform = {
   version: "v1",
   async fetcher() {
     let bearer = process.env.INSIGHT_BEARER;
-    // let data = await fetchData(bearer);
-    // console.log(data);
-    let results: FetchedQuestion[] = []; // await processPredictions(data); // somehow needed
-    return results;
+    if (!! bearer) {
+      let data = await fetchAllMarkets(bearer);
+      return data
+    } else {
+      throw Error("No INSIGHT_BEARER available in environment")
+    }
+    // let results: FetchedQuestion[] = []; // await processPredictions(data); // somehow needed
+    // return results;
   },
   calculateStars(data) {
-    return 2;
+    let nuno = () => {
+      if((data.qualityindicators.volume || 0) > 10000){
+        return 4
+      } else if((data.qualityindicators.volume || 0) > 1000){
+        return 3
+      } else{
+        return 2
+      }
+    }
+    let eli = () => null;
+    let misha = () => null;
+    let starsDecimal = average([nuno()]); //, eli(data), misha(data)])
+    let starsInteger = Math.round(starsDecimal);
+    return starsInteger;
   }
 };

src/backend/platforms/manifold.ts

@@ -6,12 +6,12 @@ import { FetchedQuestion, Platform } from "./";
 
 /* Definitions */
 const platformName = "manifold";
-const endpoint = "https://manifold.markets/api/v0/markets";
+const ENDPOINT = "https://manifold.markets/api/v0/markets";
 // See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5
 
 /* Support functions */
 
-async function fetchData() {
+async function fetchPage(endpoint: string) {
   let response = await axios({
     url: endpoint,
     method: "GET",
@@ -23,6 +23,31 @@ async function fetchData() {
   return response;
 }
 
+async function fetchAllData(){
+  let endpoint = ENDPOINT
+  let end = false
+  let allData = []
+  let counter = 1
+  while(!end){
+    console.log(`Query #${counter}: ${endpoint}`)
+    let newData = await fetchPage(endpoint)
+    if(Array.isArray(newData)){
+      allData.push(...newData)
+      let hasReachedEnd = (newData.length == 0) || (newData[newData.length -1] == undefined) || (newData[newData.length -1].id == undefined)
+      if(!hasReachedEnd){
+        let lastId = newData[newData.length -1].id
+        endpoint = `${ENDPOINT}?before=${lastId}`
+      }else{
+        end = true
+      }
+    }else{
+      end = true
+    }
+    counter = counter +1
+  }
+  return allData
+}
+
 function showStatistics(results: FetchedQuestion[]) {
   console.log(`Num unresolved markets: ${results.length}`);
   let sum = (arr: number[]) => arr.reduce((tally, a) => tally + a, 0);
@@ -63,7 +88,7 @@ function processPredictions(predictions: any[]): FetchedQuestion[] {
       id: id,
       title: prediction.question,
       url: prediction.url,
-      description: prediction.description,
+      description: prediction.description || "",
       options,
       qualityindicators: {
         createdTime: prediction.createdTime,
@@ -90,7 +115,7 @@ export const manifold: Platform = {
   color: "#793466",
   version: "v1",
   async fetcher() {
-    let data = await fetchData();
+    let data = await fetchAllData();
     let results = processPredictions(data); // somehow needed
     showStatistics(results);
     return results;

src/backend/platforms/metaculus/api.ts

@@ -64,6 +64,7 @@ const predictableProps = {
         additionalProperties: true,
       },
     },
+    nullable: true,
     additionalProperties: true,
   },
 } as const;
@@ -211,15 +212,17 @@ const fetchAndValidate = async <T = unknown>(
   url: string,
   validator: ValidateFunction<T>
 ): Promise<T> => {
-  console.log(url);
+  // console.log(url);
   const data = await fetchWithRetries<object>(url);
   if (validator(data)) {
     return data;
+  }else{
+    console.log(data)
+    throw new Error(
+      `Response validation for url ${url} failed: ` +
+        JSON.stringify(validator.errors, null, 4)
+    );
   }
-  throw new Error(
-    `Response validation for url ${url} failed: ` +
-      JSON.stringify(validator.errors)
-  );
 };
 
 export async function fetchApiQuestions(

src/backend/platforms/metaculus/index.ts

@@ -1,28 +1,27 @@
-import { FetchedQuestion, Platform } from "..";
-import { average } from "../../../utils";
-import { sleep } from "../../utils/sleep";
+import Error from "next/error";
+import {FetchedQuestion, Platform} from "..";
+import {average} from "../../../utils";
+import {sleep} from "../../utils/sleep";
 import {
   ApiCommon,
   ApiMultipleQuestions,
   ApiPredictable,
   ApiQuestion,
   fetchApiQuestions,
-  fetchSingleApiQuestion,
+  fetchSingleApiQuestion
 } from "./api";
 
 const platformName = "metaculus";
 const now = new Date().toISOString();
 const SLEEP_TIME = 1000;
 
-async function apiQuestionToFetchedQuestions(
-  apiQuestion: ApiQuestion
-): Promise<FetchedQuestion[]> {
+async function apiQuestionToFetchedQuestions(apiQuestion: ApiQuestion): Promise<FetchedQuestion[]> {
   // one item can expand:
   // - to 0 questions if we don't want it;
   // - to 1 question if it's a simple forecast
   // - to multiple questions if it's a group (see https://github.com/quantified-uncertainty/metaforecast/pull/84 for details)
 
-  const skip = (q: ApiPredictable): boolean => {
+  const skip = (q : ApiPredictable) : boolean => {
     if (q.publish_time > now || now > q.resolve_time) {
       return true;
     }
@@ -32,89 +31,119 @@ async function apiQuestionToFetchedQuestions(
     return false;
   };
 
-  const buildFetchedQuestion = (
-    q: ApiPredictable & ApiCommon
-  ): Omit<FetchedQuestion, "url" | "description" | "title"> => {
-    const isBinary = q.possibilities.type === "binary";
-    let options: FetchedQuestion["options"] = [];
-    if (isBinary) {
-      const probability = q.community_prediction.full.q2;
-      if (probability !== undefined) {
-        options = [
-          {
-            name: "Yes",
-            probability: probability,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - probability,
-            type: "PROBABILITY",
-          },
-        ];
+  const buildFetchedQuestion = (q : ApiPredictable & ApiCommon) : Omit < FetchedQuestion,
+    "url" | "description" | "title" > => {
+      const isBinary = q.possibilities.type === "binary";
+      let options: FetchedQuestion["options"] = [];
+      if (isBinary) {
+        const probability = q.community_prediction?.full.q2;
+        if (probability !== undefined) {
+          options = [
+            {
+              name: "Yes",
+              probability: probability,
+              type: "PROBABILITY"
+            }, {
+              name: "No",
+              probability: 1 - probability,
+              type: "PROBABILITY"
+            },
+          ];
+        }
       }
-    }
-    return {
-      id: `${platformName}-${q.id}`,
-      options,
-      qualityindicators: {
-        numforecasts: q.number_of_predictions,
-      },
-      extra: {
-        resolution_data: {
-          publish_time: apiQuestion.publish_time,
-          resolution: apiQuestion.resolution,
-          close_time: apiQuestion.close_time,
-          resolve_time: apiQuestion.resolve_time,
+      return {
+          id: `${platformName}-${
+          q.id
+        }`,
+        options,
+        qualityindicators: {
+          numforecasts: q.number_of_predictions
         },
-      },
+        extra: {
+          resolution_data: {
+            publish_time: apiQuestion.publish_time,
+            resolution: apiQuestion.resolution,
+            close_time: apiQuestion.close_time,
+            resolve_time: apiQuestion.resolve_time
+          }
+        }
+      };
     };
-  };
 
   if (apiQuestion.type === "group") {
     await sleep(SLEEP_TIME);
-    const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
-    if (apiQuestionDetails.type !== "group") {
-      throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript
+    let apiQuestionDetailsTemp
+    try{
+      apiQuestionDetailsTemp = await fetchSingleApiQuestion(apiQuestion.id);
+    }catch(error){
+      console.log(error)
+      return []
     }
-    return (apiQuestionDetails.sub_questions || [])
-      .filter((q) => !skip(q))
-      .map((sq) => {
-        const tmp = buildFetchedQuestion(sq);
-        return {
-          ...tmp,
-          title: `${apiQuestion.title} (${sq.title})`,
-          description: apiQuestionDetails.description || "",
-          url: `https://www.metaculus.com${apiQuestion.page_url}?sub-question=${sq.id}`,
-        };
-      });
+    const apiQuestionDetails = apiQuestionDetailsTemp
+    if (apiQuestionDetails.type !== "group") {
+      console.log("Error: expected `group` type")
+      return [] //throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript
+    }else{
+      try{
+        let result = (apiQuestionDetails.sub_questions || []).filter((q) => ! skip(q)).map((sq) => {
+          const tmp = buildFetchedQuestion(sq);
+          return {
+            ... tmp,
+            title: `${
+              apiQuestion.title
+            } (${
+              sq.title
+            })`,
+            description: apiQuestionDetails.description || "",
+            url: `https://www.metaculus.com${
+              apiQuestion.page_url
+            }?sub-question=${
+              sq.id
+            }`
+          };
+        });
+        return result
+      }catch(error){
+        console.log(error)
+        return []
+      }
+    }
+      
   } else if (apiQuestion.type === "forecast") {
     if (apiQuestion.group) {
       return []; // sub-question, should be handled on the group level
     }
     if (skip(apiQuestion)) {
+      console.log(`- [Skipping]: ${
+        apiQuestion.title
+      }`)
+      /*console.log(`Close time: ${
+        apiQuestion.close_time
+      }, resolve time: ${
+        apiQuestion.resolve_time
+      }`)*/
       return [];
     }
 
     await sleep(SLEEP_TIME);
-    const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
-    const tmp = buildFetchedQuestion(apiQuestion);
-    return [
-      {
-        ...tmp,
-        title: apiQuestion.title,
-        description: apiQuestionDetails.description || "",
-        url: "https://www.metaculus.com" + apiQuestion.page_url,
-      },
-    ];
+    try{
+      const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
+      const tmp = buildFetchedQuestion(apiQuestion);
+      return [{
+          ... tmp,
+          title: apiQuestion.title,
+          description: apiQuestionDetails.description || "",
+          url: "https://www.metaculus.com" + apiQuestion.page_url
+        },];
+    }catch(error){
+      console.log(error)
+      return []
+    }
   } else {
-    if (apiQuestion.type !== "claim") {
-      // should never happen, since `discriminator` in JTD schema causes a strict runtime check
-      console.log(
-        `Unknown metaculus question type: ${
-          (apiQuestion as any).type
-        }, skipping`
-      );
+    if (apiQuestion.type !== "claim") { // should never happen, since `discriminator` in JTD schema causes a strict runtime check
+      console.log(`Unknown metaculus question type: ${
+        (apiQuestion as any).type
+      }, skipping`);
     }
     return [];
   }
@@ -125,19 +154,25 @@ export const metaculus: Platform<"id" | "debug"> = {
   label: "Metaculus",
   color: "#006669",
   version: "v2",
-  fetcherArgs: ["id", "debug"],
+  fetcherArgs: [
+    "id", "debug"
+  ],
   async fetcher(opts) {
     let allQuestions: FetchedQuestion[] = [];
 
-    if (opts.args?.id) {
-      const id = Number(opts.args.id);
-      const apiQuestion = await fetchSingleApiQuestion(id);
-      const questions = await apiQuestionToFetchedQuestions(apiQuestion);
-      console.log(questions);
-      return {
-        questions,
-        partial: true,
-      };
+    if (opts.args ?. id) {
+      try{
+        console.log("Using optional id arg.")
+        const id = Number(opts.args.id);
+        const apiQuestion = await fetchSingleApiQuestion(id);
+        const questions = await apiQuestionToFetchedQuestions(apiQuestion);
+        console.log(questions);
+        return {questions, partial: true};
+  
+      }catch(error){
+        console.log(error)
+        return {questions: [], partial: true};
+      }
     }
 
     let next: string | null = "https://www.metaculus.com/api2/questions/";
@@ -148,14 +183,17 @@ export const metaculus: Platform<"id" | "debug"> = {
       await sleep(SLEEP_TIME);
       const apiQuestions: ApiMultipleQuestions = await fetchApiQuestions(next);
       const results = apiQuestions.results;
-
+      // console.log(results)
       let j = false;
 
       for (const result of results) {
         const questions = await apiQuestionToFetchedQuestions(result);
+        // console.log(questions)
         for (const question of questions) {
-          console.log(`- ${question.title}`);
-          if ((!j && i % 20 === 0) || opts.args?.debug) {
+          console.log(`- ${
+            question.title
+          }`);
+          if ((! j && i % 20 === 0) || opts.args ?. debug) {
             console.log(question);
             j = true;
           }
@@ -167,20 +205,16 @@ export const metaculus: Platform<"id" | "debug"> = {
       i += 1;
     }
 
-    return {
-      questions: allQuestions,
-      partial: false,
-    };
+    return {questions: allQuestions, partial: false};
   },
 
   calculateStars(data) {
-    const { numforecasts } = data.qualityindicators;
-    const nuno = () =>
-      (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
+    const {numforecasts} = data.qualityindicators;
+    const nuno = () => (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
     const eli = () => 3;
     const misha = () => 3;
     const starsDecimal = average([nuno(), eli(), misha()]);
     const starsInteger = Math.round(starsDecimal);
     return starsInteger;
-  },
+  }
 };

src/pages/_app.tsx

@@ -17,12 +17,14 @@ Router.events.on("routeChangeStart", (as, { shallow }) => {
 Router.events.on("routeChangeComplete", () => NProgress.done());
 Router.events.on("routeChangeError", () => NProgress.done());
 
+
 function MyApp({ Component, pageProps }: AppProps) {
   return (
     <PlausibleProvider domain="metaforecast.org">
       <Component {...pageProps} />
     </PlausibleProvider>
   );
+  // Workaround in package.json for: https://github.com/vercel/next.js/issues/36019#issuecomment-1103266481
 }
 
 export default withUrqlClient((ssr) => getUrqlClientOptions(ssr), {

src/web/common/Spinner.tsx

@@ -11,7 +11,7 @@ export const Spinner: React.FC = () => (
       cy="12"
       r="10"
       stroke="currentColor"
-      stroke-width="4"
+      strokeWidth="4"
     ></circle>
     <path
       className="opacity-75"

src/web/questions/components/QuestionCard/QuestionFooter.tsx

@@ -74,7 +74,7 @@ const getCurrencySymbolIfNeeded = ({
     "openInterest",
     "liquidity",
   ];
-  let dollarPlatforms = ["predictit", "kalshi", "polymarket"];
+  let dollarPlatforms = ["predictit", "kalshi", "polymarket", "insight"];
   if (indicatorsWhichNeedCurrencySymbol.includes(indicator)) {
     if (dollarPlatforms.includes(platform)) {
       return "$";
@@ -172,6 +172,7 @@ export const QuestionFooter: React.FC<Props> = ({
       >
         {question.platform.label
           .replace("Good Judgment Open", "GJOpen")
+          .replace("Insight Prediction", "Insight")
           .replace(/ /g, "\u00a0")}
       </div>
       <div

src/web/worker/searchWithAlgolia.ts

@@ -97,7 +97,7 @@ export default async function searchWithAlgolia({
         url: "https://metaforecast.org",
         platform: "metaforecast",
         platformLabel: "metaforecast",
-        description: "Maybe try a broader query?",
+        description: "Maybe try a broader query, e.g., reduce the number of 'stars' by clicking in 'Advanced options'?",
         options: [
           {
             name: "Yes",
@@ -166,7 +166,7 @@ export default async function searchWithAlgolia({
       url: "https://metaforecast.org",
       platform: "metaforecast",
       platformLabel: "metaforecast",
-      description: "Maybe try a broader query? That said, we could be wrong.",
+      description: "Maybe try a broader query? Maybe try a broader query, e.g., reduce the number of 'stars' by clicking in 'Advanced options'? That said, we could be wrong.",
       options: [
         {
           name: "Yes",