chore: Deprecate outdated or inaccessible platforms, but keep the code

README.md

@@ -21,6 +21,7 @@ The following variables are currently needed to run the `master` branch:
 - `CSETFORETELL_COOKIE`
 - `GOODJUDGMENTOPENCOOKIE`
 - `HYPERMINDCOOKIE`
+- `SECRET_BETFAIR_ENDPOINT`
 - `ALGOLIA_MASTER_API_KEY`, a string of 32 alphanumeric characters, like `6ofolyptm956j9uuev3q4v81vjbqrkp2` (not an actual key)
 
 They can either be stored as process variables (e.g., something that can be accessed as `process.env.<variable name>`), or as text in `src/input/privatekeys.json`, in the same format as `src/input/privatekeys_example.json`.  

specification.json

@@ -0,0 +1,46 @@
+{
+  "title": " String, not optional",
+  "url": " String. A very few number of platforms have a page for more than one prediction.",
+  "platform": " String, not optional",
+  "description": " String, optional",
+  "options": " Optional. Saved as a string. If this is a binary question, it gets converted to having Yes/No options. The probabilities should be probabilities, i.e., normalized to 1 first.",
+  "options": [
+    "description: optional. Some prediction platforms have distributions, rather than probabilities.",
+    {
+      "name": "Option 1",
+      "probability": "Number, 0 to 1",
+      "type": "for now just 'PROBABILITY', but could include others, like 'DISTRIBUTION'",
+    },
+    {
+      "name": "Option 2",
+      "probability": "Number, 0 to 1",
+      "type": "for now just 'PROBABILITY', but could include others, like 'DISTRIBUTION'",
+    }
+  ],
+  "timestamp": " Timestamp at which metaforecast fetches the probability",
+  "qualityindicators": {
+    "stars": "required, 0 to 5",
+    "numforecasts": " optional",
+    "numforecasters": " optional",
+    "liquidity": "optional",
+    "tradevolume": "optional",
+    "volume": "optional",
+    "address": "optional",
+    "other properties": "optional",
+  },
+  "extra": {
+    "field_description": "arbitrary extra information",
+    "country_name": " country.country_name",
+    "regime_type": " country.regime_type",
+    "month": " country.month",
+    "year": " country.year",
+    "leader_name": " country.leader_name",
+    "month_risk": " country.month_risk",
+    "annual_risk": " country.annual_risk",
+    "risk_change_percent": " country.risk_change_percent",
+    "regime_years": " country.regime_years",
+    "leader_years": " country.leader_years",
+    "country_code": " country.country_code",
+    "country_abb": " country.country_abb",
+  "superforecastercommentary": " analysis || "",
+}

src/index.js

@@ -2,28 +2,7 @@
 import fs from 'fs'
 import readline from "readline"
 
-import { astralcodexten } from "./platforms/astralcodexten-fetch.js"
-import { betfair } from "./platforms/betfair-fetch.js"
-import { coupcast } from "./platforms/coupcast-fetch.js"
-import { csetforetell } from "./platforms/csetforetell-fetch.js"
-import { elicit } from "./platforms/elicit-fetch.js"
-import { estimize } from "./platforms/estimize-fetch.js"
-import { fantasyscotus } from "./platforms/fantasyscotus-fetch.js"
-import { foretold } from "./platforms/foretold-fetch.js"
-import { goodjudgment } from "./platforms/goodjudgment-fetch.js"
-import { goodjudgmentopen } from "./platforms/goodjudmentopen-fetch.js"
-import { hypermind } from "./platforms/hypermind-fetch.js"
-import { kalshi } from "./platforms/kalshi-fetch.js"
-import { ladbrokes } from "./platforms/ladbrokes-fetch.js"
-import { manifoldmarkets } from "./platforms/manifoldmarkets-fetch.js"
-import { metaculus } from "./platforms/metaculus-fetch.js"
-import { omen } from "./platforms/omen-fetch.js"
-import { polymarket } from "./platforms/polymarket-fetch.js"
-import { predictit } from "./platforms/predictit-fetch.js"
-import { rootclaim } from "./platforms/rootclaim-fetch.js"
-import { smarkets } from "./platforms/smarkets-fetch.js"
-import { wildeford } from "./platforms/wildeford-fetch.js"
-import { williamhill } from "./platforms/williamhill-fetch.js"
+import { platformFetchers } from "./utils/platforms.js"
 import { mergeEverything } from "./utils/mergeEverything.js"
 import { updateHistory } from "./utils/history/updateHistory.js"
 import { rebuildAlgoliaDatabase } from "./utils/algolia.js"
@@ -31,11 +10,11 @@ import { rebuildNetlifySiteWithNewData } from "./utils/rebuildNetliftySiteWithNe
 import { doEverything, tryCatchTryAgain } from "./utils/doEverything.js"
 
 /* Support functions */
-let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
+let functions = [...platformFetchers, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
 let functionNames = functions.map(fun => fun.name)
 
 let whattodoMessage = functionNames
-	.slice(0, functionNames.length - 5)
+	.slice(0, platformFetchers.length)
 	.map((functionName, i) => `[${i}]: Download predictions from ${functionName}`)
 	.join('\n') +
 	`\n[${functionNames.length - 5}]: Merge jsons them into one big json (and push it to mongodb database)` +

src/platforms/deprecated/coupcast-fetch.js

@@ -9,9 +9,9 @@ import {upsert} from "../utils/mongo-wrapper.js"
 
 /* Definitions */
 let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv"
-var datenow = new Date();
+var datenow = new Date(); 
 var currentmonth = datenow.getMonth() + 1; 
-
+dd
 /* Support functions */
 let unique = arr => [...new Set(arr)]
 let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));

src/platforms/deprecated/omen-fetch.js

@@ -5,7 +5,9 @@ import { calculateStars } from "../utils/stars.js"
 import {upsert} from "../utils/mongo-wrapper.js"
 
 /* Definitions */
-let graphQLendpoint = 'https://api.thegraph.com/subgraphs/name/protofire/omen'
+let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen"
+// "https://gateway.thegraph.com/api/[api-key]/subgraphs/id/0x0503024fcc5e1bd834530e69d592dbb6e8c03968-0" 
+// 'https://api.thegraph.com/subgraphs/name/protofire/omen'
 // https://github.com/protofire/omen-subgraph
 // https://thegraph.com/explorer/subgraph/protofire/omen
 
@@ -40,7 +42,7 @@ async function fetchAllContractData() {
   })
     .then(res => res.data)
     .then(res => res.data.fixedProductMarketMakers)
-  // console.log(response)
+  console.log(response)
   return response
 }
 

src/platforms/infer-fetch.js

@@ -0,0 +1,254 @@
+/* Imports */
+import axios from "axios"
+import { getCookie, applyIfCookieExists } from "../utils/getCookies.js"
+import { Tabletojson } from "tabletojson"
+import toMarkdown from "../utils/toMarkdown.js"
+import { calculateStars } from "../utils/stars.js"
+import { upsert } from "../utils/mongo-wrapper.js"
+
+/* Definitions */
+let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
+String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
+const DEBUG_MODE = "on"// "off"
+const SLEEP_TIME_RANDOM=100//5000 // miliseconds
+const SLEEP_TIME_EXTRA=0//1000
+/* Support functions */
+
+async function fetchPage(page, cookie) {
+  console.log(`Page #${page}`)
+  if (page == 1) {
+    cookie = cookie.split(";")[0] // Interesting that it otherwise doesn't work :(
+  }
+  let urlEndpoint = htmlEndPoint + page
+  console.log(urlEndpoint)
+  let response = await axios({
+    url: urlEndpoint,
+    method: 'GET',
+    headers: ({
+      'Content-Type': 'text/html',
+      'Cookie': cookie
+    }),
+  })
+    .then(res => res.data)
+  // console.log(response)
+  return response
+}
+
+async function fetchStats(questionUrl, cookie) {
+  let response = await axios({
+    url: questionUrl + "/stats",
+    method: 'GET',
+    headers: ({
+      'Content-Type': 'text/html',
+      'Cookie': cookie,
+      'Referer': questionUrl,
+    }),
+  })
+    .then(res => res.data)
+
+  if (response.includes("Sign up or sign in to forecast")) {
+    throw Error("Not logged in")
+  }
+
+  // Is binary?
+  let isbinary = response.includes("binary?":true")
+  // console.log(`is binary? ${isbinary}`)
+  let options = []
+  if (isbinary) {
+    // Crowd percentage
+    let htmlElements = response.split("\n")
+    // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
+    let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
+    // DEBUG_MODE == "on" ? console.log(h5elements) : id()
+    let crowdpercentage = h3Element.split(">")[1].split("<")[0]
+    let probability = Number(crowdpercentage.replace("%", "")) / 100
+    options.push(({
+      name: "Yes",
+      probability: probability,
+      type: "PROBABILITY"
+    }), ({
+      name: "No",
+      probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
+      type: "PROBABILITY"
+    }))
+  } else {
+    try {
+      let optionsBody = response.split("tbody")[1] // Previously [1], then previously [3] but they added a new table. 
+      // console.log(optionsBody)
+      let optionsHtmlElement = "<table" + optionsBody + "table>"
+      let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
+      let firstTable = tablesAsJson[0]
+      options = firstTable.map(element => ({
+        name: element['0'],
+        probability: Number(element['1'].replace("%", "")) / 100,
+        type: "PROBABILITY"
+      }))
+    } catch (error) {
+      let optionsBody = response.split("tbody")[3] // Catch if the error is related to table position
+      let optionsHtmlElement = "<table" + optionsBody + "table>"
+      let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
+      let firstTable = tablesAsJson[0]
+      if (firstTable) {
+        options = firstTable.map(element => ({
+          name: element['0'],
+          probability: Number(element['1'].replace("%", "")) / 100,
+          type: "PROBABILITY"
+        }))
+      } else {
+        // New type of question, tricky to parse the options
+        // Just leave options = [] for now.
+        // https://www.cset-foretell.com/blog/rolling-question-formats
+      }
+    }
+
+  }
+  // Description  
+  let descriptionraw = response.split(`<meta name="description" content="`)[1]
+  let descriptionprocessed1 = descriptionraw.split(`">`)[0]
+  let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
+  let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
+  // console.log(descriptionprocessed3)
+  let descriptionprocessed4 = descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
+  let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n")
+  let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`)
+  let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'")
+  let descriptionprocessed8 = toMarkdown(descriptionprocessed7)
+  let description = descriptionprocessed8
+
+  // Number of forecasts
+  //console.log(response)
+  //console.log(response.split("prediction_sets_count&quot;:")[1])
+  let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0]
+  // console.log(numforecasts)
+
+  // Number of predictors
+  let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0]
+  // console.log(numpredictors)
+
+  let result = {
+    "description": description,
+    "options": options,
+    "timestamp": new Date().toISOString(),
+    "qualityindicators": {
+      "numforecasts": Number(numforecasts),
+      "numforecasters": Number(numforecasters),
+      "stars": calculateStars("CSET-foretell", { numforecasts })
+    }
+  }
+
+  return result
+}
+
+function isSignedIn(html) {
+
+  let isSignedInBool = !(html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up"))
+  if (!isSignedInBool) {
+    console.log("Error: Not signed in.")
+  }
+  console.log(`Signed in? ${isSignedInBool}`)
+  return isSignedInBool
+}
+
+function isEnd(html) {
+  let isEndBool = html.includes("No questions match your filter")
+  if (isEndBool) {
+    //console.log(html)
+  }
+  console.log(`IsEnd? ${isEndBool}`)
+  return isEndBool
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/* Body */
+
+async function csetforetell_inner(cookie) {
+  let i = 1
+  let response = await fetchPage(i, cookie)
+  let results = []
+  let init = Date.now()
+  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
+  while (!isEnd(response) && isSignedIn(response)) {
+
+    let htmlLines = response.split("\n")
+    // let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
+    let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
+    // console.log(questionHrefs)
+    
+
+    if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
+      //console.log(response)
+      console.log("questionHrefs: ")
+      console.log(questionHrefs)
+    }
+
+    //console.log("")
+    //console.log("")
+    //console.log(h4elements)
+
+    for (let questionHref of questionHrefs) {
+      //console.log(h4element)
+
+      let elementSplit = questionHref.split('"><span>')
+      let url = elementSplit[0].split('<a href="')[1]
+      let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "")
+      await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
+
+      try {
+        let moreinfo = await fetchStats(url, cookie)
+        let question = ({
+          "title": title,
+          "url": url,
+          "platform": "CSET-foretell",
+          ...moreinfo
+        })
+        if (i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) {
+          console.log(`Page #${i}` && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on"))
+          console.log(question)
+        }
+        results.push(question)
+        if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
+          console.log(url)
+          console.log(question)
+        }
+
+      } catch (error) {
+        console.log(error)
+        console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
+      }
+    }
+
+    i++
+    //i=Number(i)+1
+
+    console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
+    await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
+
+    try {
+      response = await fetchPage(i, cookie)
+    } catch (error) {
+      console.log(error)
+      console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
+    }
+  }
+  // let string = JSON.stringify(results,null,  2)
+  // fs.writeFileSync('./data/csetforetell-questions.json', string);
+  // console.log(results)
+  if (results.length > 0) {
+    await upsert(results, "csetforetell-questions")
+  } else {
+    console.log("Not updating results, as process was not signed in")
+  }
+
+  let end = Date.now()
+  let difference = end - init
+  console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`)
+}
+
+
+export async function infer() {
+  let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell")
+  await applyIfCookieExists(cookie, csetforetell_inner)
+}

src/platforms/kalshi-fetch.js

@@ -40,7 +40,7 @@ async function processMarkets(markets) {
       }
     ]
     let result = ({
-      "title": market.title,
+      "title": market.title.replaceAll("*", ""),
       "url": `https://kalshi.com/markets/${market.ticker_name}`,
       "platform": "Kalshi",
       "description": `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,

src/utils/doEverything.js

@@ -1,29 +1,9 @@
-import { astralcodexten } from "../platforms/astralcodexten-fetch.js"
-import { betfair } from "../platforms/betfair-fetch.js"
-import { coupcast } from "../platforms/coupcast-fetch.js"
-import { csetforetell } from "../platforms/csetforetell-fetch.js"
-import { elicit } from "../platforms/elicit-fetch.js"
-import { estimize } from "../platforms/estimize-fetch.js"
-import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js"
-import { foretold } from "../platforms/foretold-fetch.js"
-import { goodjudgment } from "../platforms/goodjudgment-fetch.js"
-import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js"
-import { hypermind } from "../platforms/hypermind-fetch.js"
-import { kalshi } from "../platforms/kalshi-fetch.js"
-import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
-import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
-import { metaculus } from "../platforms/metaculus-fetch.js"
-import { omen } from "../platforms/omen-fetch.js"
-import { polymarket } from "../platforms/polymarket-fetch.js"
-import { predictit } from "../platforms/predictit-fetch.js"
-import { rootclaim } from "../platforms/rootclaim-fetch.js"
-import { smarkets } from "../platforms/smarkets-fetch.js"
-import { wildeford } from "../platforms/wildeford-fetch.js"
-import { williamhill } from "../platforms/williamhill-fetch.js"
+
 import { mergeEverything } from "./mergeEverything.js"
 import { updateHistory } from "./history/updateHistory.js"
 import { rebuildAlgoliaDatabase } from "./algolia.js"
 import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData.js"
+import { platformFetchers } from "./platforms.js"
 
 /* Do everything */
 function sleep(ms) {
@@ -47,7 +27,7 @@ export async function tryCatchTryAgain(fun) {
 }
 
 export async function doEverything() {
-    let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
+    let functions = [...platformFetchers, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
     // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
 
     console.log("")

src/utils/mergeEverything.js

@@ -1,5 +1,5 @@
 import { mongoRead, upsert } from "./mongo-wrapper.js";
-
+import { platformNames } from "./platforms.js"
 /* Merge everything */
 // let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
 let sets = [

src/utils/platforms.js

@@ -0,0 +1,47 @@
+import { betfair } from "../platforms/betfair-fetch.js"
+import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js"
+import { foretold } from "../platforms/foretold-fetch.js"
+import { goodjudgment } from "../platforms/goodjudgment-fetch.js"
+import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js"
+import { infer } from "../platforms/infer-fetch.js"
+import { kalshi } from "../platforms/kalshi-fetch.js"
+import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
+import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
+import { metaculus } from "../platforms/metaculus-fetch.js"
+import { polymarket } from "../platforms/polymarket-fetch.js"
+import { predictit } from "../platforms/predictit-fetch.js"
+import { rootclaim } from "../platforms/rootclaim-fetch.js"
+import { smarkets } from "../platforms/smarkets-fetch.js"
+import { wildeford } from "../platforms/wildeford-fetch.js"
+import { williamhill } from "../platforms/williamhill-fetch.js"
+
+/* Deprecated
+import { astralcodexten } from "../platforms/astralcodexten-fetch.js"
+import { csetforetell } from "../platforms/csetforetell-fetch.js"
+import { elicit } from "../platforms/elicit-fetch.js"
+import { estimize } from "../platforms/estimize-fetch.js"
+import { hypermind } from "../platforms/hypermind-fetch.js"
+import { coupcast } from "../platforms/coupcast-fetch.js"
+*/
+
+export const platformFetchers = [betfair, fantasyscotus, foretold, goodjudgment, goodjudgmentopen, infer, ladbrokes, kalshi, manifoldmarkets, metaculus, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill]
+export const platformNames = [
+  "betfair",
+  "fantasyscotus",
+  "foretold",
+  "givewellopenphil",
+  "goodjudgment",
+  "goodjudmentopen",
+  "kalshi",
+  "ladbrokes",
+  "manifoldmarkets",
+  "metaculus",
+  "polymarket",
+  "predictit",
+  "rootclaim",
+  "smarkets",
+  "wildeford",
+  "williamhill",
+  "xrisk",
+];
+// deprecated: "astralcodexten", "csetforetell", "coupcast", "elicit", "estimize", "hypermind", "omen", etc

src/utils/stars.js

@@ -166,13 +166,15 @@ function calculateStarsLadbrokes(data) {
 
 function calculateStarsManifoldMarkets(data) {
   let nuno = (data) =>
-    data.volume7Days > 500 || (data.pool > 500 && data.volume7Days > 200)
+    data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100)
       ? 2
       : 1;
   let eli = (data) => null;
   let misha = (data) => null;
   let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)])
   let starsInteger = Math.round(starsDecimal);
+  // console.log(data);
+  // console.log(starsInteger);
   return starsInteger;
 }