Merge pull request #23 from QURIresearch/more-typescript

feat: More typescript & cleanups

src/backend/database/database-wrapper.ts

@@ -1,9 +1,5 @@
-import {
+import { mongoRead, mongoReadWithReadCredentials, mongoUpsert } from './mongo-wrapper';
-  mongoRead,
+import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper';
-  mongoReadWithReadCredentials,
-  mongoUpsert,
-} from "./mongo-wrapper";
-import { pgRead, pgReadWithReadCredentials, pgUpsert } from "./pg-wrapper";
 
 export async function databaseUpsert({ contents, group }) {
   // No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
@@ -146,7 +142,7 @@ export async function databaseReadWithReadCredentials({ group }) {
       break;
     default:
       mongoDocName = `${group}-questions`;
-      responseMongo = mongoRemongoReadWithReadCredentialsad(
+      responseMongo = mongoReadWithReadCredentials(
         mongoDocName,
         "metaforecastCollection",
         "metaforecastDatabase"

src/backend/database/mongo-wrapper.ts

@@ -1,5 +1,7 @@
-import pkg from "mongodb";
+import pkg from 'mongodb';
-import { roughSizeOfObject } from "../utils/roughSize";
+
+import { roughSizeOfObject } from '../utils/roughSize';
+
 const { MongoClient } = pkg;
 
 export async function mongoUpsert(

src/backend/database/pg-wrapper.ts

@@ -1,7 +1,9 @@
-import pkg from "pg";
+import pkg from 'pg';
-import { platformNames } from "../platforms/all/platformNames";
+
-import { hash } from "../utils/hash";
+import { platformNames } from '../platforms/all/platformNames';
-import { roughSizeOfObject } from "../utils/roughSize";
+import { hash } from '../utils/hash';
+import { roughSizeOfObject } from '../utils/roughSize';
+
 const { Pool } = pkg;
 
 // Definitions

src/backend/flow/doEverything.ts

@@ -1,8 +1,9 @@
-import { platformFetchers } from "../platforms/all-platforms";
+import { platformFetchers } from '../platforms/all-platforms';
-import { rebuildAlgoliaDatabase } from "../utils/algolia";
+import { rebuildAlgoliaDatabase } from '../utils/algolia';
-import { updateHistory } from "./history/updateHistory";
+import { updateHistory } from './history/updateHistory';
-import { mergeEverything } from "./mergeEverything";
+import { mergeEverything } from './mergeEverything';
-import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData";
+import { rebuildNetlifySiteWithNewData } from './rebuildNetliftySiteWithNewData';
+
 /* Do everything */
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));

src/backend/flow/doEverythingForScheduler.js

@@ -1,3 +0,0 @@
-import { doEverything } from "./doEverything";
-
-doEverything();

src/backend/flow/doEverythingForScheduler.ts

@@ -0,0 +1,3 @@
+import { doEverything } from './doEverything';
+
+doEverything();

src/backend/flow/history/old/addToHistory.js

@@ -1,110 +0,0 @@
-import {
-  databaseReadWithReadCredentials,
-  databaseUpsert,
-} from "../../database/database-wrapper";
-let isEmptyArray = (arr) => arr.length == 0;
-
-export async function addToHistory() {
-  let currentDate = new Date();
-  let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_");
-
-  let currentJSONwithMetaculus = await databaseReadWithReadCredentials({
-    group: "combined",
-  });
-  let currentJSON = currentJSONwithMetaculus.filter(
-    (element) =>
-      element.platform != "Metaculus" && element.platform != "Estimize"
-  ); // without Metaculus
-  // console.log(currentJSON.slice(0,20))
-  // console.log(currentJSON)
-  let historyJSON = await databaseReadWithReadCredentials({ group: "history" });
-  // console.log(historyJSON)
-
-  let currentForecastsWithAHistory = currentJSON.filter(
-    (element) =>
-      !isEmptyArray(
-        historyJSON.filter(
-          (historyElement) =>
-            historyElement.title == element.title &&
-            historyElement.url == element.url
-        )
-      )
-  );
-  // console.log(currentForecastsWithAHistory)
-
-  let currentForecastsWithoutAHistory = currentJSON.filter((element) =>
-    isEmptyArray(
-      historyJSON.filter(
-        (historyElement) =>
-          historyElement.title == element.title &&
-          historyElement.url == element.url
-      )
-    )
-  );
-  // console.log(currentForecastsWithoutAHistory)
-
-  // Add both types of forecast
-  let newHistoryJSON = [];
-  for (let historyElement of historyJSON) {
-    let correspondingNewElementArray = currentForecastsWithAHistory.filter(
-      (element) =>
-        historyElement.title == element.title &&
-        historyElement.url == element.url
-    );
-    // console.log(correspondingNewElement)
-    if (!isEmptyArray(correspondingNewElementArray)) {
-      let correspondingNewElement = correspondingNewElementArray[0];
-      let timeStampOfNewElement = correspondingNewElement.timestamp;
-      let doesHistoryAlreadyContainElement = historyElement.history
-        .map((element) => element.timestamp)
-        .includes(timeStampOfNewElement);
-      if (!doesHistoryAlreadyContainElement) {
-        let historyWithNewElement = historyElement["history"].concat({
-          timestamp: correspondingNewElement.timestamp,
-          options: correspondingNewElement.options,
-          qualityindicators: correspondingNewElement.qualityindicators,
-        });
-        let newHistoryElement = {
-          ...correspondingNewElement,
-          history: historyWithNewElement,
-        };
-        // If some element (like the description) changes, we keep the new one.
-        newHistoryJSON.push(newHistoryElement);
-      } else {
-        newHistoryJSON.push(historyElement);
-      }
-    } else {
-      // console.log(historyElement)
-      newHistoryJSON.push(historyElement);
-    }
-  }
-
-  for (let currentForecast of currentForecastsWithoutAHistory) {
-    let newHistoryElement = {
-      ...currentForecast,
-      history: [
-        {
-          timestamp: currentForecast.timestamp,
-          options: currentForecast.options,
-          qualityindicators: currentForecast.qualityindicators,
-        },
-      ],
-    };
-    delete newHistoryElement.timestamp;
-    delete newHistoryElement.options;
-    delete newHistoryElement.qualityindicators;
-    newHistoryJSON.push(newHistoryElement);
-  }
-
-  await databaseUpsert({ contents: newHistoryJSON, group: "history" });
-
-  // console.log(newHistoryJSON.slice(0,5))
-  // writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
-  // writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
-  //console.log(newHistoryJSON)
-  /*
-  let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
-  */
-  //console.log(new Date().toISOString())
-}
-// updateHistory()

src/backend/flow/history/old/createHistoryForMonth.js

@@ -1,36 +0,0 @@
-import { databaseRead, databaseUpsert } from "../../database/database-wrapper";
-
-export async function createHistoryForMonth() {
-  let currentDate = new Date();
-  let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_");
-  let metaforecasts = await databaseRead({ group: "combined" });
-  let metaforecastsHistorySeed = metaforecasts
-    .map((element) => {
-      // let moreoriginsdata = element.author ? ({author: element.author}) : ({})
-      return {
-        title: element.title,
-        url: element.url,
-        platform: element.platform,
-        moreoriginsdata: element.moreoriginsdata || {},
-        description: element.description,
-        history: [
-          {
-            timestamp: element.timestamp,
-            options: element.options,
-            qualityindicators: element.qualityindicators,
-          },
-        ],
-        extra: element.extra || {},
-      };
-    })
-    .filter(
-      (element) =>
-        element.platform != "Metaculus" && element.platform != "Estimize"
-    );
-  //console.log(metaforecastsHistorySeed)
-  await databaseUpsert({
-    contents: metaforecastsHistorySeed,
-    group: "history",
-  });
-}
-////createInitialHistory()

src/backend/flow/history/old/createInitialHistoryWithMetaculus.js

@@ -1,29 +0,0 @@
-import { databaseRead, databaseUpsert } from "../database-wrapper";
-
-let createInitialHistory = async () => {
-  let metaforecasts = await databaseRead({ group: "combined" });
-  let metaforecastsHistorySeed = metaforecasts.map((element) => {
-    // let moreoriginsdata = element.author ? ({author: element.author}) : ({})
-    return {
-      title: element.title,
-      url: element.url,
-      platform: element.platform,
-      moreoriginsdata: element.moreoriginsdata || {},
-      description: element.description,
-      history: [
-        {
-          timestamp: element.timestamp,
-          options: element.options,
-          qualityindicators: element.qualityindicators,
-        },
-      ],
-      extra: element.extra || {},
-    };
-  });
-  console.log(metaforecastsHistorySeed);
-  await databaseUpsert({
-    contents: metaforecastsHistorySeed,
-    group: "history",
-  });
-};
-createInitialHistory();

src/backend/flow/history/old/updateHistory.js

@@ -1,21 +0,0 @@
-import { addToHistory } from "./addToHistory";
-import { createHistoryForMonth } from "./createHistoryForMonth";
-
-export async function updateHistoryOld() {
-  let currentDate = new Date();
-  let dayOfMonth = currentDate.getDate();
-  if (dayOfMonth == 1) {
-    console.log(
-      `Creating history for the month ${currentDate.toISOString().slice(0, 7)}`
-    );
-    await createHistoryForMonth();
-  } else {
-    console.log(`Updating history for ${currentDate.toISOString()}`);
-    await addToHistory();
-  }
-}
-
-export async function updateHistory() {
-  let currentDate = new Date();
-  let year = currentDate.toISOString().slice(0, 4);
-}

src/backend/flow/history/updateHistory.ts

@@ -1,7 +1,4 @@
-import {
+import { databaseReadWithReadCredentials, databaseUpsert } from '../../database/database-wrapper';
-  databaseReadWithReadCredentials,
-  databaseUpsert,
-} from "../../database/database-wrapper";
 
 export async function updateHistory() {
   let latest = await databaseReadWithReadCredentials({ group: "combined" });

src/backend/flow/mergeEverything.ts

@@ -1,5 +1,6 @@
-import { databaseRead, databaseUpsert } from "../database/database-wrapper";
+import { databaseRead, databaseUpsert } from '../database/database-wrapper';
-import { platformNames } from "../platforms/all-platforms";
+import { platformNames } from '../platforms/all-platforms';
+
 /* Merge everything */
 
 export async function mergeEverythingInner() {

src/backend/flow/rebuildNetliftySiteWithNewData.ts

@@ -1,5 +1,6 @@
-import axios from "axios";
+import axios from 'axios';
-import { applyIfSecretExists } from "../utils/getSecrets";
+
+import { applyIfSecretExists } from '../utils/getSecrets';
 
 async function rebuildNetlifySiteWithNewData_inner(cookie) {
   let payload = {};

src/backend/manual/manualDownload.ts

@@ -1,6 +1,8 @@
-import "dotenv/config";
+import 'dotenv/config';
-import fs from "fs";
+
-import { databaseReadWithReadCredentials } from "../database/database-wrapper";
+import fs from 'fs';
+
+import { databaseReadWithReadCredentials } from '../database/database-wrapper';
 
 let main = async () => {
   let json = await databaseReadWithReadCredentials({ group: "combined" });

src/backend/manual/manualInitialize.js

@@ -1,3 +0,0 @@
-import { pgInitialize } from "../database/pg-wrapper";
-
-pgInitialize();

src/backend/manual/manualInitialize.ts

@@ -0,0 +1,3 @@
+import { pgInitialize } from '../database/pg-wrapper';
+
+pgInitialize();

src/backend/manual/manualSendToMongo.ts

@@ -1,5 +1,6 @@
-import fs from "fs";
+import fs from 'fs';
-import { databaseUpsert } from "../database/database-wrapper";
+
+import { databaseUpsert } from '../database/database-wrapper';
 
 /* This is necessary for estimize, the database of x-risk estimates, and for the OpenPhil/GiveWell predictions. Unlike the others, I'm not fetching them constantly, but only once. */
 
@@ -9,7 +10,9 @@ let suffixMongo = "-questions";
 
 let main = async () => {
   for (let file of pushManualFiles) {
-    let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`);
+    let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`, {
+      encoding: "utf-8",
+    });
     let fileContents = JSON.parse(fileRaw);
     console.log(fileContents);
     await databaseUpsert({ contents: fileContents, group: file });

src/backend/manual/pullSuperforecastsManually.ts

@@ -1,5 +1,5 @@
 /* Imports */
-import { goodjudgment } from "../platforms/goodjudgment-fetch";
+import { goodjudgment } from '../platforms/goodjudgment-fetch';
 
 /* Definitions */
 

src/backend/platforms/all/platformFetchers.ts

@@ -1,17 +1,17 @@
-import { betfair } from "../betfair-fetch";
+import { betfair } from '../betfair-fetch';
-import { fantasyscotus } from "../fantasyscotus-fetch";
+import { fantasyscotus } from '../fantasyscotus-fetch';
-import { foretold } from "../foretold-fetch";
+import { foretold } from '../foretold-fetch';
-import { goodjudgment } from "../goodjudgment-fetch";
+import { goodjudgment } from '../goodjudgment-fetch';
-import { goodjudgmentopen } from "../goodjudmentopen-fetch";
+import { goodjudgmentopen } from '../goodjudmentopen-fetch';
-import { infer } from "../infer-fetch";
+import { infer } from '../infer-fetch';
-import { kalshi } from "../kalshi-fetch";
+import { kalshi } from '../kalshi-fetch';
-import { manifoldmarkets } from "../manifoldmarkets-fetch";
+import { manifoldmarkets } from '../manifoldmarkets-fetch';
-import { metaculus } from "../metaculus-fetch";
+import { metaculus } from '../metaculus-fetch';
-import { polymarket } from "../polymarket-fetch";
+import { polymarket } from '../polymarket-fetch';
-import { predictit } from "../predictit-fetch";
+import { predictit } from '../predictit-fetch';
-import { rootclaim } from "../rootclaim-fetch";
+import { rootclaim } from '../rootclaim-fetch';
-import { smarkets } from "../smarkets-fetch";
+import { smarkets } from '../smarkets-fetch';
-import { wildeford } from "../wildeford-fetch";
+import { wildeford } from '../wildeford-fetch';
 
 /* Deprecated
 import { astralcodexten } from "../platforms/astralcodexten-fetch"

src/backend/platforms/deprecated/astralcodexten-fetch.js

@@ -1,97 +0,0 @@
-/* Imports */
-import axios from "axios";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { calculateStars } from "../utils/stars";
-
-/* Definitions */
-let graphQLendpoint = "https://api.foretold.io/graphql";
-let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"];
-
-/* Support functions */
-async function fetchAllCommunityQuestions(communityId) {
-  let response = await axios({
-    url: graphQLendpoint,
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    data: JSON.stringify({
-      query: `
-      query {
-        measurables(
-          channelId: "${communityId}",
-          states: OPEN,
-          first: 500
-        ){
-          total
-          edges{
-            node{
-              id
-              name
-              valueType
-              measurementCount
-              previousAggregate{
-                value{
-                  percentage
-                }
-              }
-            }
-          }
-        }
-      }
-      `,
-    }),
-  })
-    .then((res) => res.data)
-    .then((res) => res.data.measurables.edges);
-  //console.log(response)
-  return response;
-}
-
-/* Body */
-
-export async function astralcodexten() {
-  let results = [];
-  for (let community of ScottAlexanderPredictions) {
-    let questions = await fetchAllCommunityQuestions(community);
-    questions = questions.map((question) => question.node);
-    questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
-    questions.forEach((question) => {
-      let options = [];
-      if (question.valueType == "PERCENTAGE") {
-        let probability = question.previousAggregate.value.percentage;
-        options = [
-          {
-            name: "Yes",
-            probability: probability / 100,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - probability / 100,
-            type: "PROBABILITY",
-          },
-        ];
-      }
-      let result = {
-        title: question.name.split(". ")[1],
-        url: `https://www.foretold.io/c/${community}/m/${question.id}`,
-        platform: "AstralCodexTen",
-        description: "...by the end of 2021",
-        options: options,
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          numforecasts: Number((question.measurementCount + 1) / 2),
-          stars: calculateStars("AstralCodexTen", {}),
-        },
-        /*liquidity: liquidity.toFixed(2),
-          tradevolume: tradevolume.toFixed(2),
-          address: obj.address*/
-      };
-      // console.log(result)
-      results.push(result);
-    });
-  }
-  await databaseUpsert(results, "astralcodexten-questions");
-  // console.log(results)
-  console.log("Done");
-}
-// astralcodexten()

src/backend/platforms/deprecated/coupcast-fetch.js

@@ -1,184 +0,0 @@
-/* Imports */
-import axios from "axios";
-import Papa from "papaparse";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { calculateStars } from "../utils/stars";
-
-/* Definitions */
-let coupCastEndpoint =
-  "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv";
-var datenow = new Date();
-var currentmonth = datenow.getMonth() + 1;
-dd;
-/* Support functions */
-let unique = (arr) => [...new Set(arr)];
-let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-
-let sanitizeCountryName = (country_name) => {
-  let sanitized_name;
-  switch (country_name) {
-    case "Cen African Rep":
-      sanitized_name = "Central African Republic";
-      break;
-    case "Congo-Brz":
-      sanitized_name = "Republic of the Congo";
-      break;
-    case "Congo/Zaire":
-      sanitized_name = "Democratic Republic of the Congo";
-      break;
-    case "Czech Rep":
-      sanitized_name = "Czech Republic";
-      break;
-    case "Dominican Rep":
-      sanitized_name = "Dominican Republic";
-      break;
-    case "Korea North":
-      sanitized_name = "North Korea";
-      break;
-    case "Korea South":
-      sanitized_name = "South Korea";
-      break;
-    case "UKG":
-      sanitized_name = "UK";
-      break;
-    default:
-      sanitized_name = country_name;
-  }
-  return sanitized_name;
-};
-
-async function processArray(countryArray) {
-  let results = [];
-  for (let country of countryArray) {
-    let url = `https://www.oneearthfuture.org/activities/coup-cast`;
-
-    // We don't really want the prediction for all months; one is enough
-    // console.log(country.month)
-    if (Number(country.month) == currentmonth) {
-      // Monthly
-      country.country_name = sanitizeCountryName(country.country_name);
-      let processedPrediction1 = {
-        title: `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
-        url: url,
-        platform: "CoupCast",
-        description: `The current leader of ${country.country_name} is ${
-          country.leader_name
-        }, who has been in power for ${Number(country.leader_years).toFixed(
-          1
-        )} years.  ${
-          country.country_name
-        } has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${
-          country.regime_years
-        } years.`,
-        options: [
-          {
-            name: "Yes",
-            probability: country.month_risk,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - country.month_risk,
-            type: "PROBABILITY",
-          },
-        ],
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          stars: calculateStars("Coupcast", {}),
-        },
-        extra: {
-          country_name: country.country_name,
-          regime_type: country.regime_type,
-          month: country.month,
-          year: country.year,
-          leader_name: country.leader_name,
-          month_risk: country.month_risk,
-          annual_risk: country.annual_risk,
-          risk_change_percent: country.risk_change_percent,
-          regime_years: country.regime_years,
-          leader_years: country.leader_years,
-          country_code: country.country_code,
-          country_abb: country.country_abb,
-        },
-      };
-
-      // Yearly
-      let processedPrediction2 = {
-        title: `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
-        url: url,
-        platform: "CoupCast",
-        description: `The current leader of ${country.country_name} is ${
-          country.leader_name
-        }, who has been in power for ${Number(country.leader_years).toFixed(
-          1
-        )} years. ${
-          country.country_name
-        } has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${
-          country.regime_years
-        } years`,
-        options: [
-          {
-            name: "Yes",
-            probability: country.annual_risk,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - country.annual_risk,
-            type: "PROBABILITY",
-          },
-        ],
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          stars: calculateStars("CoupCast", {}),
-        },
-        extra: {
-          country_name: country.country_name,
-          regime_type: country.regime_type,
-          month: country.month,
-          year: country.year,
-          leader_name: country.leader_name,
-          month_risk: country.month_risk,
-          annual_risk: country.annual_risk,
-          risk_change_percent: country.risk_change_percent,
-          regime_years: country.regime_years,
-          leader_years: country.leader_years,
-          country_code: country.country_code,
-          country_abb: country.country_abb,
-        },
-      };
-
-      // results.push(processedPrediction1)
-      // Not pushing monthly
-      results.push(processedPrediction2);
-    }
-  }
-  await databaseUpsert(results, "coupcast-questions");
-  // console.log(results)
-  console.log("Done");
-}
-
-/* Body */
-
-export async function coupcast() {
-  let csvContent = await axios
-    .get(coupCastEndpoint)
-    .then((query) => query.data);
-  await Papa.parse(csvContent, {
-    header: true,
-    complete: async (results) => {
-      console.log("Downloaded", results.data.length, "records.");
-      /* console.log(
-        JSON.stringify(
-          unique(results.data.map(country => country.country_name)),
-          null, 
-          4
-        )
-      )*/
-      // console.log(results.data)
-      await processArray(results.data);
-    },
-  });
-  await sleep(1000); // needed to wait for Papaparse's callback to be executed.
-}
-// coupcast()

src/backend/platforms/deprecated/csetforetell-fetch.js

@@ -1,280 +0,0 @@
-/* Imports */
-import axios from "axios";
-import { Tabletojson } from "tabletojson";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { applyIfSecretExists } from "../utils/getSecrets";
-import { calculateStars } from "../utils/stars";
-import toMarkdown from "../utils/toMarkdown";
-
-/* Definitions */
-let htmlEndPoint = "https://www.cset-foretell.com/questions?page=";
-String.prototype.replaceAll = function replaceAll(search, replace) {
-  return this.split(search).join(replace);
-};
-const DEBUG_MODE = "on"; // "off"
-const SLEEP_TIME_RANDOM = 100; //5000 // miliseconds
-const SLEEP_TIME_EXTRA = 0; //1000
-/* Support functions */
-
-async function fetchPage(page, cookie) {
-  console.log(`Page #${page}`);
-  if (page == 1) {
-    cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :(
-  }
-  let urlEndpoint = htmlEndPoint + page;
-  console.log(urlEndpoint);
-  let response = await axios({
-    url: urlEndpoint,
-    method: "GET",
-    headers: {
-      "Content-Type": "text/html",
-      Cookie: cookie,
-    },
-  }).then((res) => res.data);
-  // console.log(response)
-  return response;
-}
-
-async function fetchStats(questionUrl, cookie) {
-  let response = await axios({
-    url: questionUrl + "/stats",
-    method: "GET",
-    headers: {
-      "Content-Type": "text/html",
-      Cookie: cookie,
-      Referer: questionUrl,
-    },
-  }).then((res) => res.data);
-
-  if (response.includes("Sign up or sign in to forecast")) {
-    throw Error("Not logged in");
-  }
-
-  // Is binary?
-  let isbinary = response.includes("binary?":true");
-  // console.log(`is binary? ${isbinary}`)
-  let options = [];
-  if (isbinary) {
-    // Crowd percentage
-    let htmlElements = response.split("\n");
-    // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
-    let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
-    // DEBUG_MODE == "on" ? console.log(h5elements) : id()
-    let crowdpercentage = h3Element.split(">")[1].split("<")[0];
-    let probability = Number(crowdpercentage.replace("%", "")) / 100;
-    options.push(
-      {
-        name: "Yes",
-        probability: probability,
-        type: "PROBABILITY",
-      },
-      {
-        name: "No",
-        probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
-        type: "PROBABILITY",
-      }
-    );
-  } else {
-    try {
-      let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table.
-      // console.log(optionsBody)
-      let optionsHtmlElement = "<table" + optionsBody + "table>";
-      let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
-      let firstTable = tablesAsJson[0];
-      options = firstTable.map((element) => ({
-        name: element["0"],
-        probability: Number(element["1"].replace("%", "")) / 100,
-        type: "PROBABILITY",
-      }));
-    } catch (error) {
-      let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position
-      let optionsHtmlElement = "<table" + optionsBody + "table>";
-      let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
-      let firstTable = tablesAsJson[0];
-      if (firstTable) {
-        options = firstTable.map((element) => ({
-          name: element["0"],
-          probability: Number(element["1"].replace("%", "")) / 100,
-          type: "PROBABILITY",
-        }));
-      } else {
-        // New type of question, tricky to parse the options
-        // Just leave options = [] for now.
-        // https://www.cset-foretell.com/blog/rolling-question-formats
-      }
-    }
-  }
-  // Description
-  let descriptionraw = response.split(`<meta name="description" content="`)[1];
-  let descriptionprocessed1 = descriptionraw.split(`">`)[0];
-  let descriptionprocessed2 = descriptionprocessed1.replace(">", "");
-  let descriptionprocessed3 = descriptionprocessed2.replace(
-    "To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.",
-    ``
-  );
-  // console.log(descriptionprocessed3)
-  let descriptionprocessed4 = descriptionprocessed3.replaceAll(
-    "\r\n\r\n",
-    "\n"
-  );
-  let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n");
-  let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`);
-  let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'");
-  let descriptionprocessed8 = toMarkdown(descriptionprocessed7);
-  let description = descriptionprocessed8;
-
-  // Number of forecasts
-  //console.log(response)
-  //console.log(response.split("prediction_sets_count&quot;:")[1])
-  let numforecasts = response
-    .split("prediction_sets_count&quot;:")[1]
-    .split(",")[0];
-  // console.log(numforecasts)
-
-  // Number of predictors
-  let numforecasters = response
-    .split("predictors_count&quot;:")[1]
-    .split(",")[0];
-  // console.log(numpredictors)
-
-  let result = {
-    description: description,
-    options: options,
-    timestamp: new Date().toISOString(),
-    qualityindicators: {
-      numforecasts: Number(numforecasts),
-      numforecasters: Number(numforecasters),
-      stars: calculateStars("CSET-foretell", { numforecasts }),
-    },
-  };
-
-  return result;
-}
-
-function isSignedIn(html) {
-  let isSignedInBool = !(
-    html.includes("You need to sign in or sign up before continuing") ||
-    html.includes("Sign up")
-  );
-  if (!isSignedInBool) {
-    console.log("Error: Not signed in.");
-  }
-  console.log(`Signed in? ${isSignedInBool}`);
-  return isSignedInBool;
-}
-
-function isEnd(html) {
-  let isEndBool = html.includes("No questions match your filter");
-  if (isEndBool) {
-    //console.log(html)
-  }
-  console.log(`IsEnd? ${isEndBool}`);
-  return isEndBool;
-}
-
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-/* Body */
-
-async function csetforetell_inner(cookie) {
-  let i = 1;
-  let response = await fetchPage(i, cookie);
-  let results = [];
-  let init = Date.now();
-  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
-  while (!isEnd(response) && isSignedIn(response)) {
-    let htmlLines = response.split("\n");
-    // let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
-    let questionHrefs = htmlLines.filter((str) =>
-      str.includes("https://www.cset-foretell.com/questions/")
-    );
-    // console.log(questionHrefs)
-
-    if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
-      //console.log(response)
-      console.log("questionHrefs: ");
-      console.log(questionHrefs);
-    }
-
-    //console.log("")
-    //console.log("")
-    //console.log(h4elements)
-
-    for (let questionHref of questionHrefs) {
-      //console.log(h4element)
-
-      let elementSplit = questionHref.split('"><span>');
-      let url = elementSplit[0].split('<a href="')[1];
-      let title = elementSplit[1]
-        .replace("</h4>", "")
-        .replace("</h5>", "")
-        .replace("</span></a>", "");
-      await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
-
-      try {
-        let moreinfo = await fetchStats(url, cookie);
-        let question = {
-          title: title,
-          url: url,
-          platform: "CSET-foretell",
-          ...moreinfo,
-        };
-        if (
-          i % 30 == 0 &&
-          !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
-        ) {
-          console.log(
-            `Page #${i}` &&
-              !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
-          );
-          console.log(question);
-        }
-        results.push(question);
-        if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
-          console.log(url);
-          console.log(question);
-        }
-      } catch (error) {
-        console.log(error);
-        console.log(
-          `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
-        );
-      }
-    }
-
-    i++;
-    //i=Number(i)+1
-
-    console.log(
-      "Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers"
-    );
-    await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
-
-    try {
-      response = await fetchPage(i, cookie);
-    } catch (error) {
-      console.log(error);
-      console.log(
-        `The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
-      );
-    }
-  }
-  if (results.length > 0) {
-    await databaseUpsert(results, "csetforetell-questions");
-  } else {
-    console.log("Not updating results, as process was not signed in");
-  }
-
-  let end = Date.now();
-  let difference = end - init;
-  console.log(
-    `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
-  );
-}
-
-export async function csetforetell() {
-  let cookie = process.env.CSETFORETELL_COOKIE;
-  await applyIfSecretExists(cookie, csetforetell_inner);
-}

src/backend/platforms/deprecated/elicit-fetch.js

@@ -1,121 +0,0 @@
-/* Imports */
-import axios from "axios";
-import Papa from "papaparse";
-import readline from "readline";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { calculateStars } from "../utils/stars";
-
-/* Definitions */
-let elicitEndpoint =
-  "https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";
-
-/* Support functions */
-let avg = (array) =>
-  array.reduce((a, b) => Number(a) + Number(b)) / array.length;
-let unique = (arr) => [...new Set(arr)];
-let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-
-async function processArray(arrayQuestions) {
-  let questions = arrayQuestions.map((question) => question.questionTitle);
-  let uniqueQuestions = unique(questions);
-  let questionsObj = {};
-  uniqueQuestions.forEach((questionTitle) => {
-    questionsObj[questionTitle] = {
-      title: questionTitle,
-      forecasters: [],
-      forecasts: [],
-    };
-  });
-  arrayQuestions.forEach((question) => {
-    // console.log(question.questionTitle)
-    let questionTitle = question.questionTitle;
-    let correspondingQuestion = questionsObj[questionTitle];
-    let forecasters = correspondingQuestion.forecasters.concat(
-      question.predictionCreator
-    );
-    let forecasts = correspondingQuestion.forecasts.concat(question.prediction);
-    questionsObj[questionTitle] = {
-      forecasters,
-      forecasts,
-    };
-  });
-  let onlyQuestionsWithMoreThan;
-
-  let results = [];
-  for (let question in questionsObj) {
-    let title = question;
-
-    let forecasters = questionsObj[question].forecasters;
-
-    let numforecasters = unique(forecasters).length;
-    if (numforecasters >= 10) {
-      let url = `https://elicit.org/binary?binaryQuestions.search=${title.replace(
-        / /g,
-        "%20"
-      )}&binaryQuestions.sortBy=popularity&limit=20&offset=0`;
-      let forecasts = questionsObj[question].forecasts;
-
-      //console.log(forecasts)
-      //console.log(avg(forecasts))
-      let probability = avg(forecasts) / 100;
-      let numforecasts = forecasts.length;
-      let standardObj = {
-        title: title,
-        url: url,
-        platform: "Elicit",
-        options: [
-          {
-            name: "Yes",
-            probability: probability,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - probability,
-            type: "PROBABILITY",
-          },
-        ],
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          numforecasts: Number(numforecasts),
-          numforecasters: Number(numforecasters),
-          stars: calculateStars("Elicit", {}),
-        },
-      };
-      results.push(standardObj);
-    }
-  }
-  await databaseUpsert(results, "elicit-questions");
-
-  console.log("Done");
-}
-
-async function awaitdownloadconfirmation(message, callback) {
-  const rl = readline.createInterface({
-    input: process.stdin,
-    output: process.stdout,
-  });
-  rl.question(message, (answer) => {
-    //console.log("Received");
-    rl.close();
-
-    callback();
-  });
-}
-
-/* Body */
-
-export async function elicit() {
-  let csvContent = await axios.get(elicitEndpoint).then((query) => query.data);
-  await Papa.parse(csvContent, {
-    header: true,
-    complete: async (results) => {
-      console.log("Downloaded", results.data.length, "records.");
-      //resolve(results.data);
-      //console.log(results.data)
-      await processArray(results.data);
-    },
-  });
-  await sleep(5000); // needed to wait for Papaparse's callback to be executed.
-}
-//elicit()

src/backend/platforms/deprecated/estimize-fetch.js

@@ -1,35 +0,0 @@
-import fs from "fs";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { calculateStars } from "../utils/stars";
-
-export async function estimize() {
-  let data = fs.readFileSync(
-    "./input/s-and-p-500-companies/companies.csv",
-    "utf8"
-  );
-
-  let splitData = data.split("\n");
-  let results = [];
-  for (let datum of splitData) {
-    if (datum != "") {
-      //console.log(datum)
-      let datumSplit = datum.split(",");
-      let companyStickerSymbol = datumSplit[0];
-      let companyName = datumSplit[1];
-      let standardObj = {
-        title: `Estimates for ${companyName} (${companyStickerSymbol})`,
-        url: `https://www.estimize.com/${companyStickerSymbol.toLowerCase()}`,
-        platform: "Estimize",
-        description: `A link to Estimize's forecasts for *${companyName}* (sticker symbol ${companyStickerSymbol}). Viewing them requires making a prediction, Wall Street estimates are provided for free`,
-        options: [],
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          stars: calculateStars("Estimize", {}),
-        },
-      };
-      results.push(standardObj);
-    }
-  }
-  await databaseUpsert(results, "estimize-questions");
-}
-//estimize()

src/backend/platforms/deprecated/hypermind-fetch.js

@@ -1,201 +0,0 @@
-/* Imports */
-import axios from "axios";
-import https from "https";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { applyIfSecretExists } from "../utils/getSecrets";
-import { calculateStars } from "../utils/stars";
-import toMarkdown from "../utils/toMarkdown";
-
-/* Definitions */
-let hypermindEnpoint1 = "https://predict.hypermind.com/dash/jsx.json";
-let hypermindEnpoint2 = "https://prod.hypermind.com/ngdp-jsx/jsx.json";
-const insecureHttpsAgent = new https.Agent({
-  rejectUnauthorized: false, // (NOTE: this will disable client verification)
-});
-
-/* Support Functions */
-String.prototype.replaceAll = function replaceAll(search, replace) {
-  return this.split(search).join(replace);
-};
-
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-/* Fetchers */
-async function fetchHypermindData1(slug) {
-  let jsx = `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`;
-  // console.log(jsx)
-  let response = await await axios(hypermindEnpoint1, {
-    credentials: "omit",
-    headers: {
-      "User-Agent":
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
-      Accept:
-        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-      "Accept-Language": "en-US,en;q=0.5",
-      "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
-      "Upgrade-Insecure-Requests": "1",
-      "Sec-Fetch-Dest": "document",
-      "Sec-Fetch-Mode": "navigate",
-      "Sec-Fetch-Site": "none",
-      "Sec-Fetch-User": "?1",
-      "Cache-Control": "max-age=0",
-    },
-    referrer: `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`,
-    data: jsx,
-    method: "POST",
-    mode: "cors",
-    httpsAgent: insecureHttpsAgent,
-  }).then((response) => response.data[0].questions);
-  //console.log(response)
-  return response;
-}
-
-async function fetchHypermindDataShowcases(slug, cookie) {
-  let response = await axios(hypermindEnpoint2, {
-    credentials: "include",
-    headers: {
-      "User-Agent": "",
-      Accept: "*/*",
-      "Accept-Language": "en-US,en;q=0.5",
-      "Content-Type": "application/json; charset=UTF-8",
-      //"Cookie": cookie
-    },
-    referrer:
-      "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
-    data: `[["showcase","getShowcase",{"showcase":"${slug}","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":false}}]]`,
-    method: "POST",
-    mode: "cors",
-    httpsAgent: insecureHttpsAgent,
-  })
-    .then((resp) => resp.data[0].items)
-    .then((items) => items.filter((item) => item.type == "IFP"))
-    .then((items) => items.map((item) => item.IFP));
-
-  // console.log(response)
-  // response.forEach(item => delete item.crowdFcstHist)
-  return response;
-}
-
-/* Body */
-async function hypermind_inner(cookie) {
-  // Hypermind panelists and competitors; dashboard type two: "showcase"
-  // https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=SLUG
-  // E.g., https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=AI2023
-  let slugs2 = ["AI2030", "Covid19", "DOSES", "H5N8", "NGDP", "JSAI", "AI2023"]; // []
-  let results2 = [];
-  for (let slug of slugs2) {
-    console.log(slug);
-    await sleep(1000 + Math.random() * 1000);
-    let response = await fetchHypermindDataShowcases(slug);
-    let objs = response.map((result) => {
-      let descriptionraw = result.props.details.split("<hr size=1>")[0];
-      let descriptionprocessed1 = toMarkdown(descriptionraw);
-      let descriptionprocessed2 = descriptionprocessed1.replaceAll(
-        "![image] ()",
-        ""
-      );
-      let descriptionprocessed3 = descriptionprocessed2.replaceAll(
-        " Forecasting Schedule ",
-        ""
-      );
-      let descriptionprocessed4 = descriptionprocessed3
-        .replaceAll("\n", " ")
-        .replaceAll("  ", " ");
-      let descriptionprocessed5 = descriptionprocessed4.replaceAll(
-        "Context:",
-        ""
-      );
-      let description =
-        descriptionprocessed5 || toMarkdown(result.props.details);
-      return {
-        title: result.props.title,
-        url:
-          "https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=" +
-          slug,
-        platform: "Hypermind",
-        description: description,
-        options: [],
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          stars: calculateStars("Hypermind", {}),
-          numforecasters: Number(result.fcsterCnt),
-        },
-      };
-    });
-    // console.log(objs)
-    results2.push(...objs);
-  }
-
-  // Prediction markets; dashboard type one.
-  // https://predict.hypermind.com/dash/dash/dash.html?list=SLUG
-  // e.g., https://predict.hypermind.com/dash/dash/dash.html?list=POL
-  let slugs1 = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]; // []
-  let results1 = [];
-
-  for (let slug of slugs1) {
-    console.log(slug);
-    await sleep(2000 + Math.random() * 2000);
-    let result = await fetchHypermindData1(slug);
-    let objs = result.map((res) => {
-      let descriptionraw = res.props.details;
-      let descriptionprocessed1 = descriptionraw.split("%%fr")[0];
-      let descriptionprocessed2 = descriptionprocessed1.replaceAll(
-        "<BR>",
-        "\n"
-      );
-      let descriptionprocessed3 = descriptionprocessed2.replace("%%en:", "");
-      let descriptionprocessed4 = descriptionprocessed3.replace(
-        `Shares of the correct outcome will be worth 100<sup>ℍ</sup>, while the others will be worthless (0<sup>ℍ</sup>).<p>`,
-        ""
-      );
-      let descriptionprocessed5 = toMarkdown(descriptionprocessed4);
-      let description = descriptionprocessed5
-        .replaceAll("\n", " ")
-        .replaceAll("  ", " ");
-      //console.log(res.otcms)
-      //let percentage = (res.otcms.length==2) ? Number(res.otcms[0].price).toFixed(0) +"%" : "none"
-      let options = res.otcms.map((option) => ({
-        name: option.props.title.split("%%fr")[0].replaceAll("%%en:", ""),
-        probability: Number(option.price) / 100,
-        type: "PROBABILITY",
-      }));
-      return {
-        title: res.props.title.split("%%fr")[0].replace("%%en:", ""),
-        url: "https://predict.hypermind.com/dash/dash/dash.html?list=" + slug,
-        platform: "Hypermind",
-        description: description,
-        options: options,
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          stars: calculateStars("Hypermind", {}),
-          // "numforecasters": res.fcsterCnt
-        },
-      };
-    });
-    // console.log(objs)
-    results1.push(...objs);
-  }
-
-  let resultsTotal = [...results1, ...results2];
-
-  let distinctTitles = [];
-  let resultsTotalUnique = [];
-  for (let result of resultsTotal) {
-    if (!distinctTitles.includes(result["title"])) {
-      resultsTotalUnique.push(result);
-      distinctTitles.push(result["title"]);
-    }
-  }
-  // console.log(resultsTotal)
-  // console.log(resultsTotalUnique)
-  console.log(resultsTotalUnique.length, "results");
-  await databaseUpsert(resultsTotalUnique, "hypermind-questions");
-}
-//hypermind()
-
-export async function hypermind() {
-  let cookie = process.env.HYPERMINDCOOKIE;
-  await applyIfSecretExists(cookie, hypermind_inner);
-}

src/backend/platforms/deprecated/ladbrokes-fetch.js

@@ -1,142 +0,0 @@
-/* Imports */
-import axios from "axios";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { calculateStars } from "../utils/stars";
-
-/* Definitions */
-let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`;
-let enpointDrillDown = (id) =>
-  `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`;
-
-// <header class="header-dropdown header-dropdown--large -expanded" data-id="
-
-/* Support functions */
-async function fetchUrl(url) {
-  let response = await axios(url, {
-    credentials: "include",
-    headers: {
-      "User-Agent":
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
-      Accept:
-        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-      "Accept-Language": "en-US,en;q=0.5",
-      "Upgrade-Insecure-Requests": "1",
-      "Cache-Control": "max-age=0",
-    },
-    method: "GET",
-    mode: "cors",
-  }).then((response) => response.data);
-  return response;
-}
-
-let processResults = async (json) => {
-  let results = [];
-  let children = json.SSResponse.children;
-  children.pop();
-  let ids = children.map((child) => child.event.id);
-  let markets = [];
-  for (let id of ids) {
-    let marketsContainer = await fetchUrl(enpointDrillDown(id));
-    let marketsObj = marketsContainer.SSResponse.children[0].event;
-    let newMarkets = marketsObj.children;
-    newMarkets = newMarkets.map((market) => ({
-      ...market.market,
-      parent: marketsObj.name,
-    }));
-    markets.push(...newMarkets);
-  }
-  let normalMarkets = markets.filter(
-    (market) => !market.name.includes("Specials")
-  );
-  //console.log(normalMarkets)
-
-  for (let normalMarket of normalMarkets) {
-    let title = normalMarket.parent + ": " + normalMarket.name;
-    title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
-    let options = normalMarket.children.map((child) => {
-      let name = child.outcome.name;
-
-      let priceData = child.outcome.children[0].price;
-      let priceDecimal = Number(priceData.priceDec);
-      let probability = 1 / priceDecimal;
-      let option = {
-        name: name,
-        probability: probability,
-        type: "PROBABILITY",
-      };
-      return option;
-    });
-    // normalize probabilities
-    let totalValue = options
-      .map((element) => Number(element.probability))
-      .reduce((a, b) => a + b, 0);
-    options = options.map((element) => ({
-      ...element,
-      probability: Number(element.probability) / totalValue,
-    }));
-
-    // Filter very unlikely probabilities: Not here, but on the front end
-    // options = options.filter(element => element.probability > 0.02)
-
-    let obj = {
-      title: title,
-      url: "https://sports.ladbrokes.com/sport/politics/outrights",
-      platform: "Ladbrokes",
-      description: "",
-      options: options,
-      timestamp: new Date().toISOString(),
-      qualityindicators: {
-        stars: calculateStars("Ladbrokes", {}),
-      },
-    };
-    results.push(obj);
-  }
-
-  let specialMarkets = markets.filter((market) =>
-    market.name.includes("Specials")
-  );
-  for (let specialMarket of specialMarkets) {
-    //let title = specialMarket.parent + ": " + specialMarket.name
-    //console.log(title)
-    specialMarket.children.forEach((child) => {
-      let name = specialMarket.parent.includes("Specials")
-        ? child.outcome.name
-        : specialMarket.parent + ": " + child.outcome.name;
-      name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
-      let priceData = child.outcome.children[0].price;
-      let priceDecimal = Number(priceData.priceDec);
-      let probability = 1 / priceDecimal;
-      let obj = {
-        title: name,
-        url: "https://sports.ladbrokes.com/sport/politics/outrights",
-        platform: "LadBrokes",
-        options: [
-          {
-            name: "Yes",
-            probability: probability,
-            type: "PROBABILITY",
-          },
-          {
-            name: "No",
-            probability: 1 - probability,
-            type: "PROBABILITY",
-          },
-        ],
-        qualityindicators: {
-          stars: calculateStars("Ladbrokes", {}),
-        },
-      };
-      results.push(obj);
-    });
-  }
-  return results;
-};
-
-/* Body */
-export async function ladbrokes() {
-  let response = await fetchUrl(endpointPolitics);
-  let results = await processResults(response);
-  await databaseUpsert(results, "ladbrokes-questions");
-  console.log("Done");
-}
-//ladbrokes()

src/backend/platforms/deprecated/omen-fetch.js

@@ -1,97 +0,0 @@
-/* Imports */
-import axios from "axios";
-import { databaseUpsert } from "../../utils/database-wrapper";
-import { calculateStars } from "../../utils/stars";
-
-/* Definitions */
-let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen";
-// "https://gateway.thegraph.com/api/[api-key]/subgraphs/id/0x0503024fcc5e1bd834530e69d592dbb6e8c03968-0"
-// 'https://api.thegraph.com/subgraphs/name/protofire/omen'
-// https://github.com/protofire/omen-subgraph
-// https://thegraph.com/explorer/subgraph/protofire/omen
-
-async function fetchAllContractData() {
-  let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 50; // last 30 days
-  let response = await axios({
-    url: graphQLendpoint,
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    data: JSON.stringify({
-      query: `
-      {
-          fixedProductMarketMakers(first: 1000,
-          where: {
-          lastActiveDay_gt: ${daysSinceEra}
-          }
-          ){
-            id
-            lastActiveDay
-            question{
-              title
-            }
-            outcomeSlotCount
-            outcomeTokenMarginalPrices
-            usdVolume
-            usdLiquidityMeasure
-            resolutionTimestamp
-        }
-      }
-      `,
-    }),
-  })
-    .then((res) => res.data)
-    .then((res) => res.data.fixedProductMarketMakers);
-  console.log(response);
-  return response;
-}
-
-async function fetch_all() {
-  let allData = await fetchAllContractData();
-  let results = [];
-  for (let data of allData) {
-    if (
-      (data.question != null) &
-      (data.usdLiquidityMeasure != "0") &
-      (data.resolutionTimestamp == null) &
-      (data.question.title != "ssdds")
-    ) {
-      // console.log(data)
-      // console.log(data.usdLiquidityMeasure)
-      let options = data.outcomeTokenMarginalPrices.map((price, slotnum) => {
-        let name = `Option ${slotnum}`;
-        if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 0)
-          name = "Yes";
-        if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 1)
-          name = "No";
-        return {
-          name: name,
-          probability: Number(price),
-          type: "PROBABILITY",
-        };
-      });
-
-      let obj = {
-        title: data.question.title,
-        url: "https://omen.eth.link/#/" + data.id,
-        platform: "Omen",
-        description: "",
-        options: options,
-        timestamp: new Date().toISOString(),
-        qualityindicators: {
-          stars: calculateStars("Omen", {}),
-        },
-      };
-      // console.log(obj)
-      results.push(obj);
-    }
-  }
-  return results;
-}
-
-/* Body */
-export async function omen() {
-  let results = await fetch_all();
-  await databaseUpsert(results, "omen-questions");
-  console.log("Done");
-}
-//omen()

src/backend/platforms/deprecated/williamhill-fetch.js

@@ -1,147 +0,0 @@
-/* Imports */
-import axios from "axios";
-import { databaseUpsert } from "../utils/database-wrapper";
-import { calculateStars } from "../utils/stars";
-
-/* Definitions */
-let endpoint = "https://sports.williamhill.com/betting/en-gb/politics";
-
-// <header class="header-dropdown header-dropdown--large -expanded" data-id="
-
-/* Support functions */
-async function fetchUrl(url) {
-  let response = await axios(url, {
-    credentials: "include",
-    headers: {
-      "User-Agent":
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
-      Accept:
-        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-      "Accept-Language": "en-US,en;q=0.5",
-      "Upgrade-Insecure-Requests": "1",
-      "Cache-Control": "max-age=0",
-    },
-    method: "GET",
-    mode: "cors",
-  }).then((response) => response.data);
-  return response;
-}
-
-let processResults = (html) => {
-  let results = [];
-
-  let chunks = html.split(
-    '<header class="header-dropdown header-dropdown--large -expanded" data-id="'
-  );
-  chunks.shift();
-
-  // Kamala Special
-  let kamalaspecial = chunks[0];
-  let kamalamarkets = kamalaspecial.split(
-    '<div class="btmarket__selection"><p class="btmarket__name"><span>'
-  );
-  kamalamarkets.shift();
-  for (let kamalamarket of kamalamarkets) {
-    let title = kamalamarket.split("</span>")[0];
-    let numerator = Number(kamalamarket.split('data-num="')[1].split('"')[0]);
-    let denominator = Number(
-      kamalamarket.split('data-denom="')[1].split('"')[0]
-    );
-    let probability = denominator / (numerator + denominator);
-    let obj = {
-      title: title,
-      url: "https://sports.williamhill.com/betting/en-gb/politics",
-      platform: "WilliamHill",
-      description: "",
-      timestamp: new Date().toISOString(),
-      options: [
-        {
-          name: "Yes",
-          probability: probability,
-          type: "PROBABILITY",
-        },
-        {
-          name: "No",
-          probability: 1 - probability,
-          type: "PROBABILITY",
-        },
-      ],
-      qualityindicators: {
-        stars: calculateStars("WilliamHill", {}),
-      },
-    };
-    results.push(obj);
-  }
-  chunks.shift();
-
-  // Deal with the other markets
-  for (let chunk of chunks) {
-    let title = chunk.split('"')[0];
-    let title2 = chunk.split('<a title="')[1].split('"')[0];
-    title = title.length > title2.length ? title : title2;
-
-    let options = [];
-    let alternatives = chunk.split(
-      '<div class="btmarket__selection"><p class="btmarket__name"><span>'
-    );
-    alternatives.shift();
-    for (let alternative of alternatives) {
-      let optionName = alternative.split("</span>")[0];
-      let numerator = Number(alternative.split('data-num="')[1].split('"')[0]);
-      let denominator = Number(
-        alternative.split('data-denom="')[1].split('"')[0]
-      );
-      let option = {
-        name: optionName,
-        probability: denominator / (numerator + denominator),
-        type: "PROBABILITY",
-      };
-      options.push(option);
-    }
-
-    // normalize probabilities
-    let totalValue = options
-      .map((element) => Number(element.probability))
-      .reduce((a, b) => a + b, 0);
-    options = options.map((element) => ({
-      ...element,
-      probability: Number(element.probability) / totalValue,
-    }));
-    // Filter very unlikely probabilities: Not here, but on the front end
-    // options = options.filter(element => element.probability > 0.02)
-
-    let obj = {
-      title: title,
-      url: "https://sports.williamhill.com/betting/en-gb/politics",
-      platform: "WilliamHill",
-      options: options,
-      qualityindicators: {
-        stars: calculateStars("WilliamHill", {}),
-      },
-    };
-    results = results.filter(
-      (result) => result.title.length > 4 && result.title != "2024 or later"
-    );
-    // removes some predictions because hard to parse.
-    results.push(obj);
-  }
-
-  //console.log(results)
-  return results;
-};
-
-let processhtml2 = (html) => {
-  html.split();
-};
-
-/* Body */
-export async function williamhill() {
-  let response = await fetchUrl(
-    "https://sports.williamhill.com/betting/en-gb/politics"
-  );
-  let results = processResults(response);
-  await databaseUpsert(results, "williamhill-questions");
-  console.log(results.sort((a, b) => a.title > b.title));
-  console.log("Done");
-}
-//williamhill()

src/backend/platforms/example-fetch.ts

@@ -1,7 +1,8 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { calculateStars } from "../utils/stars";
+import { databaseUpsert } from '../database/database-wrapper';
+import { calculateStars } from '../utils/stars';
 
 /* Definitions */
 let endpoint = "https://example.com/";
@@ -44,7 +45,9 @@ async function processPredictions(predictions) {
       options: options,
       timestamp: new Date().toISOString(),
       qualityindicators: {
-        stars: calculateStars("Example", { some: somex, factors: factors }),
+        stars: calculateStars("Example", {
+          /* some: somex, factors: factors */
+        }),
         other: prediction.otherx,
         indicators: prediction.indicatorx,
       },

src/backend/platforms/fantasyscotus-fetch.ts

@@ -1,7 +1,8 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { calculateStars } from "../utils/stars";
+import { databaseUpsert } from '../database/database-wrapper';
+import { calculateStars } from '../utils/stars';
 
 /* Definitions */
 let unixtime = new Date().getTime();
@@ -9,8 +10,8 @@ let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=
 
 async function fetchData() {
   let response = await axios({
+    method: "GET",
     url: endpoint,
-    credentials: "omit",
     headers: {
       "User-Agent":
         "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
@@ -19,11 +20,11 @@ async function fetchData() {
       "Content-Type": "application/x-www-form-urlencoded",
       "X-Requested-With": "XMLHttpRequest",
     },
-    referrer: "https://fantasyscotus.net/case/list/",
+    // referrer: "https://fantasyscotus.net/case/list/",
-    method: "GET",
+    // credentials: "omit",
-    mode: "cors",
+    // mode: "cors",
   }).then((res) => res.data);
-  //console.log(response)
+
   return response;
 }
 
@@ -31,8 +32,8 @@ async function getPredictionsData(caseUrl) {
   let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`;
   //console.log(newCaseUrl)
   let predictions = await axios({
+    method: "GET",
     url: newCaseUrl,
-    credentials: "include",
     headers: {
       "User-Agent":
         "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
@@ -41,9 +42,9 @@ async function getPredictionsData(caseUrl) {
       "Content-Type": "application/x-www-form-urlencoded",
       "X-Requested-With": "XMLHttpRequest",
     },
-    referrer: newCaseUrl,
+    // referrer: newCaseUrl,
-    method: "GET",
+    // credentials: "include",
-    mode: "cors",
+    // mode: "cors",
   }).then((res) => res.data);
 
   let predictionsAffirm = predictions.filter(
@@ -102,7 +103,6 @@ async function processData(data) {
           stars: calculateStars("FantasySCOTUS", {}),
         },
       };
-      // console.log(eventObject)
       results.push(eventObject);
     }
   }

src/backend/platforms/givewellopenphil-fetch.ts

@@ -1,8 +1,9 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import fs from "fs";
+import fs from 'fs';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { calculateStars } from "../utils/stars";
+import { databaseUpsert } from '../database/database-wrapper';
+import { calculateStars } from '../utils/stars';
 
 /* Support functions */
 async function fetchPage(url) {
@@ -66,7 +67,9 @@ async function main1() {
 // main1()
 
 async function main2() {
-  let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json");
+  let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json", {
+    encoding: "utf-8",
+  });
   let data = JSON.parse(rawdata);
   let dataWithDate = data.map((datum) => ({
     ...datum,

src/backend/platforms/goodjudgment-fetch.ts

@@ -1,10 +1,11 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import { Tabletojson } from "tabletojson";
+import { Tabletojson } from 'tabletojson';
-import tunnel from "tunnel";
+import tunnel from 'tunnel';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { hash } from "../utils/hash";
+import { databaseUpsert } from '../database/database-wrapper';
-import { calculateStars } from "../utils/stars";
+import { hash } from '../utils/hash';
+import { calculateStars } from '../utils/stars';
 
 /* Definitions */
 let endpoint = "https://goodjudgment.io/superforecasts/";
@@ -48,8 +49,8 @@ export async function goodjudgment() {
       headers: {
         "User-Agent": "Chrome",
       },
-      agent,
+      // agent,
-      port: 80,
+      // port: 80,
     })
     .then((query) => query.data);
 

src/backend/platforms/goodjudmentopen-fetch.ts

@@ -1,10 +1,11 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import { Tabletojson } from "tabletojson";
+import { Tabletojson } from 'tabletojson';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { applyIfSecretExists } from "../utils/getSecrets";
+import { databaseUpsert } from '../database/database-wrapper';
-import { calculateStars } from "../utils/stars";
+import { applyIfSecretExists } from '../utils/getSecrets';
-import toMarkdown from "../utils/toMarkdown";
+import { calculateStars } from '../utils/stars';
+import toMarkdown from '../utils/toMarkdown';
 
 /* Definitions */
 let htmlEndPoint = "https://www.gjopen.com/questions?page=";
@@ -14,8 +15,8 @@ let annoyingPromptUrls = [
   "https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
   "https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
 ];
-const DEBUG_MODE = "off"; // "on"
+const DEBUG_MODE: "on" | "off" = "off"; // "on"
-const id = (x) => x;
+const id = () => 0;
 
 /* Support functions */
 
@@ -121,7 +122,9 @@ async function fetchStats(questionUrl, cookie) {
         maxProbability,
       }),
     },
-  };
+    // this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now
+  } as any;
+
   return result;
 }
 

src/backend/platforms/infer-fetch.ts

@@ -1,17 +1,18 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import { Tabletojson } from "tabletojson";
+import { Tabletojson } from 'tabletojson';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { applyIfSecretExists } from "../utils/getSecrets";
+import { databaseUpsert } from '../database/database-wrapper';
-import { calculateStars } from "../utils/stars";
+import { applyIfSecretExists } from '../utils/getSecrets';
-import toMarkdown from "../utils/toMarkdown";
+import { calculateStars } from '../utils/stars';
+import toMarkdown from '../utils/toMarkdown';
 
 /* Definitions */
 let htmlEndPoint = "https://www.infer-pub.com/questions";
 String.prototype.replaceAll = function replaceAll(search, replace) {
   return this.split(search).join(replace);
 };
-const DEBUG_MODE = "off"; // "off"
+const DEBUG_MODE: "on" | "off" = "off"; // "off"
 const SLEEP_TIME_RANDOM = 7000; // miliseconds
 const SLEEP_TIME_EXTRA = 2000;
 /* Support functions */
@@ -229,10 +230,7 @@ async function infer_inner(cookie) {
           i % 30 == 0 &&
           !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
         ) {
-          console.log(
+          console.log(`Page #${i}`);
-            `Page #${i}` &&
-              !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
-          );
           console.log(question);
         }
         results.push(question);

src/backend/platforms/smarkets-fetch.ts

@@ -1,12 +1,13 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { calculateStars } from "../utils/stars";
+import { databaseUpsert } from '../database/database-wrapper';
+import { calculateStars } from '../utils/stars';
 
 /* Definitions */
 let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
 let VERBOSE = false;
-let empty = (x) => x;
+let empty = () => 0;
 /* Support functions */
 
 async function fetchEvents(url) {
@@ -82,7 +83,7 @@ export async function smarkets() {
     }));
     VERBOSE ? console.log("Markets fetched") : empty();
     VERBOSE ? console.log(event.id) : empty();
-    VERBOSE ? console.log(market) : empty();
+    VERBOSE ? console.log(eventMarkets) : empty();
     markets.push(...eventMarkets);
     //let lastPrices = await fetchPrices(market.id)
   }
@@ -102,29 +103,29 @@ export async function smarkets() {
       ? console.log("Prices: ", prices["last_executed_prices"][market.id])
       : empty();
 
-    let options = {};
+    let optionsObj = {};
     for (let contract of contracts["contracts"]) {
-      options[contract.id] = { name: contract.name };
+      optionsObj[contract.id] = { name: contract.name };
     }
     for (let price of prices["last_executed_prices"][market.id]) {
-      options[price.contract_id] = {
+      optionsObj[price.contract_id] = {
-        ...options[price.contract_id],
+        ...optionsObj[price.contract_id],
         probability: price.last_executed_price
           ? Number(price.last_executed_price)
           : null,
         type: "PROBABILITY",
       };
     }
-    options = Object.values(options);
+    let options: any[] = Object.values(optionsObj);
     // monkey patch the case where there are only two options and only one has traded.
     if (
       options.length == 2 &&
       options.map((option) => option.probability).includes(null)
     ) {
       let nonNullPrice =
-        option[0].probability == null
+        options[0].probability == null
-          ? option[1].probability
+          ? options[1].probability
-          : option[0].probability;
+          : options[0].probability;
       options = options.map((option) => {
         let probability = option.probability;
         return {

src/backend/platforms/wildeford-fetch.ts

@@ -1,10 +1,11 @@
 /* Imports */
 // import axios from "axios"
-import { GoogleSpreadsheet } from "google-spreadsheet";
+import { GoogleSpreadsheet } from 'google-spreadsheet';
-import { databaseUpsert } from "../database/database-wrapper";
+
-import { applyIfSecretExists } from "../utils/getSecrets";
+import { databaseUpsert } from '../database/database-wrapper';
-import { hash } from "../utils/hash";
+import { applyIfSecretExists } from '../utils/getSecrets';
-import { calculateStars } from "../utils/stars";
+import { hash } from '../utils/hash';
+import { calculateStars } from '../utils/stars';
 
 /* Definitions */
 const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
@@ -98,7 +99,7 @@ async function processPredictions(predictions) {
         Date.parse(prediction["Prediction Date"] + "Z")
       ).toISOString(),
       qualityindicators: {
-        stars: calculateStars("Peter Wildeford"),
+        stars: calculateStars("Peter Wildeford", null),
       },
     };
     return result;

src/backend/utils/algolia.ts

@@ -1,6 +1,7 @@
-import algoliasearch from "algoliasearch";
+import algoliasearch from 'algoliasearch';
-import { databaseReadWithReadCredentials } from "../database/database-wrapper";
+
-import { mergeEverythingInner } from "../flow/mergeEverything";
+import { databaseReadWithReadCredentials } from '../database/database-wrapper';
+import { mergeEverythingInner } from '../flow/mergeEverything';
 
 let cookie = process.env.ALGOLIA_MASTER_API_KEY;
 const client = algoliasearch("96UD3NTQ7L", cookie);

src/backend/utils/evaluations/pullForecastsToCSVForRating.ts

@@ -1,6 +1,7 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
-import { databaseReadWithReadCredentials } from "../database-wrapper";
+
+import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
 
 /* Definitions */
 

src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts

@@ -1,6 +1,7 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
-import { databaseReadWithReadCredentials } from "../database-wrapper";
+
+import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
 
 /* Definitions */
 

src/backend/utils/hash.ts

@@ -1,4 +1,4 @@
-import crypto from "crypto";
+import crypto from 'crypto';
 
-export const hash = (string) =>
+export const hash = (string: string) =>
   crypto.createHash("sha256").update(string).digest("hex").slice(0, 10);

src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.ts

@@ -1,11 +1,12 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
 
 /* Definitions */
 let locationData = "../../input";
 /* Body */
 let rawdata = fs.readFileSync(
-  `${locationData}/givewellopenphil-questions.json`
+  `${locationData}/givewellopenphil-questions.json`,
+  { encoding: "utf-8" }
 );
 let data = JSON.parse(rawdata);
 

src/backend/utils/misc/process-forecasts-from-old-xrisk.ts

@@ -1,12 +1,13 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
 
 /* Definitions */
 let locationData = "../../data/";
 
 /* Body */
 let rawdata = fs.readFileSync(
-  "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json"
+  "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json",
+  { encoding: "utf-8" }
 );
 let data = JSON.parse(rawdata);
 

src/backend/utils/misc/process-forecasts-from-xrisk.ts

@@ -1,12 +1,13 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
 
 /* Definitions */
 let locationData = "../../data/";
 
 /* Body */
 let rawdata = fs.readFileSync(
-  "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json"
+  "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json",
+  { encoding: "utf-8" }
 );
 let data = JSON.parse(rawdata);
 

src/backend/utils/misc/process-forecasts-into-elicit.ts

@@ -1,6 +1,7 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
-import { databaseReadWithReadCredentials } from "../database-wrapper";
+
+import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
 
 /* Definitions */
 let locationData = "./data/";

src/backend/utils/misc/process-forecasts-template.ts

@@ -1,11 +1,13 @@
 /* Imports */
-import fs from "fs";
+import fs from 'fs';
 
 /* Definitions */
 let locationData = "./data/";
 
 /* Body */
-let rawdata = fs.readFileSync("../data/merged-questions.json");
+let rawdata = fs.readFileSync("../data/merged-questions.json", {
+  encoding: "utf-8",
+});
 let data = JSON.parse(rawdata);
 
 let results = [];
@@ -13,5 +15,5 @@ for (let datum of data) {
   // do something
 }
 
-let string = JSON.stringify(result, null, 2);
+let string = JSON.stringify(results, null, 2);
 fs.writeFileSync("../data/output.txt", string);

src/backend/utils/misc/test.ts

@@ -1,5 +1,4 @@
-import fs from "fs";
+import axios from 'axios';
-import axios from "axios";
 
 let elicitEndpoint =
   "https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";

src/pages/api/platform/[platform].js

@@ -1,6 +0,0 @@
-// see: https://nextjs.org/docs/api-routes/dynamic-api-routes
-
-export default function handler(req, res) {
-  const { platform } = req.query;
-  res.end(`Platform: ${platform}`);
-}

src/pages/api/question/[question].js

@@ -1,6 +0,0 @@
-// see: https://nextjs.org/docs/api-routes/dynamic-api-routes
-
-export default function handler(req, res) {
-  const { question } = req.query;
-  res.end(`Platform: ${question}`);
-}

src/pages/secretDashboard.tsx

@@ -1,11 +1,11 @@
 /* Imports */
 
 // React
-import { useRouter } from "next/router"; // https://nextjs.org/docs/api-reference/next/router
+import { useRouter } from 'next/router'; // https://nextjs.org/docs/api-reference/next/router
-import React, { useState } from "react";
+import { useState } from 'react';
-// Data
+
-import displayForecasts from "../web/display/displayForecasts";
+import displayForecasts from '../web/display/displayForecasts';
-import { getDashboardForecastsByDashboardId } from "../web/worker/getDashboardForecasts";
+import { getDashboardForecastsByDashboardId } from '../web/worker/getDashboardForecasts';
 
 /* get Props */
 

src/web/worker/getForecasts.js

@@ -1,90 +0,0 @@
-/*
-import {MongoClient} from 'mongodb';
-// import pkg from 'mongodb';
-// const {MongoClient} = pkg;
-
-async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
-  const url = "mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database.
-
-  const client = new MongoClient(url, {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  });
-
-  let documentContents
-  try {
-      await client.connect();
-      // console.log(`Connected correctly to server to read ${documentName}`);
-      const db = client.db(databaseName);
-  
-      // Use the collection "data"
-      const collection = db.collection(collectionName);
-  
-      // Search options
-      const query = { "name": documentName };
-      const options = {
-        // sort matched documents in descending order by rating
-        sort: { rating: -1 },
-      };
-  
-      // Insert a single document, wait for promise so we can read it back
-      // const p = await collection.insertOne(metaforecastDocument);
-      const document = await collection.findOne(query, options);
-      documentContents = document.contentsArray
-    } catch (err) {
-      console.log(err.stack);
-    }
-    finally {
-      await client.close();
-    }
-    // console.log(documentContents.slice(0,10));
-    return documentContents
-}
-
-export async function getForecasts() {
-  const mongodbDocument = await mongoRead("metaforecasts")
-  // console.log(mongodbDocument)
-  return mongodbDocument
-}
-
-// getForecasts()
-
-//---------------------------------------//
-
-//- Previous methods: Graphql -//
-import { GraphQLClient } from "graphql-request";
-// import { request } from 'graphql-request'
-
-const graphcms = new GraphQLClient(
-  "https://api.baseql.com/airtable/graphql/apptDvDsHzSDEDARC"
-);
-
-export async function getForecasts0() {
-  return await graphcms.request(
-    `
-        query {
-            metaforecasts{
-              id
-              title
-              url
-              platform
-              binaryQuestion
-              percentage
-              forecasts
-              description
-              stars
-            }
-          }`
-  );
-}
-
-//- Previous methods: Github JSON -//
-import axios from "axios"
-
-export async function getForecasts1() {
-  const { data } = await axios.get(`https://raw.githubusercontent.com/QURIresearch/metaforecast/master/data/metaforecasts.json?ver=${Math.random().toFixed(10)}`) // this is, for now, a hack
-  //console.log(data)
-  return data
-}
-
-*/

src/web/worker/searchGuesstimate.ts

@@ -1,5 +1,5 @@
 /* Imports */
-import axios from "axios";
+import axios from 'axios';
 
 /* Definitions */
 let urlEndPoint =
@@ -10,15 +10,15 @@ let urlEndPoint =
 export default function searchGuesstimate(query) {
   let response = axios({
     url: urlEndPoint,
-    credentials: "omit",
+    // credentials: "omit",
     headers: {
       // "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0",
       Accept: "application/json",
       "Accept-Language": "en-US,en;q=0.5",
       "content-type": "application/x-www-form-urlencoded",
     },
-    referrer:
+    // referrer:
-      "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
+    //   "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
     data: `{\"params\":\"query=${query.replace(
       / /g,
       "%20"

src/web/worker/searchWithAlgolia.ts

@@ -1,4 +1,4 @@
-import algoliasearch from "algoliasearch";
+import algoliasearch from 'algoliasearch';
 
 const client = algoliasearch("96UD3NTQ7L", "618dbd0092971388cfd43aac1ae5f1f5"); // Only search.
 const index = client.initIndex("metaforecast");
@@ -89,7 +89,7 @@ export default async function searchWithAlgolia({
     //facetFilters: buildFacetFilter({filterByPlatforms}),
     getRankingInfo: true,
   });
-  let results = response.hits;
+  let results: any[] = response.hits;
   console.log(
     "searchWithAlgolia.js/searchWithAlgolia/queryString",
     queryString

src/web/worker/uploadToImgur.ts

@@ -1,8 +1,8 @@
 // import fetch from "fetch"
-import axios from "axios";
+import axios, { AxiosRequestConfig } from 'axios';
 
 export async function uploadToImgur(dataURL, handleGettingImgurlImage) {
-  let request = {
+  let request: AxiosRequestConfig = {
     method: "post",
     url: "https://api.imgur.com/3/image",
     headers: {
@@ -12,7 +12,7 @@ export async function uploadToImgur(dataURL, handleGettingImgurlImage) {
       type: "base64",
       image: dataURL.split(",")[1],
     },
-    redirect: "follow",
+    // redirect: "follow",
   };
   let url;
   try {

tsconfig.json

@@ -1,6 +1,5 @@
 {
   "compilerOptions": {
-    "allowJs": true,
     "target": "es6",
     "module": "CommonJS",
     "outDir": "out",
@@ -23,7 +22,6 @@
   "include": [
     "next-env.d.ts",
     "src/**/*.ts",
-    "src/**/*.js",
     "src/**/*.tsx"
   ],
   "exclude": [