feat: xrisk and givewellopenphil backend platforms

src/backend/manual/manualSendToDb.ts

@@ -1,20 +0,0 @@
-import fs from "fs";
-
-import { databaseUpsert } from "../database/database-wrapper";
-
-/* This is necessary for estimize, the database of x-risk estimates, and for the OpenPhil/GiveWell predictions. Unlike the others, I'm not fetching them constantly, but only once. */
-
-let pushManualFiles = ["givewellopenphil"]; // ["estimize", "givewellopenphil", "xrisk"]
-let suffixFiles = "-questions.json";
-
-let main = async () => {
-  for (let file of pushManualFiles) {
-    let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`, {
-      encoding: "utf-8",
-    });
-    let fileContents = JSON.parse(fileRaw);
-    console.log(fileContents);
-    await databaseUpsert({ contents: fileContents, group: file });
-  }
-};
-main();

src/backend/platforms/givewellopenphil.ts

@@ -4,9 +4,10 @@ import fs from "fs";
 
 import { databaseUpsert } from "../database/database-wrapper";
 import { calculateStars } from "../utils/stars";
+import { Platform } from "./";
 
 /* Support functions */
-async function fetchPage(url) {
+async function fetchPage(url: string) {
   let response = await axios({
     url: url,
     method: "GET",
@@ -14,7 +15,6 @@ async function fetchPage(url) {
       "Content-Type": "text/html",
     },
   }).then((res) => res.data);
-  //console.log(response)
   return response;
 }
 
@@ -64,17 +64,20 @@ async function main1() {
     group: "givewell-questions-unprocessed",
   });
 }
-// main1()
 
-async function main2() {
+export const givewellopenphil: Platform = {
-  let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json", {
+  name: "givewellopenphil",
-    encoding: "utf-8",
+  async fetcher() {
-  });
+    // main1()
-  let data = JSON.parse(rawdata);
+    return; // not necessary to refill the DB every time
-  let dataWithDate = data.map((datum) => ({
+    const rawdata = fs.readFileSync("./input/givewellopenphil-questions.json", {
-    ...datum,
+      encoding: "utf-8",
-    timestamp: "2021-02-23",
+    });
-  }));
+    const data = JSON.parse(rawdata);
-  await databaseUpsert({ group: "givewellopenphil", contents: dataWithDate });
+    const dataWithDate = data.map((datum: any) => ({
-}
+      ...datum,
-main2();
+      timestamp: "2021-02-23",
+    }));
+    return dataWithDate;
+  },
+};

src/backend/platforms/index.ts

@@ -2,6 +2,7 @@ import { databaseUpsert } from "../database/database-wrapper";
 import { betfair } from "./betfair";
 import { fantasyscotus } from "./fantasyscotus";
 import { foretold } from "./foretold";
+import { givewellopenphil } from "./givewellopenphil";
 import { goodjudgment } from "./goodjudgment";
 import { goodjudmentopen } from "./goodjudmentopen";
 import { infer } from "./infer";
@@ -13,6 +14,7 @@ import { predictit } from "./predictit";
 import { rootclaim } from "./rootclaim";
 import { smarkets } from "./smarkets";
 import { wildeford } from "./wildeford";
+import { xrisk } from "./xrisk";
 
 export interface Forecast {
   id: string;
@@ -31,7 +33,7 @@ export type PlatformFetcher = () => Promise<Forecast[] | null>;
 
 export interface Platform {
   name: string;
-  fetcher: PlatformFetcher;
+  fetcher?: PlatformFetcher;
 }
 
 // draft for the future callback-based streaming/chunking API:
@@ -53,6 +55,7 @@ export const platforms: Platform[] = [
   betfair,
   fantasyscotus,
   foretold,
+  givewellopenphil,
   goodjudgment,
   goodjudmentopen,
   infer,
@@ -64,9 +67,14 @@ export const platforms: Platform[] = [
   rootclaim,
   smarkets,
   wildeford,
+  xrisk,
 ];
 
 export const processPlatform = async (platform: Platform) => {
+  if (!platform.fetcher) {
+    console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`);
+    return;
+  }
   let results = await platform.fetcher();
   if (results && results.length) {
     await databaseUpsert({ contents: results, group: platform.name });

src/backend/platforms/xrisk.ts

@@ -0,0 +1,15 @@
+import fs from "fs";
+
+import { Platform } from "./";
+
+export const xrisk: Platform = {
+  name: "xrisk",
+  async fetcher() {
+    return; // not necessary to refill the DB every time
+    let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {
+      encoding: "utf-8",
+    });
+    const results = JSON.parse(fileRaw);
+    return results;
+  },
+};

src/backend/utils/misc/process-forecasts-from-xrisk.ts

@@ -5,10 +5,9 @@ import fs from "fs";
 let locationData = "../../data/";
 
 /* Body */
-let rawdata = fs.readFileSync(
+let rawdata = fs.readFileSync("./input/xrisk-questions.json", {
-  "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json",
+  encoding: "utf-8",
-  { encoding: "utf-8" }
+});
-);
 let data = JSON.parse(rawdata);
 
 let results = [];