feat: separate fetchers from upserts

src/backend/database/pg-wrapper.ts

@@ -1,6 +1,6 @@
 import pkg from "pg";
 
-import { platformNames } from "../platforms/all/platformNames";
+import { platforms } from "../platforms";
 import { hash } from "../utils/hash";
 import { roughSizeOfObject } from "../utils/roughSize";
 
@@ -18,7 +18,10 @@ const allowed_year_month_histories = [].concat(
     allowed_months.map((month) => `${year}_${month}`)
   )
 ); // h2022_01
-const tableNamesWhitelistLatest = ["combined", ...platformNames];
+const tableNamesWhitelistLatest = [
+  "combined",
+  ...platforms.map((platform) => platform.name),
+];
 const tableNamesWhiteListHistory = [
   ...allowed_years,
   ...allowed_year_month_histories,

src/backend/flow/doEverything.ts

@@ -1,11 +1,11 @@
-import { platformFetchers } from "../platforms/all-platforms";
+import { platforms, processPlatform } from "../platforms";
 import { rebuildAlgoliaDatabase } from "../utils/algolia";
 import { updateHistory } from "./history/updateHistory";
 import { mergeEverything } from "./mergeEverything";
 import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData";
 
 /* Do everything */
-function sleep(ms) {
+function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
@@ -27,7 +27,7 @@ export async function tryCatchTryAgain(fun) {
 
 export async function doEverything() {
   let functions = [
-    ...platformFetchers,
+    ...platforms.map((platform) => () => processPlatform(platform)),
     mergeEverything,
     rebuildAlgoliaDatabase,
     updateHistory,

src/backend/flow/mergeEverything.ts

@@ -1,11 +1,12 @@
 import { databaseRead, databaseUpsert } from "../database/database-wrapper";
-import { platformNames } from "../platforms/all-platforms";
+import { platforms } from "../platforms";
 
 /* Merge everything */
 
 export async function mergeEverythingInner() {
   let merged = [];
-  for (let platformName of platformNames) {
+  for (let platform of platforms) {
+    const platformName = platform.name;
     let json = await databaseRead({ group: platformName });
     console.log(`${platformName} has ${json.length} questions\n`);
     merged = merged.concat(json);

src/backend/index.ts

@@ -9,12 +9,12 @@ import { updateHistory } from "./flow/history/updateHistory";
 import { mergeEverything } from "./flow/mergeEverything";
 import { rebuildNetlifySiteWithNewData } from "./flow/rebuildNetliftySiteWithNewData";
 import { rebuildFrontpage } from "./frontpage";
-import { platformFetchers } from "./platforms/all-platforms";
+import { platforms, processPlatform } from "./platforms";
 import { rebuildAlgoliaDatabase } from "./utils/algolia";
 
 /* Support functions */
 let functions = [
-  ...platformFetchers,
+  ...platforms.map((platform) => () => processPlatform(platform)),
   mergeEverything,
   rebuildAlgoliaDatabase,
   updateHistory,
@@ -25,9 +25,9 @@ let functions = [
 ];
 
 let generateWhatToDoMessage = () => {
-  let l = platformFetchers.length;
+  let l = platforms.length;
-  let messagesForFetchers = platformFetchers.map(
+  let messagesForFetchers = platforms.map(
-    (fun, i) => `[${i}]: Download predictions from ${fun.name}`
+    (platform, i) => `[${i}]: Download predictions from ${platform.name}`
   );
   let otherMessages = [
     "Merge tables into one big table (and push the result to a pg database)",

src/backend/platforms/all-platforms.ts

@@ -1,2 +0,0 @@
-export { platformFetchers } from "./all/platformFetchers";
-export { platformNames } from "./all/platformNames";

src/backend/platforms/all/platformFetchers.ts

@@ -1,42 +0,0 @@
-import { betfair } from "../betfair-fetch";
-import { fantasyscotus } from "../fantasyscotus-fetch";
-import { foretold } from "../foretold-fetch";
-import { goodjudgment } from "../goodjudgment-fetch";
-import { goodjudgmentopen } from "../goodjudmentopen-fetch";
-import { infer } from "../infer-fetch";
-import { kalshi } from "../kalshi-fetch";
-import { manifoldmarkets } from "../manifoldmarkets-fetch";
-import { metaculus } from "../metaculus-fetch";
-import { polymarket } from "../polymarket-fetch";
-import { predictit } from "../predictit-fetch";
-import { rootclaim } from "../rootclaim-fetch";
-import { smarkets } from "../smarkets-fetch";
-import { wildeford } from "../wildeford-fetch";
-
-/* Deprecated
-import { astralcodexten } from "../platforms/astralcodexten-fetch"
-import { coupcast } from "../platforms/coupcast-fetch"
-import { csetforetell } from "../platforms/csetforetell-fetch"
-import { elicit } from "../platforms/elicit-fetch"
-import { estimize } from "../platforms/estimize-fetch"
-import { hypermind } from "../platforms/hypermind-fetch"
-import { ladbrokes } from "../platforms/ladbrokes-fetch";
-import { williamhill } from "../platforms/williamhill-fetch";
-*/
-
-export const platformFetchers = [
-  betfair,
-  fantasyscotus,
-  foretold,
-  goodjudgment,
-  goodjudgmentopen,
-  infer,
-  kalshi,
-  manifoldmarkets,
-  metaculus,
-  polymarket,
-  predictit,
-  rootclaim,
-  smarkets,
-  wildeford,
-];

src/backend/platforms/all/platformNames.ts

@@ -1,20 +0,0 @@
-// This needs to be its own file to avoid cyclical dependencies.
-export const platformNames = [
-  "betfair",
-  "fantasyscotus",
-  "foretold",
-  "givewellopenphil",
-  "goodjudgment",
-  "goodjudmentopen",
-  "infer",
-  "kalshi",
-  "manifoldmarkets",
-  "metaculus",
-  "polymarket",
-  "predictit",
-  "rootclaim",
-  "smarkets",
-  "wildeford",
-  "xrisk",
-];
-// deprecated: "astralcodexten", "csetforetell", "coupcast", "elicit", "estimize", "hypermind", "ladbrokes", "omen", "williamhill", etc

src/backend/platforms/betfair-fetch.ts

@@ -1,8 +1,9 @@
 /* Imports */
 import axios from "axios";
 import https from "https";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
+import { Forecast, PlatformFetcher } from "./";
 
 /* Definitions */
 let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
@@ -77,7 +78,7 @@ async function whipIntoShape(data) {
 async function processPredictions(data) {
   let predictions = await whipIntoShape(data);
   // console.log(JSON.stringify(predictions, null, 4))
-  let results = predictions.map((prediction) => {
+  let results: Forecast[] = predictions.map((prediction) => {
     /* if(Math.floor(Math.random() * 10) % 20 ==0){
        console.log(JSON.stringify(prediction, null, 4))
     } */
@@ -136,12 +137,9 @@ async function processPredictions(data) {
 
 /* Body */
 
-export async function betfair() {
+export const betfair: PlatformFetcher = async function () {
-  let data = await fetchPredictions();
+  const data = await fetchPredictions();
-  let results = await processPredictions(data); // somehow needed
+  const results = await processPredictions(data); // somehow needed
-  // console.log(results.map(result => ({title: result.title, description: result.description})))
+  return results;
-  // let string = JSON.stringify(results, null, 2)
+};
-  await databaseUpsert({ contents: results, group: "betfair" });
-  console.log("Done");
-}
 // betfair()

src/backend/platforms/fantasyscotus-fetch.ts

@@ -1,8 +1,8 @@
 /* Imports */
 import axios from "axios";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { calculateStars } from "../utils/stars";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let unixtime = new Date().getTime();
@@ -111,10 +111,9 @@ async function processData(data) {
 }
 
 /* Body */
-export async function fantasyscotus() {
+export const fantasyscotus: PlatformFetcher = async function () {
   let rawData = await fetchData();
   let results = await processData(rawData);
-  await databaseUpsert({ contents: results, group: "fantasyscotus" });
+  return results;
-  console.log("Done");
+};
-}
 //fantasyscotus()

src/backend/platforms/foretold-fetch.ts

@@ -1,6 +1,6 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
 
 /* Definitions */
@@ -98,8 +98,6 @@ export async function foretold() {
       results.push(result);
     });
   }
-  await databaseUpsert({ contents: results, group: "foretold" });
+  return results;
-
-  console.log("Done");
 }
 // foretold()

src/backend/platforms/goodjudgment-fetch.ts

@@ -3,9 +3,9 @@ import axios from "axios";
 import { Tabletojson } from "tabletojson";
 import tunnel from "tunnel";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { hash } from "../utils/hash";
 import { calculateStars } from "../utils/stars";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let endpoint = "https://goodjudgment.io/superforecasts/";
@@ -17,7 +17,7 @@ String.prototype.replaceAll = function replaceAll(search, replace) {
 /* Support functions */
 
 /* Body */
-export async function goodjudgment() {
+export const goodjudgment: PlatformFetcher = async function () {
   // Proxy fuckery
   let proxy;
   /*
@@ -116,14 +116,11 @@ export async function goodjudgment() {
       results.push(standardObj);
     }
   }
-  // console.log(results.slice(0,10))
-  let string = JSON.stringify(results, null, 2);
-  console.log(results);
-  await databaseUpsert({ contents: results, group: "goodjudgment" });
 
   console.log(
     "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js"
   );
-  console.log("Done");
+
-}
+  return results;
+};
 // goodjudgment()

src/backend/platforms/goodjudmentopen-fetch.ts

@@ -2,10 +2,10 @@
 import axios from "axios";
 import { Tabletojson } from "tabletojson";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { applyIfSecretExists } from "../utils/getSecrets";
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let htmlEndPoint = "https://www.gjopen.com/questions?page=";
@@ -150,7 +150,7 @@ function isEnd(html) {
   return isEndBool;
 }
 
-function sleep(ms) {
+function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
@@ -221,11 +221,10 @@ async function goodjudgmentopen_inner(cookie) {
       );
     }
   }
-  console.log(results);
+
-  if (results.length > 0) {
+  if (results.length === 0) {
-    await databaseUpsert({ contents: results, group: "goodjudmentopen" });
-  } else {
     console.log("Not updating results, as process was not signed in");
+    return;
   }
 
   let end = Date.now();
@@ -233,9 +232,11 @@ async function goodjudgmentopen_inner(cookie) {
   console.log(
     `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
   );
+
+  return results;
 }
 
-export async function goodjudgmentopen() {
+export const goodjudgmentopen: PlatformFetcher = async function () {
   let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
-  await applyIfSecretExists(cookie, goodjudgmentopen_inner);
+  return await applyIfSecretExists(cookie, goodjudgmentopen_inner);
-}
+};

src/backend/platforms/index.ts

@@ -0,0 +1,61 @@
+import { databaseUpsert } from "../database/database-wrapper";
+import { betfair } from "./betfair-fetch";
+import { fantasyscotus } from "./fantasyscotus-fetch";
+import { foretold } from "./foretold-fetch";
+import { goodjudgment } from "./goodjudgment-fetch";
+import { goodjudgmentopen } from "./goodjudmentopen-fetch";
+import { infer } from "./infer-fetch";
+import { kalshi } from "./kalshi-fetch";
+import { manifoldmarkets } from "./manifoldmarkets-fetch";
+import { metaculus } from "./metaculus-fetch";
+import { polymarket } from "./polymarket-fetch";
+import { predictit } from "./predictit-fetch";
+import { rootclaim } from "./rootclaim-fetch";
+import { smarkets } from "./smarkets-fetch";
+import { wildeford } from "./wildeford-fetch";
+
+export interface Forecast {
+  id: string;
+  title: string;
+  url: string;
+  description: string;
+  platform: string;
+  options: any[];
+  timestamp: string;
+  qualityindicators: any;
+  extra?: any;
+}
+
+export type PlatformFetcher = () => Promise<Forecast[] | null>;
+
+interface Platform {
+  name: string;
+  fetcher: PlatformFetcher;
+}
+
+export const platforms: Platform[] = [
+  betfair,
+  fantasyscotus,
+  foretold,
+  goodjudgment,
+  goodjudgmentopen,
+  infer,
+  kalshi,
+  manifoldmarkets,
+  metaculus,
+  polymarket,
+  predictit,
+  rootclaim,
+  smarkets,
+  wildeford,
+].map((fun) => ({ name: fun.name, fetcher: fun }));
+
+export const processPlatform = async (platform: Platform) => {
+  let results = await platform.fetcher();
+  if (results && results.length) {
+    await databaseUpsert({ contents: results, group: platform.name });
+    console.log("Done");
+  } else {
+    console.log(`Platform ${platform.name} didn't return any results`);
+  }
+};

src/backend/platforms/infer-fetch.ts

@@ -2,10 +2,10 @@
 import axios from "axios";
 import { Tabletojson } from "tabletojson";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { applyIfSecretExists } from "../utils/getSecrets";
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
+import { Forecast, PlatformFetcher } from "./";
 
 /* Definitions */
 let htmlEndPoint = "https://www.infer-pub.com/questions";
@@ -182,7 +182,7 @@ function sleep(ms) {
 async function infer_inner(cookie) {
   let i = 1;
   let response = await fetchPage(i, cookie);
-  let results = [];
+  let results: Forecast[] = [];
   let init = Date.now();
   // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while (!isEnd(response) && isSignedIn(response)) {
@@ -263,20 +263,21 @@ async function infer_inner(cookie) {
       );
     }
   }
-  if (results.length > 0) {
-    await databaseUpsert({ contents: results, group: "infer" });
-  } else {
-    console.log("Not updating results, as process was not signed in");
-  }
 
   let end = Date.now();
   let difference = end - init;
   console.log(
     `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
   );
+
+  if (results.length === 0) {
+    console.log("Not updating results, as process was not signed in");
+    return;
+  }
+  return results;
 }
 
-export async function infer() {
+export const infer: PlatformFetcher = async function () {
   let cookie = process.env.INFER_COOKIE;
-  await applyIfSecretExists(cookie, infer_inner);
+  return await applyIfSecretExists(cookie, infer_inner);
-}
+};

src/backend/platforms/kalshi-fetch.ts

@@ -1,7 +1,8 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
@@ -69,11 +70,8 @@ async function processMarkets(markets) {
 }
 
 /* Body */
-export async function kalshi() {
+export const kalshi: PlatformFetcher = async function () {
   let markets = await fetchAllMarkets();
-  let results = await processMarkets(markets); // somehow needed
+  return await processMarkets(markets);
-  await databaseUpsert({ contents: results, group: "kalshi" });
+};
-
-  console.log("Done");
-}
 // kalshi()

src/backend/platforms/manifoldmarkets-fetch.ts

@@ -1,6 +1,6 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
 
 /* Definitions */
@@ -89,12 +89,10 @@ async function processPredictions(predictions) {
 
 /* Body */
 
-export async function manifoldmarkets() {
+export const manifoldmarkets = async function () {
   let data = await fetchData();
   let results = await processPredictions(data); // somehow needed
   showStatistics(results);
-  await databaseUpsert({ contents: results, group: "manifoldmarkets" });
+  return results;
-
+};
-  console.log("Done");
-}
 // manifoldmarkets()

src/backend/platforms/metaculus-fetch.ts

@@ -1,8 +1,9 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
@@ -94,7 +95,7 @@ async function fetchMetaculusQuestionDescription(slug) {
 
 /* Body */
 
-export async function metaculus() {
+export const metaculus: PlatformFetcher = async function () {
   // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
   // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
   // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
@@ -189,8 +190,6 @@ export async function metaculus() {
     i = i + 1;
   }
 
-  await databaseUpsert({ contents: all_questions, group: "metaculus" });
+  return all_questions;
-
+};
-  console.log("Done");
-}
 //metaculus()

src/backend/platforms/polymarket-fetch.ts

@@ -1,11 +1,12 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
+import { Forecast, PlatformFetcher } from "./";
 
 /* Definitions */
 let graphQLendpoint =
-  "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
+  "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5";
 let units = 10 ** 6;
 
 async function fetchAllContractInfo() {
@@ -55,19 +56,24 @@ async function fetchAllContractData() {
   })
     .then((res) => res.data)
     .then((res) => res.data.fixedProductMarketMakers);
-  // console.log(response)
+
   return response;
 }
 
-async function fetch_all() {
+export const polymarket: PlatformFetcher = async function () {
   let allData = await fetchAllContractData();
   let allInfo = await fetchAllContractInfo();
 
+  let used = process.memoryUsage().heapUsed / 1024 / 1024;
+  console.log(
+    `The script uses approximately ${Math.round(used * 100) / 100} MB`
+  );
+
   let infos = {};
   for (let info of allInfo) {
     let address = info.marketMakerAddress;
     let addressLowerCase = address.toLowerCase();
-    //delete info.history
+
     if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long")
       infos[addressLowerCase] = {
         title: info.question,
@@ -83,9 +89,8 @@ async function fetch_all() {
   let results = [];
   for (let data of allData) {
     let addressLowerCase = data.id;
-    // console.log(data)
+
     if (infos[addressLowerCase] != undefined) {
-      // console.log(addressLowerCase)
       let id = `polymarket-${addressLowerCase.slice(0, 10)}`;
       let info = infos[addressLowerCase];
       let numforecasts = Number(data.tradesQuantity);
@@ -105,7 +110,7 @@ async function fetch_all() {
         });
       }
 
-      let result = {
+      let result: Forecast = {
         id: id,
         title: info.title,
         url: info.url,
@@ -126,23 +131,12 @@ async function fetch_all() {
         extra: {
           address: info.address,
         },
-        /*
-         */
       };
       if (info.category != "Sports") {
-        // console.log(result)
         results.push(result);
       }
     }
   }
-  return results; //resultsProcessed
-}
 
-/* Body */
+  return results;
-export async function polymarket() {
+};
-  let results = await fetch_all();
-  await databaseUpsert({ contents: results, group: "polymarket" });
-
-  console.log("Done");
-}
-// polymarket()

src/backend/platforms/predictit-fetch.ts

@@ -1,8 +1,9 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
+import { PlatformFetcher } from "./";
 
 /* Support functions */
 async function fetchmarkets() {
@@ -37,7 +38,7 @@ function sleep(ms) {
 }
 
 /* Body */
-export async function predictit() {
+export const predictit: PlatformFetcher = async function () {
   let markets = await fetchmarkets();
   let marketVolumes = await fetchmarketvolumes();
 
@@ -106,7 +107,6 @@ export async function predictit() {
     // console.log(obj)
     results.push(obj);
   }
-  await databaseUpsert({ contents: results, group: "predictit" });
 
-  console.log("Done");
+  return results;
-}
+};

src/backend/platforms/rootclaim-fetch.ts

@@ -1,8 +1,9 @@
 /* Imports */
 import axios from "axios";
-import { databaseUpsert } from "../database/database-wrapper";
+
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let jsonEndpoint =
@@ -22,7 +23,7 @@ async function fetchAllRootclaims() {
   return response;
 }
 
-async function fetchAndProcessData() {
+export const rootclaim: PlatformFetcher = async function () {
   let claims = await fetchAllRootclaims();
   let results = [];
   for (let claim of claims) {
@@ -53,13 +54,4 @@ async function fetchAndProcessData() {
     results.push(obj);
   }
   return results;
-}
+};
-
-/* Body */
-export async function rootclaim() {
-  let results = await fetchAndProcessData();
-  await databaseUpsert({ contents: results, group: "rootclaim" });
-
-  console.log("Done");
-}
-//rootclaim()

src/backend/platforms/smarkets-fetch.ts

@@ -1,8 +1,8 @@
 /* Imports */
 import axios from "axios";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { calculateStars } from "../utils/stars";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
@@ -61,7 +61,7 @@ async function fetchPrices(marketid) {
 
 /* Body */
 
-export async function smarkets() {
+export const smarkets: PlatformFetcher = async function () {
   let htmlPath =
     "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
 
@@ -172,9 +172,6 @@ export async function smarkets() {
     results.push(result);
   }
   VERBOSE ? console.log(results) : empty();
-
+  return results;
-  await databaseUpsert({ contents: results, group: "smarkets" });
+};
-  VERBOSE ? console.log(JSON.stringify(results, null, 4)) : empty();
-  VERBOSE ? console.dir(results, { depth: null }) : empty();
-}
 //smarkets()

src/backend/platforms/wildeford-fetch.ts

@@ -2,10 +2,10 @@
 // import axios from "axios"
 import { GoogleSpreadsheet } from "google-spreadsheet";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { applyIfSecretExists } from "../utils/getSecrets";
 import { hash } from "../utils/hash";
 import { calculateStars } from "../utils/stars";
+import { PlatformFetcher } from "./";
 
 /* Definitions */
 const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
@@ -120,15 +120,11 @@ async function processPredictions(predictions) {
 /* Body */
 export async function wildeford_inner(google_api_key) {
   let predictions = await fetchGoogleDoc(google_api_key);
-  let results = await processPredictions(predictions); // somehow needed
+  return await processPredictions(predictions);
-  // console.log(results.sort((a,b) => (a.title > b.title)))
-  await databaseUpsert({ contents: results, group: "wildeford" });
-
-  console.log("Done");
 }
 //example()
 
-export async function wildeford() {
+export const wildeford: PlatformFetcher = async function () {
   const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
-  await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);
+  return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);
-}
+};

src/backend/utils/getSecrets.ts

@@ -1,6 +1,9 @@
-export async function applyIfSecretExists(cookie, fun) {
+export async function applyIfSecretExists<T>(
+  cookie,
+  fun: (...args: any[]) => T
+) {
   if (cookie) {
-    await fun(cookie);
+    return await fun(cookie);
   } else if (!cookie) {
     console.log(
       `Cannot proceed with ${fun.name} because cookie does not exist`