From efae18600f91e763798a5f3f8e70994f6600f330 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Tue, 29 Mar 2022 04:34:27 +0300 Subject: [PATCH] feat: separate fetchers from upserts --- src/backend/database/pg-wrapper.ts | 7 ++- src/backend/flow/doEverything.ts | 6 +- src/backend/flow/mergeEverything.ts | 5 +- src/backend/index.ts | 10 +-- src/backend/platforms/all-platforms.ts | 2 - src/backend/platforms/all/platformFetchers.ts | 42 ------------- src/backend/platforms/all/platformNames.ts | 20 ------ src/backend/platforms/betfair-fetch.ts | 18 +++--- src/backend/platforms/fantasyscotus-fetch.ts | 9 ++- src/backend/platforms/foretold-fetch.ts | 6 +- src/backend/platforms/goodjudgment-fetch.ts | 13 ++-- .../platforms/goodjudmentopen-fetch.ts | 19 +++--- src/backend/platforms/index.ts | 61 +++++++++++++++++++ src/backend/platforms/infer-fetch.ts | 21 ++++--- src/backend/platforms/kalshi-fetch.ts | 12 ++-- .../platforms/manifoldmarkets-fetch.ts | 10 ++- src/backend/platforms/metaculus-fetch.ts | 11 ++-- src/backend/platforms/polymarket-fetch.ts | 36 +++++------ src/backend/platforms/predictit-fetch.ts | 10 +-- src/backend/platforms/rootclaim-fetch.ts | 16 ++--- src/backend/platforms/smarkets-fetch.ts | 11 ++-- src/backend/platforms/wildeford-fetch.ts | 14 ++--- src/backend/utils/getSecrets.ts | 7 ++- 23 files changed, 169 insertions(+), 197 deletions(-) delete mode 100644 src/backend/platforms/all-platforms.ts delete mode 100644 src/backend/platforms/all/platformFetchers.ts delete mode 100644 src/backend/platforms/all/platformNames.ts create mode 100644 src/backend/platforms/index.ts diff --git a/src/backend/database/pg-wrapper.ts b/src/backend/database/pg-wrapper.ts index cfa91d5..09d2aae 100644 --- a/src/backend/database/pg-wrapper.ts +++ b/src/backend/database/pg-wrapper.ts @@ -1,6 +1,6 @@ import pkg from "pg"; -import { platformNames } from "../platforms/all/platformNames"; +import { platforms } from "../platforms"; import { hash } from "../utils/hash"; import { roughSizeOfObject } from "../utils/roughSize"; @@ -18,7 +18,10 @@ const allowed_year_month_histories = [].concat( allowed_months.map((month) => `${year}_${month}`) ) ); // h2022_01 -const tableNamesWhitelistLatest = ["combined", ...platformNames]; +const tableNamesWhitelistLatest = [ + "combined", + ...platforms.map((platform) => platform.name), +]; const tableNamesWhiteListHistory = [ ...allowed_years, ...allowed_year_month_histories, diff --git a/src/backend/flow/doEverything.ts b/src/backend/flow/doEverything.ts index 47b50ae..1f61bb4 100644 --- a/src/backend/flow/doEverything.ts +++ b/src/backend/flow/doEverything.ts @@ -1,11 +1,11 @@ -import { platformFetchers } from "../platforms/all-platforms"; +import { platforms, processPlatform } from "../platforms"; import { rebuildAlgoliaDatabase } from "../utils/algolia"; import { updateHistory } from "./history/updateHistory"; import { mergeEverything } from "./mergeEverything"; import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData"; /* Do everything */ -function sleep(ms) { +function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -27,7 +27,7 @@ export async function tryCatchTryAgain(fun) { export async function doEverything() { let functions = [ - ...platformFetchers, + ...platforms.map((platform) => () => processPlatform(platform)), mergeEverything, rebuildAlgoliaDatabase, updateHistory, diff --git a/src/backend/flow/mergeEverything.ts b/src/backend/flow/mergeEverything.ts index ad06214..ba50b88 100644 --- a/src/backend/flow/mergeEverything.ts +++ b/src/backend/flow/mergeEverything.ts @@ -1,11 +1,12 @@ import { databaseRead, databaseUpsert } from "../database/database-wrapper"; -import { platformNames } from "../platforms/all-platforms"; +import { platforms } from "../platforms"; /* Merge everything */ export async function mergeEverythingInner() { let merged = []; - for (let platformName of platformNames) { + for (let platform of platforms) { + const platformName = platform.name; let json = await databaseRead({ group: platformName }); console.log(`${platformName} has ${json.length} questions\n`); merged = merged.concat(json); diff --git a/src/backend/index.ts b/src/backend/index.ts index 2961419..b81a09f 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -9,12 +9,12 @@ import { updateHistory } from "./flow/history/updateHistory"; import { mergeEverything } from "./flow/mergeEverything"; import { rebuildNetlifySiteWithNewData } from "./flow/rebuildNetliftySiteWithNewData"; import { rebuildFrontpage } from "./frontpage"; -import { platformFetchers } from "./platforms/all-platforms"; +import { platforms, processPlatform } from "./platforms"; import { rebuildAlgoliaDatabase } from "./utils/algolia"; /* Support functions */ let functions = [ - ...platformFetchers, + ...platforms.map((platform) => () => processPlatform(platform)), mergeEverything, rebuildAlgoliaDatabase, updateHistory, @@ -25,9 +25,9 @@ let functions = [ ]; let generateWhatToDoMessage = () => { - let l = platformFetchers.length; - let messagesForFetchers = platformFetchers.map( - (fun, i) => `[${i}]: Download predictions from ${fun.name}` + let l = platforms.length; + let messagesForFetchers = platforms.map( + (platform, i) => `[${i}]: Download predictions from ${platform.name}` ); let otherMessages = [ "Merge tables into one big table (and push the result to a pg database)", diff --git a/src/backend/platforms/all-platforms.ts b/src/backend/platforms/all-platforms.ts deleted file mode 100644 index 92258b3..0000000 --- a/src/backend/platforms/all-platforms.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { platformFetchers } from "./all/platformFetchers"; -export { platformNames } from "./all/platformNames"; diff --git a/src/backend/platforms/all/platformFetchers.ts b/src/backend/platforms/all/platformFetchers.ts deleted file mode 100644 index 04c1ce5..0000000 --- a/src/backend/platforms/all/platformFetchers.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { betfair } from "../betfair-fetch"; -import { fantasyscotus } from "../fantasyscotus-fetch"; -import { foretold } from "../foretold-fetch"; -import { goodjudgment } from "../goodjudgment-fetch"; -import { goodjudgmentopen } from "../goodjudmentopen-fetch"; -import { infer } from "../infer-fetch"; -import { kalshi } from "../kalshi-fetch"; -import { manifoldmarkets } from "../manifoldmarkets-fetch"; -import { metaculus } from "../metaculus-fetch"; -import { polymarket } from "../polymarket-fetch"; -import { predictit } from "../predictit-fetch"; -import { rootclaim } from "../rootclaim-fetch"; -import { smarkets } from "../smarkets-fetch"; -import { wildeford } from "../wildeford-fetch"; - -/* Deprecated -import { astralcodexten } from "../platforms/astralcodexten-fetch" -import { coupcast } from "../platforms/coupcast-fetch" -import { csetforetell } from "../platforms/csetforetell-fetch" -import { elicit } from "../platforms/elicit-fetch" -import { estimize } from "../platforms/estimize-fetch" -import { hypermind } from "../platforms/hypermind-fetch" -import { ladbrokes } from "../platforms/ladbrokes-fetch"; -import { williamhill } from "../platforms/williamhill-fetch"; -*/ - -export const platformFetchers = [ - betfair, - fantasyscotus, - foretold, - goodjudgment, - goodjudgmentopen, - infer, - kalshi, - manifoldmarkets, - metaculus, - polymarket, - predictit, - rootclaim, - smarkets, - wildeford, -]; diff --git a/src/backend/platforms/all/platformNames.ts b/src/backend/platforms/all/platformNames.ts deleted file mode 100644 index edbe248..0000000 --- a/src/backend/platforms/all/platformNames.ts +++ /dev/null @@ -1,20 +0,0 @@ -// This needs to be its own file to avoid cyclical dependencies. -export const platformNames = [ - "betfair", - "fantasyscotus", - "foretold", - "givewellopenphil", - "goodjudgment", - "goodjudmentopen", - "infer", - "kalshi", - "manifoldmarkets", - "metaculus", - "polymarket", - "predictit", - "rootclaim", - "smarkets", - "wildeford", - "xrisk", -]; -// deprecated: "astralcodexten", "csetforetell", "coupcast", "elicit", "estimize", "hypermind", "ladbrokes", "omen", "williamhill", etc diff --git a/src/backend/platforms/betfair-fetch.ts b/src/backend/platforms/betfair-fetch.ts index ad8a130..1e1caf4 100644 --- a/src/backend/platforms/betfair-fetch.ts +++ b/src/backend/platforms/betfair-fetch.ts @@ -1,8 +1,9 @@ /* Imports */ import axios from "axios"; import https from "https"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; +import { Forecast, PlatformFetcher } from "./"; /* Definitions */ let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; @@ -77,7 +78,7 @@ async function whipIntoShape(data) { async function processPredictions(data) { let predictions = await whipIntoShape(data); // console.log(JSON.stringify(predictions, null, 4)) - let results = predictions.map((prediction) => { + let results: Forecast[] = predictions.map((prediction) => { /* if(Math.floor(Math.random() * 10) % 20 ==0){ console.log(JSON.stringify(prediction, null, 4)) } */ @@ -136,12 +137,9 @@ async function processPredictions(data) { /* Body */ -export async function betfair() { - let data = await fetchPredictions(); - let results = await processPredictions(data); // somehow needed - // console.log(results.map(result => ({title: result.title, description: result.description}))) - // let string = JSON.stringify(results, null, 2) - await databaseUpsert({ contents: results, group: "betfair" }); - console.log("Done"); -} +export const betfair: PlatformFetcher = async function () { + const data = await fetchPredictions(); + const results = await processPredictions(data); // somehow needed + return results; +}; // betfair() diff --git a/src/backend/platforms/fantasyscotus-fetch.ts b/src/backend/platforms/fantasyscotus-fetch.ts index 9ed3ff4..1d99eb3 100644 --- a/src/backend/platforms/fantasyscotus-fetch.ts +++ b/src/backend/platforms/fantasyscotus-fetch.ts @@ -1,8 +1,8 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; import { calculateStars } from "../utils/stars"; +import { PlatformFetcher } from "./"; /* Definitions */ let unixtime = new Date().getTime(); @@ -111,10 +111,9 @@ async function processData(data) { } /* Body */ -export async function fantasyscotus() { +export const fantasyscotus: PlatformFetcher = async function () { let rawData = await fetchData(); let results = await processData(rawData); - await databaseUpsert({ contents: results, group: "fantasyscotus" }); - console.log("Done"); -} + return results; +}; //fantasyscotus() diff --git a/src/backend/platforms/foretold-fetch.ts b/src/backend/platforms/foretold-fetch.ts index c970952..59fce7a 100644 --- a/src/backend/platforms/foretold-fetch.ts +++ b/src/backend/platforms/foretold-fetch.ts @@ -1,6 +1,6 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; /* Definitions */ @@ -98,8 +98,6 @@ export async function foretold() { results.push(result); }); } - await databaseUpsert({ contents: results, group: "foretold" }); - - console.log("Done"); + return results; } // foretold() diff --git a/src/backend/platforms/goodjudgment-fetch.ts b/src/backend/platforms/goodjudgment-fetch.ts index ace57cc..34a7e70 100644 --- a/src/backend/platforms/goodjudgment-fetch.ts +++ b/src/backend/platforms/goodjudgment-fetch.ts @@ -3,9 +3,9 @@ import axios from "axios"; import { Tabletojson } from "tabletojson"; import tunnel from "tunnel"; -import { databaseUpsert } from "../database/database-wrapper"; import { hash } from "../utils/hash"; import { calculateStars } from "../utils/stars"; +import { PlatformFetcher } from "./"; /* Definitions */ let endpoint = "https://goodjudgment.io/superforecasts/"; @@ -17,7 +17,7 @@ String.prototype.replaceAll = function replaceAll(search, replace) { /* Support functions */ /* Body */ -export async function goodjudgment() { +export const goodjudgment: PlatformFetcher = async function () { // Proxy fuckery let proxy; /* @@ -116,14 +116,11 @@ export async function goodjudgment() { results.push(standardObj); } } - // console.log(results.slice(0,10)) - let string = JSON.stringify(results, null, 2); - console.log(results); - await databaseUpsert({ contents: results, group: "goodjudgment" }); console.log( "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js" ); - console.log("Done"); -} + + return results; +}; // goodjudgment() diff --git a/src/backend/platforms/goodjudmentopen-fetch.ts b/src/backend/platforms/goodjudmentopen-fetch.ts index f09032f..6f28c39 100644 --- a/src/backend/platforms/goodjudmentopen-fetch.ts +++ b/src/backend/platforms/goodjudmentopen-fetch.ts @@ -2,10 +2,10 @@ import axios from "axios"; import { Tabletojson } from "tabletojson"; -import { databaseUpsert } from "../database/database-wrapper"; import { applyIfSecretExists } from "../utils/getSecrets"; import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; +import { PlatformFetcher } from "./"; /* Definitions */ let htmlEndPoint = "https://www.gjopen.com/questions?page="; @@ -150,7 +150,7 @@ function isEnd(html) { return isEndBool; } -function sleep(ms) { +function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -221,11 +221,10 @@ async function goodjudgmentopen_inner(cookie) { ); } } - console.log(results); - if (results.length > 0) { - await databaseUpsert({ contents: results, group: "goodjudmentopen" }); - } else { + + if (results.length === 0) { console.log("Not updating results, as process was not signed in"); + return; } let end = Date.now(); @@ -233,9 +232,11 @@ async function goodjudgmentopen_inner(cookie) { console.log( `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` ); + + return results; } -export async function goodjudgmentopen() { +export const goodjudgmentopen: PlatformFetcher = async function () { let cookie = process.env.GOODJUDGMENTOPENCOOKIE; - await applyIfSecretExists(cookie, goodjudgmentopen_inner); -} + return await applyIfSecretExists(cookie, goodjudgmentopen_inner); +}; diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts new file mode 100644 index 0000000..b14f333 --- /dev/null +++ b/src/backend/platforms/index.ts @@ -0,0 +1,61 @@ +import { databaseUpsert } from "../database/database-wrapper"; +import { betfair } from "./betfair-fetch"; +import { fantasyscotus } from "./fantasyscotus-fetch"; +import { foretold } from "./foretold-fetch"; +import { goodjudgment } from "./goodjudgment-fetch"; +import { goodjudgmentopen } from "./goodjudmentopen-fetch"; +import { infer } from "./infer-fetch"; +import { kalshi } from "./kalshi-fetch"; +import { manifoldmarkets } from "./manifoldmarkets-fetch"; +import { metaculus } from "./metaculus-fetch"; +import { polymarket } from "./polymarket-fetch"; +import { predictit } from "./predictit-fetch"; +import { rootclaim } from "./rootclaim-fetch"; +import { smarkets } from "./smarkets-fetch"; +import { wildeford } from "./wildeford-fetch"; + +export interface Forecast { + id: string; + title: string; + url: string; + description: string; + platform: string; + options: any[]; + timestamp: string; + qualityindicators: any; + extra?: any; +} + +export type PlatformFetcher = () => Promise; + +interface Platform { + name: string; + fetcher: PlatformFetcher; +} + +export const platforms: Platform[] = [ + betfair, + fantasyscotus, + foretold, + goodjudgment, + goodjudgmentopen, + infer, + kalshi, + manifoldmarkets, + metaculus, + polymarket, + predictit, + rootclaim, + smarkets, + wildeford, +].map((fun) => ({ name: fun.name, fetcher: fun })); + +export const processPlatform = async (platform: Platform) => { + let results = await platform.fetcher(); + if (results && results.length) { + await databaseUpsert({ contents: results, group: platform.name }); + console.log("Done"); + } else { + console.log(`Platform ${platform.name} didn't return any results`); + } +}; diff --git a/src/backend/platforms/infer-fetch.ts b/src/backend/platforms/infer-fetch.ts index 952ad76..b1995a5 100644 --- a/src/backend/platforms/infer-fetch.ts +++ b/src/backend/platforms/infer-fetch.ts @@ -2,10 +2,10 @@ import axios from "axios"; import { Tabletojson } from "tabletojson"; -import { databaseUpsert } from "../database/database-wrapper"; import { applyIfSecretExists } from "../utils/getSecrets"; import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; +import { Forecast, PlatformFetcher } from "./"; /* Definitions */ let htmlEndPoint = "https://www.infer-pub.com/questions"; @@ -182,7 +182,7 @@ function sleep(ms) { async function infer_inner(cookie) { let i = 1; let response = await fetchPage(i, cookie); - let results = []; + let results: Forecast[] = []; let init = Date.now(); // console.log("Downloading... This might take a couple of minutes. Results will be shown.") while (!isEnd(response) && isSignedIn(response)) { @@ -263,20 +263,21 @@ async function infer_inner(cookie) { ); } } - if (results.length > 0) { - await databaseUpsert({ contents: results, group: "infer" }); - } else { - console.log("Not updating results, as process was not signed in"); - } let end = Date.now(); let difference = end - init; console.log( `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` ); + + if (results.length === 0) { + console.log("Not updating results, as process was not signed in"); + return; + } + return results; } -export async function infer() { +export const infer: PlatformFetcher = async function () { let cookie = process.env.INFER_COOKIE; - await applyIfSecretExists(cookie, infer_inner); -} + return await applyIfSecretExists(cookie, infer_inner); +}; diff --git a/src/backend/platforms/kalshi-fetch.ts b/src/backend/platforms/kalshi-fetch.ts index d1706c6..66fab9d 100644 --- a/src/backend/platforms/kalshi-fetch.ts +++ b/src/backend/platforms/kalshi-fetch.ts @@ -1,7 +1,8 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; +import { PlatformFetcher } from "./"; /* Definitions */ let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' @@ -69,11 +70,8 @@ async function processMarkets(markets) { } /* Body */ -export async function kalshi() { +export const kalshi: PlatformFetcher = async function () { let markets = await fetchAllMarkets(); - let results = await processMarkets(markets); // somehow needed - await databaseUpsert({ contents: results, group: "kalshi" }); - - console.log("Done"); -} + return await processMarkets(markets); +}; // kalshi() diff --git a/src/backend/platforms/manifoldmarkets-fetch.ts b/src/backend/platforms/manifoldmarkets-fetch.ts index 1a078db..5251e57 100644 --- a/src/backend/platforms/manifoldmarkets-fetch.ts +++ b/src/backend/platforms/manifoldmarkets-fetch.ts @@ -1,6 +1,6 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; /* Definitions */ @@ -89,12 +89,10 @@ async function processPredictions(predictions) { /* Body */ -export async function manifoldmarkets() { +export const manifoldmarkets = async function () { let data = await fetchData(); let results = await processPredictions(data); // somehow needed showStatistics(results); - await databaseUpsert({ contents: results, group: "manifoldmarkets" }); - - console.log("Done"); -} + return results; +}; // manifoldmarkets() diff --git a/src/backend/platforms/metaculus-fetch.ts b/src/backend/platforms/metaculus-fetch.ts index 87f6d91..5012816 100644 --- a/src/backend/platforms/metaculus-fetch.ts +++ b/src/backend/platforms/metaculus-fetch.ts @@ -1,8 +1,9 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; +import { PlatformFetcher } from "./"; /* Definitions */ let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; @@ -94,7 +95,7 @@ async function fetchMetaculusQuestionDescription(slug) { /* Body */ -export async function metaculus() { +export const metaculus: PlatformFetcher = async function () { // let metaculusQuestionsInit = await fetchMetaculusQuestions(1) // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) @@ -189,8 +190,6 @@ export async function metaculus() { i = i + 1; } - await databaseUpsert({ contents: all_questions, group: "metaculus" }); - - console.log("Done"); -} + return all_questions; +}; //metaculus() diff --git a/src/backend/platforms/polymarket-fetch.ts b/src/backend/platforms/polymarket-fetch.ts index 4a0ad91..a0e14e7 100644 --- a/src/backend/platforms/polymarket-fetch.ts +++ b/src/backend/platforms/polymarket-fetch.ts @@ -1,11 +1,12 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; +import { Forecast, PlatformFetcher } from "./"; /* Definitions */ let graphQLendpoint = - "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' + "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; let units = 10 ** 6; async function fetchAllContractInfo() { @@ -55,19 +56,24 @@ async function fetchAllContractData() { }) .then((res) => res.data) .then((res) => res.data.fixedProductMarketMakers); - // console.log(response) + return response; } -async function fetch_all() { +export const polymarket: PlatformFetcher = async function () { let allData = await fetchAllContractData(); let allInfo = await fetchAllContractInfo(); + let used = process.memoryUsage().heapUsed / 1024 / 1024; + console.log( + `The script uses approximately ${Math.round(used * 100) / 100} MB` + ); + let infos = {}; for (let info of allInfo) { let address = info.marketMakerAddress; let addressLowerCase = address.toLowerCase(); - //delete info.history + if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") infos[addressLowerCase] = { title: info.question, @@ -83,9 +89,8 @@ async function fetch_all() { let results = []; for (let data of allData) { let addressLowerCase = data.id; - // console.log(data) + if (infos[addressLowerCase] != undefined) { - // console.log(addressLowerCase) let id = `polymarket-${addressLowerCase.slice(0, 10)}`; let info = infos[addressLowerCase]; let numforecasts = Number(data.tradesQuantity); @@ -105,7 +110,7 @@ async function fetch_all() { }); } - let result = { + let result: Forecast = { id: id, title: info.title, url: info.url, @@ -126,23 +131,12 @@ async function fetch_all() { extra: { address: info.address, }, - /* - */ }; if (info.category != "Sports") { - // console.log(result) results.push(result); } } } - return results; //resultsProcessed -} -/* Body */ -export async function polymarket() { - let results = await fetch_all(); - await databaseUpsert({ contents: results, group: "polymarket" }); - - console.log("Done"); -} -// polymarket() + return results; +}; diff --git a/src/backend/platforms/predictit-fetch.ts b/src/backend/platforms/predictit-fetch.ts index 852f361..3dbab8a 100644 --- a/src/backend/platforms/predictit-fetch.ts +++ b/src/backend/platforms/predictit-fetch.ts @@ -1,8 +1,9 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; +import { PlatformFetcher } from "./"; /* Support functions */ async function fetchmarkets() { @@ -37,7 +38,7 @@ function sleep(ms) { } /* Body */ -export async function predictit() { +export const predictit: PlatformFetcher = async function () { let markets = await fetchmarkets(); let marketVolumes = await fetchmarketvolumes(); @@ -106,7 +107,6 @@ export async function predictit() { // console.log(obj) results.push(obj); } - await databaseUpsert({ contents: results, group: "predictit" }); - console.log("Done"); -} + return results; +}; diff --git a/src/backend/platforms/rootclaim-fetch.ts b/src/backend/platforms/rootclaim-fetch.ts index 2a39b4d..8b4daa8 100644 --- a/src/backend/platforms/rootclaim-fetch.ts +++ b/src/backend/platforms/rootclaim-fetch.ts @@ -1,8 +1,9 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; + import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; +import { PlatformFetcher } from "./"; /* Definitions */ let jsonEndpoint = @@ -22,7 +23,7 @@ async function fetchAllRootclaims() { return response; } -async function fetchAndProcessData() { +export const rootclaim: PlatformFetcher = async function () { let claims = await fetchAllRootclaims(); let results = []; for (let claim of claims) { @@ -53,13 +54,4 @@ async function fetchAndProcessData() { results.push(obj); } return results; -} - -/* Body */ -export async function rootclaim() { - let results = await fetchAndProcessData(); - await databaseUpsert({ contents: results, group: "rootclaim" }); - - console.log("Done"); -} -//rootclaim() +}; diff --git a/src/backend/platforms/smarkets-fetch.ts b/src/backend/platforms/smarkets-fetch.ts index fdc234b..fbf45ed 100644 --- a/src/backend/platforms/smarkets-fetch.ts +++ b/src/backend/platforms/smarkets-fetch.ts @@ -1,8 +1,8 @@ /* Imports */ import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; import { calculateStars } from "../utils/stars"; +import { PlatformFetcher } from "./"; /* Definitions */ let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; @@ -61,7 +61,7 @@ async function fetchPrices(marketid) { /* Body */ -export async function smarkets() { +export const smarkets: PlatformFetcher = async function () { let htmlPath = "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; @@ -172,9 +172,6 @@ export async function smarkets() { results.push(result); } VERBOSE ? console.log(results) : empty(); - - await databaseUpsert({ contents: results, group: "smarkets" }); - VERBOSE ? console.log(JSON.stringify(results, null, 4)) : empty(); - VERBOSE ? console.dir(results, { depth: null }) : empty(); -} + return results; +}; //smarkets() diff --git a/src/backend/platforms/wildeford-fetch.ts b/src/backend/platforms/wildeford-fetch.ts index f3abace..e68836f 100644 --- a/src/backend/platforms/wildeford-fetch.ts +++ b/src/backend/platforms/wildeford-fetch.ts @@ -2,10 +2,10 @@ // import axios from "axios" import { GoogleSpreadsheet } from "google-spreadsheet"; -import { databaseUpsert } from "../database/database-wrapper"; import { applyIfSecretExists } from "../utils/getSecrets"; import { hash } from "../utils/hash"; import { calculateStars } from "../utils/stars"; +import { PlatformFetcher } from "./"; /* Definitions */ const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL @@ -120,15 +120,11 @@ async function processPredictions(predictions) { /* Body */ export async function wildeford_inner(google_api_key) { let predictions = await fetchGoogleDoc(google_api_key); - let results = await processPredictions(predictions); // somehow needed - // console.log(results.sort((a,b) => (a.title > b.title))) - await databaseUpsert({ contents: results, group: "wildeford" }); - - console.log("Done"); + return await processPredictions(predictions); } //example() -export async function wildeford() { +export const wildeford: PlatformFetcher = async function () { const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey - await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); -} + return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); +}; diff --git a/src/backend/utils/getSecrets.ts b/src/backend/utils/getSecrets.ts index 5ce5e1b..41e9284 100644 --- a/src/backend/utils/getSecrets.ts +++ b/src/backend/utils/getSecrets.ts @@ -1,6 +1,9 @@ -export async function applyIfSecretExists(cookie, fun) { +export async function applyIfSecretExists( + cookie, + fun: (...args: any[]) => T +) { if (cookie) { - await fun(cookie); + return await fun(cookie); } else if (!cookie) { console.log( `Cannot proceed with ${fun.name} because cookie does not exist`