From 89fc5ec8b612135251015df4f37ba203583f1d59 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Tue, 29 Mar 2022 18:10:28 +0300 Subject: [PATCH] refactor: reorganize platforms code further --- .../manual/pullSuperforecastsManually.ts | 13 +- .../{betfair-fetch.ts => betfair.ts} | 16 +- ...antasyscotus-fetch.ts => fantasyscotus.ts} | 14 +- src/backend/platforms/foretold-fetch.ts | 103 --------- src/backend/platforms/foretold.ts | 104 ++++++++++ ...lopenphil-fetch.ts => givewellopenphil.ts} | 0 src/backend/platforms/goodjudgment-fetch.ts | 126 ----------- src/backend/platforms/goodjudgment.ts | 125 +++++++++++ ...udmentopen-fetch.ts => goodjudmentopen.ts} | 11 +- src/backend/platforms/index.ts | 35 ++-- .../platforms/{infer-fetch.ts => infer.ts} | 11 +- .../platforms/{kalshi-fetch.ts => kalshi.ts} | 13 +- ...oldmarkets-fetch.ts => manifoldmarkets.ts} | 17 +- src/backend/platforms/metaculus-fetch.ts | 195 ------------------ src/backend/platforms/metaculus.ts | 195 ++++++++++++++++++ src/backend/platforms/polymarket-fetch.ts | 142 ------------- src/backend/platforms/polymarket.ts | 146 +++++++++++++ src/backend/platforms/predictit-fetch.ts | 112 ---------- src/backend/platforms/predictit.ts | 115 +++++++++++ src/backend/platforms/rootclaim-fetch.ts | 57 ----- src/backend/platforms/rootclaim.ts | 62 ++++++ src/backend/platforms/smarkets-fetch.ts | 177 ---------------- src/backend/platforms/smarkets.ts | 177 ++++++++++++++++ .../{wildeford-fetch.ts => wildeford.ts} | 18 +- 24 files changed, 999 insertions(+), 985 deletions(-) rename src/backend/platforms/{betfair-fetch.ts => betfair.ts} (93%) rename src/backend/platforms/{fantasyscotus-fetch.ts => fantasyscotus.ts} (94%) delete mode 100644 src/backend/platforms/foretold-fetch.ts create mode 100644 src/backend/platforms/foretold.ts rename src/backend/platforms/{givewellopenphil-fetch.ts => givewellopenphil.ts} (100%) delete mode 100644 src/backend/platforms/goodjudgment-fetch.ts create mode 100644 src/backend/platforms/goodjudgment.ts rename src/backend/platforms/{goodjudmentopen-fetch.ts => goodjudmentopen.ts} (95%) rename src/backend/platforms/{infer-fetch.ts => infer.ts} (97%) rename src/backend/platforms/{kalshi-fetch.ts => kalshi.ts} (92%) rename src/backend/platforms/{manifoldmarkets-fetch.ts => manifoldmarkets.ts} (90%) delete mode 100644 src/backend/platforms/metaculus-fetch.ts create mode 100644 src/backend/platforms/metaculus.ts delete mode 100644 src/backend/platforms/polymarket-fetch.ts create mode 100644 src/backend/platforms/polymarket.ts delete mode 100644 src/backend/platforms/predictit-fetch.ts create mode 100644 src/backend/platforms/predictit.ts delete mode 100644 src/backend/platforms/rootclaim-fetch.ts create mode 100644 src/backend/platforms/rootclaim.ts delete mode 100644 src/backend/platforms/smarkets-fetch.ts create mode 100644 src/backend/platforms/smarkets.ts rename src/backend/platforms/{wildeford-fetch.ts => wildeford.ts} (88%) diff --git a/src/backend/manual/pullSuperforecastsManually.ts b/src/backend/manual/pullSuperforecastsManually.ts index 1a4829a..c1c79fb 100644 --- a/src/backend/manual/pullSuperforecastsManually.ts +++ b/src/backend/manual/pullSuperforecastsManually.ts @@ -1,11 +1,4 @@ -/* Imports */ -import { goodjudgment } from "../platforms/goodjudgment-fetch"; +import { processPlatform } from "../platforms"; +import { goodjudgment } from "../platforms/goodjudgment"; -/* Definitions */ - -/* Utilities */ - -/* Support functions */ - -/* Body */ -goodjudgment(); +processPlatform(goodjudgment); diff --git a/src/backend/platforms/betfair-fetch.ts b/src/backend/platforms/betfair.ts similarity index 93% rename from src/backend/platforms/betfair-fetch.ts rename to src/backend/platforms/betfair.ts index 1e1caf4..867df56 100644 --- a/src/backend/platforms/betfair-fetch.ts +++ b/src/backend/platforms/betfair.ts @@ -3,7 +3,7 @@ import axios from "axios"; import https from "https"; import { calculateStars } from "../utils/stars"; -import { Forecast, PlatformFetcher } from "./"; +import { Forecast, Platform } from "./"; /* Definitions */ let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; @@ -135,11 +135,11 @@ async function processPredictions(data) { return results; //resultsProcessed } -/* Body */ - -export const betfair: PlatformFetcher = async function () { - const data = await fetchPredictions(); - const results = await processPredictions(data); // somehow needed - return results; +export const betfair: Platform = { + name: "betfair", + async fetcher() { + const data = await fetchPredictions(); + const results = await processPredictions(data); // somehow needed + return results; + }, }; -// betfair() diff --git a/src/backend/platforms/fantasyscotus-fetch.ts b/src/backend/platforms/fantasyscotus.ts similarity index 94% rename from src/backend/platforms/fantasyscotus-fetch.ts rename to src/backend/platforms/fantasyscotus.ts index 1d99eb3..d32a32b 100644 --- a/src/backend/platforms/fantasyscotus-fetch.ts +++ b/src/backend/platforms/fantasyscotus.ts @@ -2,7 +2,7 @@ import axios from "axios"; import { calculateStars } from "../utils/stars"; -import { PlatformFetcher } from "./"; +import { Platform } from "./"; /* Definitions */ let unixtime = new Date().getTime(); @@ -111,9 +111,11 @@ async function processData(data) { } /* Body */ -export const fantasyscotus: PlatformFetcher = async function () { - let rawData = await fetchData(); - let results = await processData(rawData); - return results; +export const fantasyscotus: Platform = { + name: "fantasyscotus", + async fetcher() { + let rawData = await fetchData(); + let results = await processData(rawData); + return results; + }, }; -//fantasyscotus() diff --git a/src/backend/platforms/foretold-fetch.ts b/src/backend/platforms/foretold-fetch.ts deleted file mode 100644 index 59fce7a..0000000 --- a/src/backend/platforms/foretold-fetch.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* Imports */ -import axios from "axios"; - -import { calculateStars } from "../utils/stars"; - -/* Definitions */ -let graphQLendpoint = "https://api.foretold.io/graphql"; -let highQualityCommunities = [ - "0104d8e8-07e4-464b-8b32-74ef22b49f21", - "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", - "cf663021-f87f-4632-ad82-962d889a2d39", - "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", - "b2412a1d-0aa4-4e37-a12a-0aca9e440a96", -]; - -/* Support functions */ -async function fetchAllCommunityQuestions(communityId) { - let response = await axios({ - url: graphQLendpoint, - method: "POST", - headers: { "Content-Type": "application/json" }, - data: JSON.stringify({ - query: ` - query { - measurables( - channelId: "${communityId}", - states: OPEN, - first: 500 - ){ - total - edges{ - node{ - id - name - valueType - measurementCount - previousAggregate{ - value{ - percentage - } - } - } - } - } - } - `, - }), - }) - .then((res) => res.data) - .then((res) => res.data.measurables.edges); - //console.log(response) - return response; -} - -/* Body */ - -export async function foretold() { - let results = []; - for (let community of highQualityCommunities) { - let questions = await fetchAllCommunityQuestions(community); - questions = questions.map((question) => question.node); - questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions - questions.forEach((question) => { - let id = `foretold-${question.id}`; - let options = []; - if (question.valueType == "PERCENTAGE") { - let probability = question.previousAggregate.value.percentage; - options = [ - { - name: "Yes", - probability: probability / 100, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability / 100, - type: "PROBABILITY", - }, - ]; - } - let result = { - id: id, - title: question.name, - url: `https://www.foretold.io/c/${community}/m/${question.id}`, - platform: "Foretold", - description: "", - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: Math.floor(Number(question.measurementCount) / 2), - stars: calculateStars("Foretold", {}), - }, - /*liquidity: liquidity.toFixed(2), - tradevolume: tradevolume.toFixed(2), - address: obj.address*/ - }; - // console.log(result) - results.push(result); - }); - } - return results; -} -// foretold() diff --git a/src/backend/platforms/foretold.ts b/src/backend/platforms/foretold.ts new file mode 100644 index 0000000..b41f655 --- /dev/null +++ b/src/backend/platforms/foretold.ts @@ -0,0 +1,104 @@ +/* Imports */ +import axios from "axios"; + +import { calculateStars } from "../utils/stars"; +import { Platform } from "./"; + +/* Definitions */ +let graphQLendpoint = "https://api.foretold.io/graphql"; +let highQualityCommunities = [ + "0104d8e8-07e4-464b-8b32-74ef22b49f21", + "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", + "cf663021-f87f-4632-ad82-962d889a2d39", + "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", + "b2412a1d-0aa4-4e37-a12a-0aca9e440a96", +]; + +/* Support functions */ +async function fetchAllCommunityQuestions(communityId) { + let response = await axios({ + url: graphQLendpoint, + method: "POST", + headers: { "Content-Type": "application/json" }, + data: JSON.stringify({ + query: ` + query { + measurables( + channelId: "${communityId}", + states: OPEN, + first: 500 + ){ + total + edges{ + node{ + id + name + valueType + measurementCount + previousAggregate{ + value{ + percentage + } + } + } + } + } + } + `, + }), + }) + .then((res) => res.data) + .then((res) => res.data.measurables.edges); + //console.log(response) + return response; +} + +export const foretold: Platform = { + name: "foretold", + async fetcher() { + let results = []; + for (let community of highQualityCommunities) { + let questions = await fetchAllCommunityQuestions(community); + questions = questions.map((question) => question.node); + questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions + questions.forEach((question) => { + let id = `foretold-${question.id}`; + let options = []; + if (question.valueType == "PERCENTAGE") { + let probability = question.previousAggregate.value.percentage; + options = [ + { + name: "Yes", + probability: probability / 100, + type: "PROBABILITY", + }, + { + name: "No", + probability: 1 - probability / 100, + type: "PROBABILITY", + }, + ]; + } + let result = { + id: id, + title: question.name, + url: `https://www.foretold.io/c/${community}/m/${question.id}`, + platform: "Foretold", + description: "", + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Math.floor(Number(question.measurementCount) / 2), + stars: calculateStars("Foretold", {}), + }, + /*liquidity: liquidity.toFixed(2), + tradevolume: tradevolume.toFixed(2), + address: obj.address*/ + }; + // console.log(result) + results.push(result); + }); + } + return results; + }, +}; diff --git a/src/backend/platforms/givewellopenphil-fetch.ts b/src/backend/platforms/givewellopenphil.ts similarity index 100% rename from src/backend/platforms/givewellopenphil-fetch.ts rename to src/backend/platforms/givewellopenphil.ts diff --git a/src/backend/platforms/goodjudgment-fetch.ts b/src/backend/platforms/goodjudgment-fetch.ts deleted file mode 100644 index 34a7e70..0000000 --- a/src/backend/platforms/goodjudgment-fetch.ts +++ /dev/null @@ -1,126 +0,0 @@ -/* Imports */ -import axios from "axios"; -import { Tabletojson } from "tabletojson"; -import tunnel from "tunnel"; - -import { hash } from "../utils/hash"; -import { calculateStars } from "../utils/stars"; -import { PlatformFetcher } from "./"; - -/* Definitions */ -let endpoint = "https://goodjudgment.io/superforecasts/"; -String.prototype.replaceAll = function replaceAll(search, replace) { - return this.split(search).join(replace); -}; - -// Tunelling -/* Support functions */ - -/* Body */ -export const goodjudgment: PlatformFetcher = async function () { - // Proxy fuckery - let proxy; - /* - * try { - proxy = await axios - .get("http://pubproxy.com/api/proxy") - .then((query) => query.data); - console.log(proxy); - } catch (error) { - console.log("Proxy generation failed; using backup proxy instead"); - // hard-coded backup proxy - */ - proxy = { - ip: process.env.BACKUP_PROXY_IP, - port: process.env.BACKUP_PROXY_PORT, - }; - // } - let agent = tunnel.httpsOverHttp({ - proxy: { - host: proxy.ip, - port: proxy.port, - }, - }); - - let content = await axios - .request({ - url: "https://goodjudgment.io/superforecasts/", - method: "get", - headers: { - "User-Agent": "Chrome", - }, - // agent, - // port: 80, - }) - .then((query) => query.data); - - // Processing - let results = []; - let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); - jsonTable.shift(); // deletes first element - jsonTable.pop(); // deletes last element - // console.log(jsonTable) - for (let table of jsonTable) { - // console.log(table) - let title = table[0]["0"].split("\t\t\t").splice(3)[0]; - if (title != undefined) { - title = title.replaceAll("", ""); - let id = `goodjudgment-${hash(title)}`; - let description = table - .filter((row) => row["0"].includes("BACKGROUND:")) - .map((row) => row["0"]) - .map((text) => - text - .split("BACKGROUND:")[1] - .split("Examples of Superforecaster")[0] - .split("AT A GLANCE")[0] - .replaceAll("\n\n", "\n") - .split("\n") - .slice(3) - .join(" ") - .replaceAll(" ", "") - .replaceAll("
", "") - )[0]; - let options = table - .filter((row) => "4" in row) - .map((row) => ({ - name: row["2"] - .split('')[1] - .replace("", ""), - probability: Number(row["3"].split("%")[0]) / 100, - type: "PROBABILITY", - })); - let analysis = table.filter((row) => - row[0] ? row[0].toLowerCase().includes("commentary") : false - ); - // "Examples of Superforecaster Commentary" / Analysis - // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list. - analysis = analysis ? analysis[0] : ""; - analysis = analysis ? analysis[0] : ""; // not a duplicate - // console.log(analysis) - let standardObj = { - id: id, - title: title, - url: endpoint, - platform: "Good Judgment", - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Good Judgment", {}), - }, - extra: { - superforecastercommentary: analysis || "", - }, - }; - results.push(standardObj); - } - } - - console.log( - "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js" - ); - - return results; -}; -// goodjudgment() diff --git a/src/backend/platforms/goodjudgment.ts b/src/backend/platforms/goodjudgment.ts new file mode 100644 index 0000000..992bee2 --- /dev/null +++ b/src/backend/platforms/goodjudgment.ts @@ -0,0 +1,125 @@ +/* Imports */ +import axios from "axios"; +import { Tabletojson } from "tabletojson"; +import tunnel from "tunnel"; + +import { hash } from "../utils/hash"; +import { calculateStars } from "../utils/stars"; +import { Platform } from "./"; + +/* Definitions */ +let endpoint = "https://goodjudgment.io/superforecasts/"; +String.prototype.replaceAll = function replaceAll(search, replace) { + return this.split(search).join(replace); +}; + +/* Body */ +export const goodjudgment: Platform = { + name: "goodjudgment", + async fetcher() { + // Proxy fuckery + let proxy; + /* + * try { + proxy = await axios + .get("http://pubproxy.com/api/proxy") + .then((query) => query.data); + console.log(proxy); + } catch (error) { + console.log("Proxy generation failed; using backup proxy instead"); + // hard-coded backup proxy + */ + proxy = { + ip: process.env.BACKUP_PROXY_IP, + port: process.env.BACKUP_PROXY_PORT, + }; + // } + let agent = tunnel.httpsOverHttp({ + proxy: { + host: proxy.ip, + port: proxy.port, + }, + }); + + let content = await axios + .request({ + url: "https://goodjudgment.io/superforecasts/", + method: "get", + headers: { + "User-Agent": "Chrome", + }, + // agent, + // port: 80, + }) + .then((query) => query.data); + + // Processing + let results = []; + let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); + jsonTable.shift(); // deletes first element + jsonTable.pop(); // deletes last element + // console.log(jsonTable) + for (let table of jsonTable) { + // console.log(table) + let title = table[0]["0"].split("\t\t\t").splice(3)[0]; + if (title != undefined) { + title = title.replaceAll("", ""); + let id = `goodjudgment-${hash(title)}`; + let description = table + .filter((row) => row["0"].includes("BACKGROUND:")) + .map((row) => row["0"]) + .map((text) => + text + .split("BACKGROUND:")[1] + .split("Examples of Superforecaster")[0] + .split("AT A GLANCE")[0] + .replaceAll("\n\n", "\n") + .split("\n") + .slice(3) + .join(" ") + .replaceAll(" ", "") + .replaceAll("
", "") + )[0]; + let options = table + .filter((row) => "4" in row) + .map((row) => ({ + name: row["2"] + .split('')[1] + .replace("", ""), + probability: Number(row["3"].split("%")[0]) / 100, + type: "PROBABILITY", + })); + let analysis = table.filter((row) => + row[0] ? row[0].toLowerCase().includes("commentary") : false + ); + // "Examples of Superforecaster Commentary" / Analysis + // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list. + analysis = analysis ? analysis[0] : ""; + analysis = analysis ? analysis[0] : ""; // not a duplicate + // console.log(analysis) + let standardObj = { + id: id, + title: title, + url: endpoint, + platform: "Good Judgment", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Good Judgment", {}), + }, + extra: { + superforecastercommentary: analysis || "", + }, + }; + results.push(standardObj); + } + } + + console.log( + "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js" + ); + + return results; + }, +}; diff --git a/src/backend/platforms/goodjudmentopen-fetch.ts b/src/backend/platforms/goodjudmentopen.ts similarity index 95% rename from src/backend/platforms/goodjudmentopen-fetch.ts rename to src/backend/platforms/goodjudmentopen.ts index 88a2c3f..e2ce6b0 100644 --- a/src/backend/platforms/goodjudmentopen-fetch.ts +++ b/src/backend/platforms/goodjudmentopen.ts @@ -5,7 +5,7 @@ import { Tabletojson } from "tabletojson"; import { applyIfSecretExists } from "../utils/getSecrets"; import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; -import { PlatformFetcher } from "./"; +import { Platform } from "./"; /* Definitions */ let htmlEndPoint = "https://www.gjopen.com/questions?page="; @@ -236,7 +236,10 @@ async function goodjudgmentopen_inner(cookie) { return results; } -export const goodjudmentopen: PlatformFetcher = async function () { - let cookie = process.env.GOODJUDGMENTOPENCOOKIE; - return await applyIfSecretExists(cookie, goodjudgmentopen_inner); +export const goodjudmentopen: Platform = { + name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen` + async fetcher() { + let cookie = process.env.GOODJUDGMENTOPENCOOKIE; + return await applyIfSecretExists(cookie, goodjudgmentopen_inner); + }, }; diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index 29598f9..82fee5e 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -1,18 +1,18 @@ import { databaseUpsert } from "../database/database-wrapper"; -import { betfair } from "./betfair-fetch"; -import { fantasyscotus } from "./fantasyscotus-fetch"; -import { foretold } from "./foretold-fetch"; -import { goodjudgment } from "./goodjudgment-fetch"; -import { goodjudmentopen } from "./goodjudmentopen-fetch"; -import { infer } from "./infer-fetch"; -import { kalshi } from "./kalshi-fetch"; -import { manifoldmarkets } from "./manifoldmarkets-fetch"; -import { metaculus } from "./metaculus-fetch"; -import { polymarket } from "./polymarket-fetch"; -import { predictit } from "./predictit-fetch"; -import { rootclaim } from "./rootclaim-fetch"; -import { smarkets } from "./smarkets-fetch"; -import { wildeford } from "./wildeford-fetch"; +import { betfair } from "./betfair"; +import { fantasyscotus } from "./fantasyscotus"; +import { foretold } from "./foretold"; +import { goodjudgment } from "./goodjudgment"; +import { goodjudmentopen } from "./goodjudmentopen"; +import { infer } from "./infer"; +import { kalshi } from "./kalshi"; +import { manifoldmarkets } from "./manifoldmarkets"; +import { metaculus } from "./metaculus"; +import { polymarket } from "./polymarket"; +import { predictit } from "./predictit"; +import { rootclaim } from "./rootclaim"; +import { smarkets } from "./smarkets"; +import { wildeford } from "./wildeford"; export interface Forecast { id: string; @@ -26,9 +26,10 @@ export interface Forecast { extra?: any; } +// fetcher should return null if platform failed to fetch forecasts for some reason export type PlatformFetcher = () => Promise; -interface Platform { +export interface Platform { name: string; fetcher: PlatformFetcher; } @@ -53,7 +54,7 @@ export const platforms: Platform[] = [ fantasyscotus, foretold, goodjudgment, - goodjudmentopen, // note the typo! current table name is without `g`, `goodjudmentopen` + goodjudmentopen, infer, kalshi, manifoldmarkets, @@ -63,7 +64,7 @@ export const platforms: Platform[] = [ rootclaim, smarkets, wildeford, -].map((fun) => ({ name: fun.name, fetcher: fun })); +]; export const processPlatform = async (platform: Platform) => { let results = await platform.fetcher(); diff --git a/src/backend/platforms/infer-fetch.ts b/src/backend/platforms/infer.ts similarity index 97% rename from src/backend/platforms/infer-fetch.ts rename to src/backend/platforms/infer.ts index b1995a5..6a9bfba 100644 --- a/src/backend/platforms/infer-fetch.ts +++ b/src/backend/platforms/infer.ts @@ -5,7 +5,7 @@ import { Tabletojson } from "tabletojson"; import { applyIfSecretExists } from "../utils/getSecrets"; import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; -import { Forecast, PlatformFetcher } from "./"; +import { Forecast, Platform } from "./"; /* Definitions */ let htmlEndPoint = "https://www.infer-pub.com/questions"; @@ -277,7 +277,10 @@ async function infer_inner(cookie) { return results; } -export const infer: PlatformFetcher = async function () { - let cookie = process.env.INFER_COOKIE; - return await applyIfSecretExists(cookie, infer_inner); +export const infer: Platform = { + name: "infer", + async fetcher() { + let cookie = process.env.INFER_COOKIE; + return await applyIfSecretExists(cookie, infer_inner); + }, }; diff --git a/src/backend/platforms/kalshi-fetch.ts b/src/backend/platforms/kalshi.ts similarity index 92% rename from src/backend/platforms/kalshi-fetch.ts rename to src/backend/platforms/kalshi.ts index 66fab9d..212506a 100644 --- a/src/backend/platforms/kalshi-fetch.ts +++ b/src/backend/platforms/kalshi.ts @@ -2,7 +2,7 @@ import axios from "axios"; import { calculateStars } from "../utils/stars"; -import { PlatformFetcher } from "./"; +import { Platform } from "./"; /* Definitions */ let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' @@ -69,9 +69,10 @@ async function processMarkets(markets) { return results; //resultsProcessed } -/* Body */ -export const kalshi: PlatformFetcher = async function () { - let markets = await fetchAllMarkets(); - return await processMarkets(markets); +export const kalshi: Platform = { + name: "kalshi", + fetcher: async function () { + let markets = await fetchAllMarkets(); + return await processMarkets(markets); + }, }; -// kalshi() diff --git a/src/backend/platforms/manifoldmarkets-fetch.ts b/src/backend/platforms/manifoldmarkets.ts similarity index 90% rename from src/backend/platforms/manifoldmarkets-fetch.ts rename to src/backend/platforms/manifoldmarkets.ts index 5251e57..0969243 100644 --- a/src/backend/platforms/manifoldmarkets-fetch.ts +++ b/src/backend/platforms/manifoldmarkets.ts @@ -2,6 +2,7 @@ import axios from "axios"; import { calculateStars } from "../utils/stars"; +import { Platform } from "./"; /* Definitions */ let endpoint = "https://manifold.markets/api/v0/markets"; @@ -87,12 +88,12 @@ async function processPredictions(predictions) { return unresolvedResults; //resultsProcessed } -/* Body */ - -export const manifoldmarkets = async function () { - let data = await fetchData(); - let results = await processPredictions(data); // somehow needed - showStatistics(results); - return results; +export const manifoldmarkets: Platform = { + name: "manifoldmarkets", + async fetcher() { + let data = await fetchData(); + let results = await processPredictions(data); // somehow needed + showStatistics(results); + return results; + }, }; -// manifoldmarkets() diff --git a/src/backend/platforms/metaculus-fetch.ts b/src/backend/platforms/metaculus-fetch.ts deleted file mode 100644 index 5012816..0000000 --- a/src/backend/platforms/metaculus-fetch.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* Imports */ -import axios from "axios"; - -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; -import { PlatformFetcher } from "./"; - -/* Definitions */ -let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; -let now = new Date().toISOString(); -let DEBUG_MODE = "off"; -let SLEEP_TIME = 5000; -/* Support functions */ -async function fetchMetaculusQuestions(next) { - // Numbers about a given address: how many, how much, at what price, etc. - let response; - let data; - try { - response = await axios({ - url: next, - method: "GET", - headers: { "Content-Type": "application/json" }, - }); - data = response.data; - } catch (error) { - console.log(`Error in async function fetchMetaculusQuestions(next)`); - if (!!error.response.headers["retry-after"]) { - let timeout = error.response.headers["retry-after"]; - console.log(`Timeout: ${timeout}`); - await sleep(Number(timeout) * 1000 + SLEEP_TIME); - } else { - await sleep(SLEEP_TIME); - } - console.log(error); - } finally { - try { - response = await axios({ - url: next, - method: "GET", - headers: { "Content-Type": "application/json" }, - }); - data = response.data; - } catch (error) { - console.log(error); - return { results: [] }; - } - } - // console.log(response) - return data; -} - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function fetchMetaculusQuestionDescription(slug) { - try { - let response = await axios({ - method: "get", - url: "https://www.metaculus.com" + slug, - }).then((response) => response.data); - return response; - } catch (error) { - console.log(`Error in: fetchMetaculusQuestionDescription`); - console.log( - `We encountered some error when attempting to fetch a metaculus page. Trying again` - ); - if ( - typeof error.response != "undefined" && - typeof error.response.headers != "undefined" && - typeof error.response.headers["retry-after"] != "undefined" - ) { - let timeout = error.response.headers["retry-after"]; - console.log(`Timeout: ${timeout}`); - await sleep(Number(timeout) * 1000 + SLEEP_TIME); - } else { - await sleep(SLEEP_TIME); - } - try { - let response = await axios({ - method: "get", - url: "https://www.metaculus.com" + slug, - }).then((response) => response.data); - // console.log(response) - return response; - } catch (error) { - console.log( - `We encountered some error when attempting to fetch a metaculus page.` - ); - console.log("Error", error); - throw "Giving up"; - } - } -} - -/* Body */ - -export const metaculus: PlatformFetcher = async function () { - // let metaculusQuestionsInit = await fetchMetaculusQuestions(1) - // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) - // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) - // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test - let all_questions = []; - let next = "https://www.metaculus.com/api2/questions/"; - let i = 1; - while (next) { - if (i % 20 == 0) { - console.log("Sleeping for 500ms"); - await sleep(SLEEP_TIME); - } - console.log(`\nQuery #${i}`); - let metaculusQuestions = await fetchMetaculusQuestions(next); - let results = metaculusQuestions.results; - let j = false; - for (let result of results) { - if (result.publish_time < now && now < result.resolve_time) { - await sleep(SLEEP_TIME / 2); - let questionPage = await fetchMetaculusQuestionDescription( - result.page_url - ); - if (!questionPage.includes("A public prediction by")) { - // console.log(questionPage) - let descriptionraw = questionPage.split( - `
` - )[1]; //.split(`
`)[1] - let descriptionprocessed1 = descriptionraw.split("
")[0]; - let descriptionprocessed2 = toMarkdown(descriptionprocessed1); - let description = descriptionprocessed2; - - let isbinary = result.possibilities.type == "binary"; - let options = []; - if (isbinary) { - let probability = Number(result.community_prediction.full.q2); - options = [ - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability, - type: "PROBABILITY", - }, - ]; - } - let id = `metaculus-${result.id}`; - let interestingInfo = { - id: id, - title: result.title, - url: "https://www.metaculus.com" + result.page_url, - platform: "Metaculus", - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: Number(result.number_of_predictions), - stars: calculateStars("Metaculus", { - numforecasts: result.number_of_predictions, - }), - }, - extra: { - resolution_data: { - publish_time: result.publish_time, - resolution: result.resolution, - close_time: result.close_time, - resolve_time: result.resolve_time, - }, - }, - //"status": result.status, - //"publish_time": result.publish_time, - //"close_time": result.close_time, - //"type": result.possibilities.type, // We want binary ones here. - //"last_activity_time": result.last_activity_time, - }; - if (Number(result.number_of_predictions) >= 10) { - console.log(`- ${interestingInfo.title}`); - all_questions.push(interestingInfo); - if ((!j && i % 20 == 0) || DEBUG_MODE == "on") { - console.log(interestingInfo); - j = true; - } - } - } else { - console.log("- [Skipping public prediction]"); - } - } - } - next = metaculusQuestions.next; - i = i + 1; - } - - return all_questions; -}; -//metaculus() diff --git a/src/backend/platforms/metaculus.ts b/src/backend/platforms/metaculus.ts new file mode 100644 index 0000000..78369ee --- /dev/null +++ b/src/backend/platforms/metaculus.ts @@ -0,0 +1,195 @@ +/* Imports */ +import axios from "axios"; + +import { calculateStars } from "../utils/stars"; +import toMarkdown from "../utils/toMarkdown"; +import { Platform } from "./"; + +/* Definitions */ +let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; +let now = new Date().toISOString(); +let DEBUG_MODE = "off"; +let SLEEP_TIME = 5000; +/* Support functions */ +async function fetchMetaculusQuestions(next) { + // Numbers about a given address: how many, how much, at what price, etc. + let response; + let data; + try { + response = await axios({ + url: next, + method: "GET", + headers: { "Content-Type": "application/json" }, + }); + data = response.data; + } catch (error) { + console.log(`Error in async function fetchMetaculusQuestions(next)`); + if (!!error.response.headers["retry-after"]) { + let timeout = error.response.headers["retry-after"]; + console.log(`Timeout: ${timeout}`); + await sleep(Number(timeout) * 1000 + SLEEP_TIME); + } else { + await sleep(SLEEP_TIME); + } + console.log(error); + } finally { + try { + response = await axios({ + url: next, + method: "GET", + headers: { "Content-Type": "application/json" }, + }); + data = response.data; + } catch (error) { + console.log(error); + return { results: [] }; + } + } + // console.log(response) + return data; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function fetchMetaculusQuestionDescription(slug) { + try { + let response = await axios({ + method: "get", + url: "https://www.metaculus.com" + slug, + }).then((response) => response.data); + return response; + } catch (error) { + console.log(`Error in: fetchMetaculusQuestionDescription`); + console.log( + `We encountered some error when attempting to fetch a metaculus page. Trying again` + ); + if ( + typeof error.response != "undefined" && + typeof error.response.headers != "undefined" && + typeof error.response.headers["retry-after"] != "undefined" + ) { + let timeout = error.response.headers["retry-after"]; + console.log(`Timeout: ${timeout}`); + await sleep(Number(timeout) * 1000 + SLEEP_TIME); + } else { + await sleep(SLEEP_TIME); + } + try { + let response = await axios({ + method: "get", + url: "https://www.metaculus.com" + slug, + }).then((response) => response.data); + // console.log(response) + return response; + } catch (error) { + console.log( + `We encountered some error when attempting to fetch a metaculus page.` + ); + console.log("Error", error); + throw "Giving up"; + } + } +} + +export const metaculus: Platform = { + name: "metaculus", + async fetcher() { + // let metaculusQuestionsInit = await fetchMetaculusQuestions(1) + // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) + // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) + // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test } + let all_questions = []; + let next = "https://www.metaculus.com/api2/questions/"; + let i = 1; + while (next) { + if (i % 20 == 0) { + console.log("Sleeping for 500ms"); + await sleep(SLEEP_TIME); + } + console.log(`\nQuery #${i}`); + let metaculusQuestions = await fetchMetaculusQuestions(next); + let results = metaculusQuestions.results; + let j = false; + for (let result of results) { + if (result.publish_time < now && now < result.resolve_time) { + await sleep(SLEEP_TIME / 2); + let questionPage = await fetchMetaculusQuestionDescription( + result.page_url + ); + if (!questionPage.includes("A public prediction by")) { + // console.log(questionPage) + let descriptionraw = questionPage.split( + `
` + )[1]; //.split(`
`)[1] + let descriptionprocessed1 = descriptionraw.split("
")[0]; + let descriptionprocessed2 = toMarkdown(descriptionprocessed1); + let description = descriptionprocessed2; + + let isbinary = result.possibilities.type == "binary"; + let options = []; + if (isbinary) { + let probability = Number(result.community_prediction.full.q2); + options = [ + { + name: "Yes", + probability: probability, + type: "PROBABILITY", + }, + { + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; + } + let id = `metaculus-${result.id}`; + let interestingInfo = { + id: id, + title: result.title, + url: "https://www.metaculus.com" + result.page_url, + platform: "Metaculus", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Number(result.number_of_predictions), + stars: calculateStars("Metaculus", { + numforecasts: result.number_of_predictions, + }), + }, + extra: { + resolution_data: { + publish_time: result.publish_time, + resolution: result.resolution, + close_time: result.close_time, + resolve_time: result.resolve_time, + }, + }, + //"status": result.status, + //"publish_time": result.publish_time, + //"close_time": result.close_time, + //"type": result.possibilities.type, // We want binary ones here. + //"last_activity_time": result.last_activity_time, + }; + if (Number(result.number_of_predictions) >= 10) { + console.log(`- ${interestingInfo.title}`); + all_questions.push(interestingInfo); + if ((!j && i % 20 == 0) || DEBUG_MODE == "on") { + console.log(interestingInfo); + j = true; + } + } + } else { + console.log("- [Skipping public prediction]"); + } + } + } + next = metaculusQuestions.next; + i = i + 1; + } + + return all_questions; + }, +}; diff --git a/src/backend/platforms/polymarket-fetch.ts b/src/backend/platforms/polymarket-fetch.ts deleted file mode 100644 index a0e14e7..0000000 --- a/src/backend/platforms/polymarket-fetch.ts +++ /dev/null @@ -1,142 +0,0 @@ -/* Imports */ -import axios from "axios"; - -import { calculateStars } from "../utils/stars"; -import { Forecast, PlatformFetcher } from "./"; - -/* Definitions */ -let graphQLendpoint = - "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; -let units = 10 ** 6; - -async function fetchAllContractInfo() { - // for info which the polymarket graphql API - let response = await axios - .get( - "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1" - ) - .then((query) => query.data); - response = response.filter((res) => res.closed != true); - return response; -} - -async function fetchAllContractData() { - let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week - let response = await axios({ - url: graphQLendpoint, - method: "POST", - headers: { "Content-Type": "application/json" }, - data: JSON.stringify({ - query: ` - { - fixedProductMarketMakers(first: 1000 - where: { - lastActiveDay_gt: ${daysSinceEra} - }){ - id - creator - creationTimestamp - fee - tradesQuantity - buysQuantity - sellsQuantity - lastActiveDay - outcomeTokenPrices - outcomeTokenAmounts - liquidityParameter - collateralBuyVolume - collateralSellVolume - conditions { - outcomeSlotCount - } - } - } - `, - }), - }) - .then((res) => res.data) - .then((res) => res.data.fixedProductMarketMakers); - - return response; -} - -export const polymarket: PlatformFetcher = async function () { - let allData = await fetchAllContractData(); - let allInfo = await fetchAllContractInfo(); - - let used = process.memoryUsage().heapUsed / 1024 / 1024; - console.log( - `The script uses approximately ${Math.round(used * 100) / 100} MB` - ); - - let infos = {}; - for (let info of allInfo) { - let address = info.marketMakerAddress; - let addressLowerCase = address.toLowerCase(); - - if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") - infos[addressLowerCase] = { - title: info.question, - url: "https://polymarket.com/market/" + info.slug, - address: address, - description: info.description, - outcomes: info.outcomes, - options: [], - category: info.category, - }; - } - - let results = []; - for (let data of allData) { - let addressLowerCase = data.id; - - if (infos[addressLowerCase] != undefined) { - let id = `polymarket-${addressLowerCase.slice(0, 10)}`; - let info = infos[addressLowerCase]; - let numforecasts = Number(data.tradesQuantity); - let tradevolume = - (Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) / - units; - let liquidity = Number(data.liquidityParameter) / units; - // let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2 - // let percentage = Number(data.outcomeTokenPrices[0]) * 100 - // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none" - let options = []; - for (let outcome in data.outcomeTokenPrices) { - options.push({ - name: info.outcomes[outcome], - probability: data.outcomeTokenPrices[outcome], - type: "PROBABILITY", - }); - } - - let result: Forecast = { - id: id, - title: info.title, - url: info.url, - platform: "PolyMarket", - description: info.description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: numforecasts.toFixed(0), - liquidity: liquidity.toFixed(2), - tradevolume: tradevolume.toFixed(2), - stars: calculateStars("Polymarket", { - liquidity, - option: options[0], - volume: tradevolume, - }), - }, - extra: { - address: info.address, - }, - }; - if (info.category != "Sports") { - results.push(result); - } - } - } - - return results; -}; diff --git a/src/backend/platforms/polymarket.ts b/src/backend/platforms/polymarket.ts new file mode 100644 index 0000000..aa67bdf --- /dev/null +++ b/src/backend/platforms/polymarket.ts @@ -0,0 +1,146 @@ +/* Imports */ +import axios from "axios"; + +import { calculateStars } from "../utils/stars"; +import { Forecast, Platform } from "./"; + +/* Definitions */ +let graphQLendpoint = + "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; +let units = 10 ** 6; + +async function fetchAllContractInfo() { + // for info which the polymarket graphql API + let response = await axios + .get( + "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1" + ) + .then((query) => query.data); + response = response.filter((res) => res.closed != true); + return response; +} + +async function fetchAllContractData() { + let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week + let response = await axios({ + url: graphQLendpoint, + method: "POST", + headers: { "Content-Type": "application/json" }, + data: JSON.stringify({ + query: ` + { + fixedProductMarketMakers(first: 1000 + where: { + lastActiveDay_gt: ${daysSinceEra} + }){ + id + creator + creationTimestamp + fee + tradesQuantity + buysQuantity + sellsQuantity + lastActiveDay + outcomeTokenPrices + outcomeTokenAmounts + liquidityParameter + collateralBuyVolume + collateralSellVolume + conditions { + outcomeSlotCount + } + } + } + `, + }), + }) + .then((res) => res.data) + .then((res) => res.data.fixedProductMarketMakers); + + return response; +} + +export const polymarket: Platform = { + name: "polymarket", + async fetcher() { + let allData = await fetchAllContractData(); + let allInfo = await fetchAllContractInfo(); + + let used = process.memoryUsage().heapUsed / 1024 / 1024; + console.log( + `The script uses approximately ${Math.round(used * 100) / 100} MB` + ); + + let infos = {}; + for (let info of allInfo) { + let address = info.marketMakerAddress; + let addressLowerCase = address.toLowerCase(); + + if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") + infos[addressLowerCase] = { + title: info.question, + url: "https://polymarket.com/market/" + info.slug, + address: address, + description: info.description, + outcomes: info.outcomes, + options: [], + category: info.category, + }; + } + + let results = []; + for (let data of allData) { + let addressLowerCase = data.id; + + if (infos[addressLowerCase] != undefined) { + let id = `polymarket-${addressLowerCase.slice(0, 10)}`; + let info = infos[addressLowerCase]; + let numforecasts = Number(data.tradesQuantity); + let tradevolume = + (Number(data.collateralBuyVolume) + + Number(data.collateralSellVolume)) / + units; + let liquidity = Number(data.liquidityParameter) / units; + // let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2 + // let percentage = Number(data.outcomeTokenPrices[0]) * 100 + // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none" + let options = []; + for (let outcome in data.outcomeTokenPrices) { + options.push({ + name: info.outcomes[outcome], + probability: data.outcomeTokenPrices[outcome], + type: "PROBABILITY", + }); + } + + let result: Forecast = { + id: id, + title: info.title, + url: info.url, + platform: "PolyMarket", + description: info.description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: numforecasts.toFixed(0), + liquidity: liquidity.toFixed(2), + tradevolume: tradevolume.toFixed(2), + stars: calculateStars("Polymarket", { + liquidity, + option: options[0], + volume: tradevolume, + }), + }, + extra: { + address: info.address, + }, + }; + if (info.category != "Sports") { + results.push(result); + } + } + } + + return results; + }, +}; diff --git a/src/backend/platforms/predictit-fetch.ts b/src/backend/platforms/predictit-fetch.ts deleted file mode 100644 index 3dbab8a..0000000 --- a/src/backend/platforms/predictit-fetch.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* Imports */ -import axios from "axios"; - -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; -import { PlatformFetcher } from "./"; - -/* Support functions */ -async function fetchmarkets() { - let response = await axios({ - method: "get", - url: "https://www.predictit.org/api/marketdata/all/", - }); - let openMarkets = response.data.markets.filter( - (market) => market.status == "Open" - ); - return openMarkets; -} - -async function fetchmarketrules(market_id) { - let response = await axios({ - method: "get", - url: "https://www.predictit.org/api/Market/" + market_id, - }); - return response.data.rule; -} - -async function fetchmarketvolumes() { - let response = await axios({ - method: "get", - url: "https://predictit-f497e.firebaseio.com/marketStats.json", - }); - return response.data; -} - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/* Body */ -export const predictit: PlatformFetcher = async function () { - let markets = await fetchmarkets(); - let marketVolumes = await fetchmarketvolumes(); - - markets = markets.map((market) => ({ - ...market, - TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"], - })); - // console.log(markets) - - let results = []; - for (let market of markets) { - // console.log(market.name) - let id = `predictit-${market.id}`; - let isbinary = market.contracts.length == 1; - await sleep(3000 * (1 + Math.random())); - let descriptionraw = await fetchmarketrules(market.id); - let descriptionprocessed1 = toMarkdown(descriptionraw); - let description = descriptionprocessed1; - let shares_volume = market["TotalSharesTraded"]; - // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none" - - let options = market.contracts.map((contract) => ({ - name: contract.name, - probability: contract.lastTradePrice, - type: "PROBABILITY", - })); - let totalValue = options - .map((element) => Number(element.probability)) - .reduce((a, b) => a + b, 0); - - if (options.length != 1 && totalValue > 1) { - options = options.map((element) => ({ - ...element, - probability: Number(element.probability) / totalValue, - })); - } else if (options.length == 1) { - let option = options[0]; - let probability = option["probability"]; - options = [ - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability, - type: "PROBABILITY", - }, - ]; - } - - let obj = { - id: id, - title: market["name"], - url: market.url, - platform: "PredictIt", - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("PredictIt", {}), - shares_volume: shares_volume, - }, - }; - // console.log(obj) - results.push(obj); - } - - return results; -}; diff --git a/src/backend/platforms/predictit.ts b/src/backend/platforms/predictit.ts new file mode 100644 index 0000000..fb0f645 --- /dev/null +++ b/src/backend/platforms/predictit.ts @@ -0,0 +1,115 @@ +/* Imports */ +import axios from "axios"; + +import { calculateStars } from "../utils/stars"; +import toMarkdown from "../utils/toMarkdown"; +import { Platform } from "./"; + +/* Support functions */ +async function fetchmarkets() { + let response = await axios({ + method: "get", + url: "https://www.predictit.org/api/marketdata/all/", + }); + let openMarkets = response.data.markets.filter( + (market) => market.status == "Open" + ); + return openMarkets; +} + +async function fetchmarketrules(market_id) { + let response = await axios({ + method: "get", + url: "https://www.predictit.org/api/Market/" + market_id, + }); + return response.data.rule; +} + +async function fetchmarketvolumes() { + let response = await axios({ + method: "get", + url: "https://predictit-f497e.firebaseio.com/marketStats.json", + }); + return response.data; +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/* Body */ +export const predictit: Platform = { + name: "predictit", + async fetcher() { + let markets = await fetchmarkets(); + let marketVolumes = await fetchmarketvolumes(); + + markets = markets.map((market) => ({ + ...market, + TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"], + })); + // console.log(markets) + + let results = []; + for (let market of markets) { + // console.log(market.name) + let id = `predictit-${market.id}`; + let isbinary = market.contracts.length == 1; + await sleep(3000 * (1 + Math.random())); + let descriptionraw = await fetchmarketrules(market.id); + let descriptionprocessed1 = toMarkdown(descriptionraw); + let description = descriptionprocessed1; + let shares_volume = market["TotalSharesTraded"]; + // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none" + + let options = market.contracts.map((contract) => ({ + name: contract.name, + probability: contract.lastTradePrice, + type: "PROBABILITY", + })); + let totalValue = options + .map((element) => Number(element.probability)) + .reduce((a, b) => a + b, 0); + + if (options.length != 1 && totalValue > 1) { + options = options.map((element) => ({ + ...element, + probability: Number(element.probability) / totalValue, + })); + } else if (options.length == 1) { + let option = options[0]; + let probability = option["probability"]; + options = [ + { + name: "Yes", + probability: probability, + type: "PROBABILITY", + }, + { + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; + } + + let obj = { + id: id, + title: market["name"], + url: market.url, + platform: "PredictIt", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("PredictIt", {}), + shares_volume: shares_volume, + }, + }; + // console.log(obj) + results.push(obj); + } + + return results; + }, +}; diff --git a/src/backend/platforms/rootclaim-fetch.ts b/src/backend/platforms/rootclaim-fetch.ts deleted file mode 100644 index 8b4daa8..0000000 --- a/src/backend/platforms/rootclaim-fetch.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* Imports */ -import axios from "axios"; - -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; -import { PlatformFetcher } from "./"; - -/* Definitions */ -let jsonEndpoint = - "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' - -async function fetchAllRootclaims() { - // for info which the polymarket graphql API - let response = await axios - .get(jsonEndpoint) - .then((response) => response.data); - if (response.length != response[0] + 1) { - console.log(response.length); - console.log(response[0]); - //throw Error("Rootclaim's backend has changed.") - } - response.shift(); - return response; -} - -export const rootclaim: PlatformFetcher = async function () { - let claims = await fetchAllRootclaims(); - let results = []; - for (let claim of claims) { - let id = `rootclaim-${claim.slug.toLowerCase()}`; - let options = []; - for (let scenario of claim.scenarios) { - //console.log(scenario) - options.push({ - name: toMarkdown(scenario.text).replace("\n", "").replace("'", "'"), - probability: scenario.net_prob / 100, - type: "PROBABILITY", - }); - } - let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"; - let obj = { - id: id, - title: toMarkdown(claim.question).replace("\n", ""), - url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, - platform: "Rootclaim", - description: toMarkdown(claim.background).replace("'", "'"), - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: 1, - stars: calculateStars("Rootclaim", {}), - }, - }; - results.push(obj); - } - return results; -}; diff --git a/src/backend/platforms/rootclaim.ts b/src/backend/platforms/rootclaim.ts new file mode 100644 index 0000000..6520458 --- /dev/null +++ b/src/backend/platforms/rootclaim.ts @@ -0,0 +1,62 @@ +/* Imports */ +import axios from "axios"; + +import { calculateStars } from "../utils/stars"; +import toMarkdown from "../utils/toMarkdown"; +import { Platform } from "./"; + +/* Definitions */ +let jsonEndpoint = + "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' + +async function fetchAllRootclaims() { + // for info which the polymarket graphql API + let response = await axios + .get(jsonEndpoint) + .then((response) => response.data); + if (response.length != response[0] + 1) { + console.log(response.length); + console.log(response[0]); + //throw Error("Rootclaim's backend has changed.") + } + response.shift(); + return response; +} + +export const rootclaim: Platform = { + name: "rootclaim", + async fetcher() { + let claims = await fetchAllRootclaims(); + let results = []; + for (let claim of claims) { + let id = `rootclaim-${claim.slug.toLowerCase()}`; + let options = []; + for (let scenario of claim.scenarios) { + //console.log(scenario) + options.push({ + name: toMarkdown(scenario.text) + .replace("\n", "") + .replace("'", "'"), + probability: scenario.net_prob / 100, + type: "PROBABILITY", + }); + } + let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"; + let obj = { + id: id, + title: toMarkdown(claim.question).replace("\n", ""), + url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, + platform: "Rootclaim", + description: toMarkdown(claim.background).replace("'", "'"), + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: 1, + stars: calculateStars("Rootclaim", {}), + }, + }; + results.push(obj); + } + return results; + }, +}; diff --git a/src/backend/platforms/smarkets-fetch.ts b/src/backend/platforms/smarkets-fetch.ts deleted file mode 100644 index fbf45ed..0000000 --- a/src/backend/platforms/smarkets-fetch.ts +++ /dev/null @@ -1,177 +0,0 @@ -/* Imports */ -import axios from "axios"; - -import { calculateStars } from "../utils/stars"; -import { PlatformFetcher } from "./"; - -/* Definitions */ -let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; -let VERBOSE = false; -let empty = () => 0; -/* Support functions */ - -async function fetchEvents(url) { - let response = await axios({ - url: htmlEndPointEntrance + url, - method: "GET", - headers: { - "Content-Type": "text/html", - }, - }).then((res) => res.data); - VERBOSE ? console.log(response) : empty(); - return response; -} - -async function fetchMarkets(eventid) { - let response = await axios({ - url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, - method: "GET", - headers: { - "Content-Type": "text/json", - }, - }) - .then((res) => res.data) - .then((res) => res.markets); - return response; -} - -async function fetchContracts(marketid) { - let response = await axios({ - url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, - method: "GET", - headers: { - "Content-Type": "text/html", - }, - }).then((res) => res.data); - VERBOSE ? console.log(response) : empty(); - return response; -} - -async function fetchPrices(marketid) { - let response = await axios({ - url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, - method: "GET", - headers: { - "Content-Type": "text/html", - }, - }).then((res) => res.data); - VERBOSE ? console.log(response) : empty(); - return response; -} - -/* Body */ - -export const smarkets: PlatformFetcher = async function () { - let htmlPath = - "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; - - let events = []; - while (htmlPath) { - let data = await fetchEvents(htmlPath); - events.push(...data.events); - htmlPath = data.pagination.next_page; - } - VERBOSE ? console.log(events) : empty(); - let markets = []; - for (let event of events) { - VERBOSE ? console.log(Date.now()) : empty(); - VERBOSE ? console.log(event.name) : empty(); - let eventMarkets = await fetchMarkets(event.id); - eventMarkets = eventMarkets.map((market) => ({ - ...market, - slug: event.full_slug, - })); - VERBOSE ? console.log("Markets fetched") : empty(); - VERBOSE ? console.log(event.id) : empty(); - VERBOSE ? console.log(eventMarkets) : empty(); - markets.push(...eventMarkets); - //let lastPrices = await fetchPrices(market.id) - } - VERBOSE ? console.log(markets) : empty(); - - let results = []; - for (let market of markets) { - VERBOSE ? console.log("================") : empty(); - VERBOSE ? console.log("Market: ", market) : empty(); - let id = `smarkets-${market.id}`; - let name = market.name; - - let contracts = await fetchContracts(market.id); - VERBOSE ? console.log("Contracts: ", contracts) : empty(); - let prices = await fetchPrices(market.id); - VERBOSE - ? console.log("Prices: ", prices["last_executed_prices"][market.id]) - : empty(); - - let optionsObj = {}; - for (let contract of contracts["contracts"]) { - optionsObj[contract.id] = { name: contract.name }; - } - for (let price of prices["last_executed_prices"][market.id]) { - optionsObj[price.contract_id] = { - ...optionsObj[price.contract_id], - probability: price.last_executed_price - ? Number(price.last_executed_price) - : null, - type: "PROBABILITY", - }; - } - let options: any[] = Object.values(optionsObj); - // monkey patch the case where there are only two options and only one has traded. - if ( - options.length == 2 && - options.map((option) => option.probability).includes(null) - ) { - let nonNullPrice = - options[0].probability == null - ? options[1].probability - : options[0].probability; - options = options.map((option) => { - let probability = option.probability; - return { - ...option, - probability: probability == null ? 100 - nonNullPrice : probability, - // yes, 100, because prices are not yet normalized. - }; - }); - } - - // Normalize normally - let totalValue = options - .map((element) => Number(element.probability)) - .reduce((a, b) => a + b, 0); - - options = options.map((element) => ({ - ...element, - probability: Number(element.probability) / totalValue, - })); - VERBOSE ? console.log(options) : empty(); - - /* - if(contracts["contracts"].length == 2){ - isBinary = true - percentage = ( Number(prices["last_executed_prices"][market.id][0].last_executed_price) + (100 - Number(prices["last_executed_prices"][market.id][1].last_executed_price)) ) / 2 - percentage = Math.round(percentage)+"%" - let contractName = contracts["contracts"][0].name - name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) - } - */ - let result = { - id: id, - title: name, - url: "https://smarkets.com/event/" + market.event_id + market.slug, - platform: "Smarkets", - description: market.description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Smarkets", {}), - }, - }; - VERBOSE ? console.log(result) : empty(); - results.push(result); - } - VERBOSE ? console.log(results) : empty(); - return results; -}; -//smarkets() diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts new file mode 100644 index 0000000..4a922de --- /dev/null +++ b/src/backend/platforms/smarkets.ts @@ -0,0 +1,177 @@ +/* Imports */ +import axios from "axios"; + +import { calculateStars } from "../utils/stars"; +import { Platform } from "./"; + +/* Definitions */ +let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; +let VERBOSE = false; +let empty = () => 0; +/* Support functions */ + +async function fetchEvents(url) { + let response = await axios({ + url: htmlEndPointEntrance + url, + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); + VERBOSE ? console.log(response) : empty(); + return response; +} + +async function fetchMarkets(eventid) { + let response = await axios({ + url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, + method: "GET", + headers: { + "Content-Type": "text/json", + }, + }) + .then((res) => res.data) + .then((res) => res.markets); + return response; +} + +async function fetchContracts(marketid) { + let response = await axios({ + url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); + VERBOSE ? console.log(response) : empty(); + return response; +} + +async function fetchPrices(marketid) { + let response = await axios({ + url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); + VERBOSE ? console.log(response) : empty(); + return response; +} + +export const smarkets: Platform = { + name: "smarkets", + async fetcher() { + let htmlPath = + "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; + + let events = []; + while (htmlPath) { + let data = await fetchEvents(htmlPath); + events.push(...data.events); + htmlPath = data.pagination.next_page; + } + VERBOSE ? console.log(events) : empty(); + let markets = []; + for (let event of events) { + VERBOSE ? console.log(Date.now()) : empty(); + VERBOSE ? console.log(event.name) : empty(); + let eventMarkets = await fetchMarkets(event.id); + eventMarkets = eventMarkets.map((market) => ({ + ...market, + slug: event.full_slug, + })); + VERBOSE ? console.log("Markets fetched") : empty(); + VERBOSE ? console.log(event.id) : empty(); + VERBOSE ? console.log(eventMarkets) : empty(); + markets.push(...eventMarkets); + //let lastPrices = await fetchPrices(market.id) + } + VERBOSE ? console.log(markets) : empty(); + + let results = []; + for (let market of markets) { + VERBOSE ? console.log("================") : empty(); + VERBOSE ? console.log("Market: ", market) : empty(); + let id = `smarkets-${market.id}`; + let name = market.name; + + let contracts = await fetchContracts(market.id); + VERBOSE ? console.log("Contracts: ", contracts) : empty(); + let prices = await fetchPrices(market.id); + VERBOSE + ? console.log("Prices: ", prices["last_executed_prices"][market.id]) + : empty(); + + let optionsObj = {}; + for (let contract of contracts["contracts"]) { + optionsObj[contract.id] = { name: contract.name }; + } + for (let price of prices["last_executed_prices"][market.id]) { + optionsObj[price.contract_id] = { + ...optionsObj[price.contract_id], + probability: price.last_executed_price + ? Number(price.last_executed_price) + : null, + type: "PROBABILITY", + }; + } + let options: any[] = Object.values(optionsObj); + // monkey patch the case where there are only two options and only one has traded. + if ( + options.length == 2 && + options.map((option) => option.probability).includes(null) + ) { + let nonNullPrice = + options[0].probability == null + ? options[1].probability + : options[0].probability; + options = options.map((option) => { + let probability = option.probability; + return { + ...option, + probability: probability == null ? 100 - nonNullPrice : probability, + // yes, 100, because prices are not yet normalized. + }; + }); + } + + // Normalize normally + let totalValue = options + .map((element) => Number(element.probability)) + .reduce((a, b) => a + b, 0); + + options = options.map((element) => ({ + ...element, + probability: Number(element.probability) / totalValue, + })); + VERBOSE ? console.log(options) : empty(); + + /* + if(contracts["contracts"].length == 2){ + isBinary = true + percentage = ( Number(prices["last_executed_prices"][market.id][0].last_executed_price) + (100 - Number(prices["last_executed_prices"][market.id][1].last_executed_price)) ) / 2 + percentage = Math.round(percentage)+"%" + let contractName = contracts["contracts"][0].name + name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) + } + */ + let result = { + id: id, + title: name, + url: "https://smarkets.com/event/" + market.event_id + market.slug, + platform: "Smarkets", + description: market.description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Smarkets", {}), + }, + }; + VERBOSE ? console.log(result) : empty(); + results.push(result); + } + VERBOSE ? console.log(results) : empty(); + return results; + }, +}; diff --git a/src/backend/platforms/wildeford-fetch.ts b/src/backend/platforms/wildeford.ts similarity index 88% rename from src/backend/platforms/wildeford-fetch.ts rename to src/backend/platforms/wildeford.ts index e68836f..727739a 100644 --- a/src/backend/platforms/wildeford-fetch.ts +++ b/src/backend/platforms/wildeford.ts @@ -1,11 +1,10 @@ /* Imports */ -// import axios from "axios" import { GoogleSpreadsheet } from "google-spreadsheet"; import { applyIfSecretExists } from "../utils/getSecrets"; import { hash } from "../utils/hash"; import { calculateStars } from "../utils/stars"; -import { PlatformFetcher } from "./"; +import { Platform } from "./"; /* Definitions */ const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL @@ -113,18 +112,17 @@ async function processPredictions(predictions) { uniqueTitles.push(result.title); }); return uniqueResults; - // console.log(results) - // console.log(results.map(result => result.options)) - // processPredictions() } -/* Body */ + export async function wildeford_inner(google_api_key) { let predictions = await fetchGoogleDoc(google_api_key); return await processPredictions(predictions); } -//example() -export const wildeford: PlatformFetcher = async function () { - const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey - return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); +export const wildeford: Platform = { + name: "wildeford", + async fetcher() { + const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey + return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); + }, };