diff --git a/src/backend/platforms/example-fetch.ts b/src/backend/platforms/_example.ts similarity index 87% rename from src/backend/platforms/example-fetch.ts rename to src/backend/platforms/_example.ts index 7c2e9ac..92119b9 100644 --- a/src/backend/platforms/example-fetch.ts +++ b/src/backend/platforms/_example.ts @@ -5,7 +5,8 @@ import { calculateStars } from "../utils/stars"; import { Platform } from "./"; /* Definitions */ -let endpoint = "https://example.com/"; +const platformName = "example"; +const endpoint = "https://example.com/"; /* Support functions */ @@ -23,7 +24,7 @@ async function fetchData() { async function processPredictions(predictions) { let results = await predictions.map((prediction) => { - let id = `example-${prediction.id}`; + let id = `${platformName}-${prediction.id}`; let probability = prediction.probability; let options = [ { @@ -40,12 +41,12 @@ async function processPredictions(predictions) { let result = { title: prediction.title, url: `https://example.com`, - platform: "Example", + platform: platformName, description: prediction.description, options: options, timestamp: new Date().toISOString(), qualityindicators: { - stars: calculateStars("Example", { + stars: calculateStars(platformName, { /* some: somex, factors: factors */ }), other: prediction.otherx, @@ -60,7 +61,7 @@ async function processPredictions(predictions) { /* Body */ export const example: Platform = { - name: "example", + name: platformName, label: "Example platform", color: "#ff0000", async fetcher() { diff --git a/src/backend/platforms/betfair.ts b/src/backend/platforms/betfair.ts index e7a1b1e..db7c410 100644 --- a/src/backend/platforms/betfair.ts +++ b/src/backend/platforms/betfair.ts @@ -84,7 +84,7 @@ async function processPredictions(data) { /* if(Math.floor(Math.random() * 10) % 20 ==0){ console.log(JSON.stringify(prediction, null, 4)) } */ - let id = `betfair-${prediction.marketId}`; + let id = `${platformName}-${prediction.marketId}`; let normalizationFactor = prediction.options .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0) .map((option) => option.lastPriceTraded) diff --git a/src/backend/platforms/givewellopenphil.ts b/src/backend/platforms/givewellopenphil.ts index 13aa8ae..e8170ac 100644 --- a/src/backend/platforms/givewellopenphil.ts +++ b/src/backend/platforms/givewellopenphil.ts @@ -79,6 +79,7 @@ export const givewellopenphil: Platform = { const data = JSON.parse(rawdata); const dataWithDate = data.map((datum: any) => ({ ...datum, + platform: platformName, timestamp: "2021-02-23", })); return dataWithDate; diff --git a/src/backend/platforms/infer.ts b/src/backend/platforms/infer.ts index 5a9301f..2ac48d0 100644 --- a/src/backend/platforms/infer.ts +++ b/src/backend/platforms/infer.ts @@ -3,6 +3,7 @@ import axios from "axios"; import { Tabletojson } from "tabletojson"; import { applyIfSecretExists } from "../utils/getSecrets"; +import { measureTime } from "../utils/measureTime"; import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; import { Forecast, Platform } from "./"; @@ -180,96 +181,84 @@ function sleep(ms) { /* Body */ -async function infer_inner(cookie) { +async function infer_inner(cookie: string) { let i = 1; let response = await fetchPage(i, cookie); let results: Forecast[] = []; - let init = Date.now(); - // console.log("Downloading... This might take a couple of minutes. Results will be shown.") - while (!isEnd(response) && isSignedIn(response)) { - let htmlLines = response.split("\n"); - // let h4elements = htmlLines.filter(str => str.includes("
{ + // console.log("Downloading... This might take a couple of minutes. Results will be shown.") + while (!isEnd(response) && isSignedIn(response)) { + let htmlLines = response.split("\n"); + // let h4elements = htmlLines.filter(str => str.includes("
'); + let url = elementSplit[0].split('", "") + .replace("
", "") + .replace("", ""); + await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable - let elementSplit = questionHref.split('">'); - let url = elementSplit[0].split('", "") - .replace("
", "") - .replace("", ""); + try { + let moreinfo = await fetchStats(url, cookie); + let questionNumRegex = new RegExp("questions/([0-9]+)"); + let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; + let id = `${platformName}-${questionNum}`; + let question: Forecast = { + id: id, + title: title, + url: url, + platform: platformName, + ...moreinfo, + }; + if ( + i % 30 == 0 && + !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") + ) { + console.log(`Page #${i}`); + console.log(question); + } + results.push(question); + if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { + console.log(url); + console.log(question); + } + } catch (error) { + console.log(error); + console.log( + `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json` + ); + } + } + + i++; + + console.log( + "Sleeping for ~5secs so as to not be as noticeable to the infer servers" + ); await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable try { - let moreinfo = await fetchStats(url, cookie); - let questionNumRegex = new RegExp("questions/([0-9]+)"); - let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; - let id = `${platformName}-${questionNum}`; - let question = { - id: id, - title: title, - url: url, - platform: "Infer", - ...moreinfo, - }; - if ( - i % 30 == 0 && - !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") - ) { - console.log(`Page #${i}`); - console.log(question); - } - results.push(question); - if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { - console.log(url); - console.log(question); - } + response = await fetchPage(i, cookie); } catch (error) { console.log(error); console.log( - `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json` + `The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages` ); } } - - i++; - //i=Number(i)+1 - - console.log( - "Sleeping for ~5secs so as to not be as noticeable to the infer servers" - ); - await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable - - try { - response = await fetchPage(i, cookie); - } catch (error) { - console.log(error); - console.log( - `The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages` - ); - } - } - - let end = Date.now(); - let difference = end - init; - console.log( - `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` - ); + }); if (results.length === 0) { console.log("Not updating results, as process was not signed in"); diff --git a/src/backend/platforms/manifold.ts b/src/backend/platforms/manifold.ts index 9368013..7de7756 100644 --- a/src/backend/platforms/manifold.ts +++ b/src/backend/platforms/manifold.ts @@ -2,7 +2,7 @@ import axios from "axios"; import { calculateStars } from "../utils/stars"; -import { Platform } from "./"; +import { Forecast, Platform } from "./"; /* Definitions */ const platformName = "manifold"; @@ -23,7 +23,7 @@ async function fetchData() { return response; } -function showStatistics(results) { +function showStatistics(results: Forecast[]) { console.log(`Num unresolved markets: ${results.length}`); let sum = (arr) => arr.reduce((tally, a) => tally + a, 0); let num2StarsOrMore = results.filter( @@ -44,8 +44,8 @@ function showStatistics(results) { } async function processPredictions(predictions) { - let results = await predictions.map((prediction) => { - let id = `manifold-${prediction.id}`; // oops, doesn't match platform name + let results: Forecast[] = await predictions.map((prediction) => { + let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name let probability = prediction.probability; let options = [ { @@ -59,11 +59,11 @@ async function processPredictions(predictions) { type: "PROBABILITY", }, ]; - let result = { + const result: Forecast = { id: id, title: prediction.question, url: prediction.url, - platform: "Manifold Markets", + platform: platformName, description: prediction.description, options: options, timestamp: new Date().toISOString(), @@ -84,9 +84,11 @@ async function processPredictions(predictions) { }; return result; }); - let unresolvedResults = results.filter((result) => !result.extra.isResolved); - // console.log(unresolvedResults); - return unresolvedResults; //resultsProcessed + + const unresolvedResults = results.filter( + (result) => !result.extra.isResolved + ); + return unresolvedResults; } export const manifold: Platform = { diff --git a/src/backend/platforms/metaculus.ts b/src/backend/platforms/metaculus.ts index f04b12f..76ac37f 100644 --- a/src/backend/platforms/metaculus.ts +++ b/src/backend/platforms/metaculus.ts @@ -50,7 +50,7 @@ async function fetchMetaculusQuestions(next) { return data; } -function sleep(ms) { +function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts index 4be7a54..260df9a 100644 --- a/src/backend/platforms/smarkets.ts +++ b/src/backend/platforms/smarkets.ts @@ -1,7 +1,7 @@ import axios from "axios"; import { calculateStars } from "../utils/stars"; -import { Platform } from "./"; +import { Forecast, Platform } from "./"; /* Definitions */ const platformName = "smarkets"; @@ -159,7 +159,7 @@ export const smarkets: Platform = { name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) } */ - let result = { + let result: Forecast = { id: id, title: name, url: "https://smarkets.com/event/" + market.event_id + market.slug, diff --git a/src/backend/utils/getSecrets.ts b/src/backend/utils/getSecrets.ts index 41e9284..2a73260 100644 --- a/src/backend/utils/getSecrets.ts +++ b/src/backend/utils/getSecrets.ts @@ -1,10 +1,10 @@ export async function applyIfSecretExists( - cookie, - fun: (...args: any[]) => T + cookie: string, + fun: (cookie: string) => T ) { if (cookie) { return await fun(cookie); - } else if (!cookie) { + } else { console.log( `Cannot proceed with ${fun.name} because cookie does not exist` ); diff --git a/src/pages/dashboards.tsx b/src/pages/dashboards.tsx index 26f7d8d..4aa8f65 100644 --- a/src/pages/dashboards.tsx +++ b/src/pages/dashboards.tsx @@ -93,7 +93,7 @@ export default function Home({ let isGraubardEasterEgg = (name) => (name == "Clay Graubard" ? true : false); return ( - + {/* Display forecasts */}

({ - item: result, - score: 0, - })); - } else { - console.log("Error in getDashboardForecastsByDashboardId"); - } } catch (error) { console.log(error); } finally { return { - dashboardForecasts: dashboardForecastCompatibleWithFuse, + dashboardForecasts: dashboardContents, dashboardItem, }; }