From 494e8d74ada8232848f7a5cc9da7656d42fa50ff Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Fri, 8 Jul 2022 22:25:11 -0400 Subject: [PATCH] feat: Fix Good Judgment flow, start writting the Insight Parser. Note that the Insight Prediction parser is in fact not complete --- src/backend/platforms/goodjudgmentopen.ts | 120 +++++++++++----------- src/backend/platforms/insight.ts | 118 +++++++++++++++++++++ src/backend/platforms/registry.ts | 2 + 3 files changed, 178 insertions(+), 62 deletions(-) create mode 100644 src/backend/platforms/insight.ts diff --git a/src/backend/platforms/goodjudgmentopen.ts b/src/backend/platforms/goodjudgmentopen.ts index cf17e72..3cb9e7c 100644 --- a/src/backend/platforms/goodjudgmentopen.ts +++ b/src/backend/platforms/goodjudgmentopen.ts @@ -17,12 +17,19 @@ const annoyingPromptUrls = [ "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters", "https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread", "https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", + "https://www.gjopen.com/questions/2437-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen" ]; const DEBUG_MODE: "on" | "off" = "off"; // "on" const id = () => 0; /* Support functions */ +function cleanDescription(text: string) { + let md = toMarkdown(text); + let result = md.replaceAll("---", "-").replaceAll(" ", " "); + return result; +} + async function fetchPage(page: number, cookie: string) { const response: string = await axios({ url: htmlEndPoint + page, @@ -40,82 +47,68 @@ async function fetchStats(questionUrl: string, cookie: string) { url: questionUrl + "/stats", method: "GET", headers: { + "Content-Type": "text/html", Cookie: cookie, Referer: questionUrl, }, }).then((res) => res.data); - //console.log(response) - // Is binary? - let isbinary = response.includes("binary?":true"); + if (response.includes("Sign up or sign in to forecast")) { + throw Error("Not logged in"); + } + // Init + let options: FullQuestionOption[] = []; - let options: FetchedQuestion["options"] = []; - if (isbinary) { - // Crowd percentage - let htmlElements = response.split("\n"); - let h3Element = htmlElements.filter((str) => str.includes("

"))[0]; - // console.log(h3Element) - let crowdpercentage = h3Element.split(">")[1].split("<")[0]; - let probability = Number(crowdpercentage.replace("%", "")) / 100; - options.push( - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans - type: "PROBABILITY", - } - ); - } else { - let optionsHtmlElement = ""; - let tablesAsJson = Tabletojson.convert(optionsHtmlElement); - let firstTable = tablesAsJson[0]; - options = firstTable.map((element: any) => ({ - name: element["0"], - probability: Number(element["1"].replace("%", "")) / 100, + // Parse the embedded json + let htmlElements = response.split("\n"); + let jsonLines = htmlElements.filter((element) => + element.includes("data-react-props") + ); + let embeddedJsons = jsonLines.map((jsonLine, i) => { + let innerJSONasHTML = jsonLine.split('data-react-props="')[1].split('"')[0]; + let json = JSON.parse(innerJSONasHTML.replaceAll(""", '"')); + return json; + }); + let firstEmbeddedJson = embeddedJsons[0]; + let title = firstEmbeddedJson.question.name; + let description = cleanDescription(firstEmbeddedJson.question.description); + let comments_count = firstEmbeddedJson.question.comments_count; + let numforecasters = firstEmbeddedJson.question.predictors_count; + let numforecasts = firstEmbeddedJson.question.prediction_sets_count; + let questionType = firstEmbeddedJson.question.type; + if ( + questionType.includes("Binary") || + questionType.includes("NonExclusiveOpinionPoolQuestion") || + questionType.includes("Forecast::Question") || + !questionType.includes("Forecast::MultiTimePeriodQuestion") + ) { + options = firstEmbeddedJson.question.answers.map((answer: any) => ({ + name: answer.name, + probability: answer.normalized_probability, type: "PROBABILITY", })); - //console.log(optionsHtmlElement) - //console.log(options) + if (options.length == 1 && options[0].name == "Yes") { + let probabilityNo = + options[0].probability > 1 + ? 1 - options[0].probability / 100 + : 1 - options[0].probability; + options.push({ + name: "No", + probability: probabilityNo, + type: "PROBABILITY", + }); + } } - - // Description - let descriptionraw = response.split( - `
` - )[1]; - let descriptionprocessed1 = descriptionraw.split(`
`)[0]; - let descriptionprocessed2 = toMarkdown(descriptionprocessed1); - let descriptionprocessed3 = descriptionprocessed2 - .split("\n") - .filter((string) => !string.includes("Confused? Check our")) - .join("\n"); - let description = descriptionprocessed3; - - // Number of forecasts - let numforecasts = response - .split("prediction_sets_count":")[1] - .split(",")[0]; - //console.log(numforecasts) - - // Number of predictors - let numforecasters = response - .split("predictors_count":")[1] - .split(",")[0]; - //console.log(numpredictors) - let result = { - description, - options, + description: description, + options: options, qualityindicators: { numforecasts: Number(numforecasts), numforecasters: Number(numforecasters), + comments_count: Number(comments_count), }, - // this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now - } as any; - + }; + // console.log(JSON.stringify(result, null, 4)); return result; } @@ -150,6 +143,7 @@ async function goodjudgmentopen_inner(cookie: string) { let results = []; let init = Date.now(); // console.log("Downloading... This might take a couple of minutes. Results will be shown.") + console.log("Page #1") while (!reachedEnd(response) && isSignedIn(response)) { let htmlLines = response.split("\n"); DEBUG_MODE == "on" ? htmlLines.forEach((line) => console.log(line)) : id(); @@ -187,6 +181,8 @@ async function goodjudgmentopen_inner(cookie: string) { if (j % 30 == 0 || DEBUG_MODE == "on") { console.log(`Page #${i}`); console.log(question); + }else{ + console.log(question.title) } // console.log(question) results.push(question); diff --git a/src/backend/platforms/insight.ts b/src/backend/platforms/insight.ts new file mode 100644 index 0000000..c3a7e89 --- /dev/null +++ b/src/backend/platforms/insight.ts @@ -0,0 +1,118 @@ +/* Imports */ +import axios from "axios"; + +import { FetchedQuestion, Platform } from "."; + +/* Definitions */ +const platformName = "insight"; +const marketsEnpoint = "https://insightprediction.com/api/markets"; +const getMarketEndpoint = (id: number) => `https://insightprediction.com/api/markets/${id}` + +/* Support functions */ + +async function fetchQuestionStats(bearer: string, marketId: number){ + const response = await axios({ + url: getMarketEndpoint(marketId), + method: "GET", + headers: { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": `Bearer ${bearer}`, + }, + }).then((res) => res.data); + // console.log(response) + return response; +} + +async function fetchPage(bearer: string, pageNum: number) { + const response = await axios({ + url: `${marketsEnpoint}?page=${pageNum}`, + method: "GET", + headers: { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": `Bearer ${bearer}`, + }, + }).then((res) => res.data); + // console.log(response) + return response; +} + +async function fetchData(bearer: string){ + let pageNum = 1 + let reachedEnd = false + let results = [] + while(!reachedEnd){ + let newPage = await fetchPage(bearer, pageNum) + let newPageData = newPage.data + + let marketsWithStats = newPageData.map(marketData => { + let marketStats = fetchQuestionStats(bearer, marketData.id) + return ({...marketStats, ...marketData}) + }) + + console.log(`Page = #${pageNum}`) + // console.log(newPageData) + console.log(marketsWithStats) + results.push(...marketsWithStats) + + let newPagination = newPage.meta.pagination + if(newPagination.total_pages == pageNum ){ + reachedEnd = true + }else{ + pageNum = pageNum + 1 + } + } +} + +async function processPredictions(predictions: any[]) { + let results = await predictions.map((prediction) => { + const id = `${platformName}-${prediction.id}`; + const probability = prediction.probability; + const options: FetchedQuestion["options"] = [ + { + name: "Yes", + probability: probability, + type: "PROBABILITY", + }, + { + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; + const result: FetchedQuestion = { + id, + title: prediction.title, + url: "https://example.com", + description: prediction.description, + options, + qualityindicators: { + // other: prediction.otherx, + // indicators: prediction.indicatorx, + }, + }; + return result; + }); + return results; //resultsProcessed +} + +/* Body */ + +export const insight: Platform = { + name: platformName, + label: "Insight Prediction", + color: "#ff0000", + version: "v0", + async fetcher() { + let bearer = process.env.INSIGHT_BEARER; + let pageNum = 1 + let data = await fetchData(bearer); + console.log(data) + let results = [] // await processPredictions(data); // somehow needed + return results; + }, + calculateStars(data) { + return 2; + }, +}; diff --git a/src/backend/platforms/registry.ts b/src/backend/platforms/registry.ts index ce022a6..d678bfc 100644 --- a/src/backend/platforms/registry.ts +++ b/src/backend/platforms/registry.ts @@ -7,6 +7,7 @@ import { goodjudgmentopen } from "./goodjudgmentopen"; import { guesstimate } from "./guesstimate"; import { Platform, PlatformConfig } from "./index"; import { infer } from "./infer"; +import { insight } from "./insight"; import { kalshi } from "./kalshi"; import { manifold } from "./manifold"; import { metaculus } from "./metaculus"; @@ -28,6 +29,7 @@ export const getPlatforms = (): Platform[] => { goodjudgmentopen, guesstimate, infer, + insight, kalshi, manifold, metaculus,