From 20c9c820fb946a3ae22bf7c9047434331ec6391b Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Fri, 4 Feb 2022 14:54:04 -0500 Subject: [PATCH] feat: Added infer --- src/platforms/deprecated/omen-fetch.js | 4 ++-- src/platforms/infer-fetch.js | 28 +++++++++++----------- src/platforms/wildeford-fetch.js | 5 ++-- src/platforms/williamhill-fetch.js | 4 +++- src/utils/mergeEverything.js | 32 +++----------------------- src/utils/platforms.js | 1 + src/utils/stars.js | 12 ++++++++++ 7 files changed, 37 insertions(+), 49 deletions(-) diff --git a/src/platforms/deprecated/omen-fetch.js b/src/platforms/deprecated/omen-fetch.js index b4548e4..93afc30 100644 --- a/src/platforms/deprecated/omen-fetch.js +++ b/src/platforms/deprecated/omen-fetch.js @@ -1,8 +1,8 @@ /* Imports */ import fs from 'fs' import axios from "axios" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import { calculateStars } from "../../utils/stars.js" +import {upsert} from "../../utils/mongo-wrapper.js" /* Definitions */ let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen" diff --git a/src/platforms/infer-fetch.js b/src/platforms/infer-fetch.js index 0981b69..cdd6ce0 100644 --- a/src/platforms/infer-fetch.js +++ b/src/platforms/infer-fetch.js @@ -7,11 +7,11 @@ import { calculateStars } from "../utils/stars.js" import { upsert } from "../utils/mongo-wrapper.js" /* Definitions */ -let htmlEndPoint = 'https://www.cset-foretell.com/questions?page=' +let htmlEndPoint = 'https://www.infer-pub.com/questions' String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } const DEBUG_MODE = "on"// "off" -const SLEEP_TIME_RANDOM=100//5000 // miliseconds -const SLEEP_TIME_EXTRA=0//1000 +const SLEEP_TIME_RANDOM = 7000 // miliseconds +const SLEEP_TIME_EXTRA = 2000 /* Support functions */ async function fetchPage(page, cookie) { @@ -19,7 +19,7 @@ async function fetchPage(page, cookie) { if (page == 1) { cookie = cookie.split(";")[0] // Interesting that it otherwise doesn't work :( } - let urlEndpoint = htmlEndPoint + page + let urlEndpoint = `${htmlEndPoint}/?page=${page}` console.log(urlEndpoint) let response = await axios({ url: urlEndpoint, @@ -132,7 +132,7 @@ async function fetchStats(questionUrl, cookie) { "qualityindicators": { "numforecasts": Number(numforecasts), "numforecasters": Number(numforecasters), - "stars": calculateStars("CSET-foretell", { numforecasts }) + "stars": calculateStars("Infer", { numforecasts }) } } @@ -164,7 +164,7 @@ function sleep(ms) { /* Body */ -async function csetforetell_inner(cookie) { +async function infer_inner(cookie) { let i = 1 let response = await fetchPage(i, cookie) let results = [] @@ -174,9 +174,9 @@ async function csetforetell_inner(cookie) { let htmlLines = response.split("\n") // let h4elements = htmlLines.filter(str => str.includes("
str.includes("https://www.infer-pub.com/questions/")) // console.log(questionHrefs) - + if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { //console.log(response) @@ -201,7 +201,7 @@ async function csetforetell_inner(cookie) { let question = ({ "title": title, "url": url, - "platform": "CSET-foretell", + "platform": "Infer", ...moreinfo }) if (i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) { @@ -223,7 +223,7 @@ async function csetforetell_inner(cookie) { i++ //i=Number(i)+1 - console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers") + console.log("Sleeping for ~5secs so as to not be as noticeable to the infer servers") await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable try { @@ -234,10 +234,10 @@ async function csetforetell_inner(cookie) { } } // let string = JSON.stringify(results,null, 2) - // fs.writeFileSync('./data/csetforetell-questions.json', string); + // fs.writeFileSync('./data/infer-questions.json', string); // console.log(results) if (results.length > 0) { - await upsert(results, "csetforetell-questions") + await upsert(results, "infer-questions") } else { console.log("Not updating results, as process was not signed in") } @@ -249,6 +249,6 @@ async function csetforetell_inner(cookie) { export async function infer() { - let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell") - await applyIfCookieExists(cookie, csetforetell_inner) + let cookie = process.env.INFER_COOKIE || getCookie("infer") + await applyIfCookieExists(cookie, infer_inner) } diff --git a/src/platforms/wildeford-fetch.js b/src/platforms/wildeford-fetch.js index 01fe09d..aa693dd 100644 --- a/src/platforms/wildeford-fetch.js +++ b/src/platforms/wildeford-fetch.js @@ -10,8 +10,7 @@ import {upsert} from "../utils/mongo-wrapper.js" /* Definitions */ const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0" // spreadsheet key is the long id in the sheets URL const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0` -// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0&range=C4 - +// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0 /* Support functions */ const formatRow = row => { @@ -106,7 +105,7 @@ async function processPredictions(predictions){ export async function wildeford_inner(google_api_key) { let predictions = await fetchGoogleDoc(google_api_key) let results = await processPredictions(predictions) // somehow needed - // console.log(results) + // console.log(results.sort((a,b) => (a.title > b.title))) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('polyprediction-questions.json', string); await upsert(results, "wildeford-questions") diff --git a/src/platforms/williamhill-fetch.js b/src/platforms/williamhill-fetch.js index e97c4b5..82a44d6 100644 --- a/src/platforms/williamhill-fetch.js +++ b/src/platforms/williamhill-fetch.js @@ -110,6 +110,8 @@ let processResults = (html) => { "stars": calculateStars("WilliamHill", ({})) } }) + results = results.filter(result => result.title.length > 4 && result.title != "2024 or later") + // removes some predictions because hard to parse. results.push(obj) } @@ -128,7 +130,7 @@ export async function williamhill() { // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/williamhill-questions.json', string); await upsert(results, "williamhill-questions") - + console.log(results.sort((a,b) => (a.title > b.title))) console.log("Done") } //williamhill() diff --git a/src/utils/mergeEverything.js b/src/utils/mergeEverything.js index a45e913..27a87e1 100644 --- a/src/utils/mergeEverything.js +++ b/src/utils/mergeEverything.js @@ -1,39 +1,13 @@ import { mongoRead, upsert } from "./mongo-wrapper.js"; import { platformNames } from "./platforms.js" /* Merge everything */ -// let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"] -let sets = [ - "betfair", - "coupcast", - "csetforetell", - "elicit", - /* "estimize" ,*/ "fantasyscotus", - "foretold", - "givewellopenphil", - "goodjudgment", - "goodjudmentopen", - "hypermind", - "kalshi", - "ladbrokes", - "manifoldmarkets", - "metaculus", - "omen", - "polymarket", - "predictit", - "rootclaim", - "smarkets", - "wildeford", - "williamhill", - "xrisk", -]; - let suffix = "-questions"; export async function mergeEverythingInner() { let merged = []; - for (let set of sets) { - let json = await mongoRead(set + suffix); - console.log(`${set} has ${json.length} questions\n`); + for (let platformName of platformNames) { + let json = await mongoRead(platformName + suffix); + console.log(`${platformName} has ${json.length} questions\n`); merged = merged.concat(json); } let mergedprocessed = merged.map((element) => ({ diff --git a/src/utils/platforms.js b/src/utils/platforms.js index 5f3fee0..25e4ac0 100644 --- a/src/utils/platforms.js +++ b/src/utils/platforms.js @@ -32,6 +32,7 @@ export const platformNames = [ "givewellopenphil", "goodjudgment", "goodjudmentopen", + "infer", "kalshi", "ladbrokes", "manifoldmarkets", diff --git a/src/utils/stars.js b/src/utils/stars.js index 86304be..41efe41 100644 --- a/src/utils/stars.js +++ b/src/utils/stars.js @@ -133,6 +133,15 @@ function calculateStarsHypermind(data) { return starsInteger; } +function calculateStarsInfer(data) { + let nuno = (data) => 2; + let eli = (data) => null; + let misha = (data) => null; + let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; +} + function calculateStarsKalshi(data) { let nuno = (data) => data.interest > 500 && data.shares_volume > 10000 @@ -295,6 +304,9 @@ export function calculateStars(platform, data) { case "Hypermind": stars = calculateStarsHypermind(data); break; + case "Infer": + stars = calculateStarsInfer(data); + break; case "Kalshi": stars = calculateStarsKalshi(data); break;