From 08715357fda21dc6dfebf8781e1f1a2abbadc4f0 Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Tue, 11 Jan 2022 22:55:46 +0100 Subject: [PATCH] feat: Added Manifold Markets --- src/index.js | 91 +++++++++++++------------- src/platforms/csetforetell-fetch.js | 23 ++++--- src/platforms/manifoldmarkets-fetch.js | 76 +++++++++++++++++++++ src/utils/doEverything.js | 69 +++++++++---------- src/utils/downloadFrontpage.js | 2 +- src/utils/mergeEverything.js | 4 +- src/utils/stars.js | 12 ++++ 7 files changed, 185 insertions(+), 92 deletions(-) create mode 100644 src/platforms/manifoldmarkets-fetch.js diff --git a/src/index.js b/src/index.js index d8d3f85..2af3853 100644 --- a/src/index.js +++ b/src/index.js @@ -2,52 +2,53 @@ import fs from 'fs' import readline from "readline" -import {astralcodexten} from "./platforms/astralcodexten-fetch.js" -import {betfair} from "./platforms/betfair-fetch.js" -import {coupcast} from "./platforms/coupcast-fetch.js" -import {csetforetell} from "./platforms/csetforetell-fetch.js" -import {elicit} from "./platforms/elicit-fetch.js" -import {estimize} from "./platforms/estimize-fetch.js" -import {fantasyscotus} from "./platforms/fantasyscotus-fetch.js" -import {foretold} from "./platforms/foretold-fetch.js" -import {goodjudgment} from "./platforms/goodjudgment-fetch.js" -import {goodjudgmentopen} from "./platforms/goodjudmentopen-fetch.js" -import {hypermind} from "./platforms/hypermind-fetch.js" -import {kalshi} from "./platforms/kalshi-fetch.js" -import {ladbrokes} from "./platforms/ladbrokes-fetch.js" -import {metaculus} from "./platforms/metaculus-fetch.js" -import {omen} from "./platforms/omen-fetch.js" -import {polymarket} from "./platforms/polymarket-fetch.js" -import {predictit} from "./platforms/predictit-fetch.js" -import {rootclaim} from "./platforms/rootclaim-fetch.js" -import {smarkets} from "./platforms/smarkets-fetch.js" -import {wildeford} from "./platforms/wildeford-fetch.js" -import {williamhill} from "./platforms/williamhill-fetch.js" -import {mergeEverything} from "./utils/mergeEverything.js" -import {updateHistory} from "./utils/history/updateHistory.js" -import {rebuildAlgoliaDatabase} from "./utils/algolia.js" -import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js" -import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js" +import { astralcodexten } from "./platforms/astralcodexten-fetch.js" +import { betfair } from "./platforms/betfair-fetch.js" +import { coupcast } from "./platforms/coupcast-fetch.js" +import { csetforetell } from "./platforms/csetforetell-fetch.js" +import { elicit } from "./platforms/elicit-fetch.js" +import { estimize } from "./platforms/estimize-fetch.js" +import { fantasyscotus } from "./platforms/fantasyscotus-fetch.js" +import { foretold } from "./platforms/foretold-fetch.js" +import { goodjudgment } from "./platforms/goodjudgment-fetch.js" +import { goodjudgmentopen } from "./platforms/goodjudmentopen-fetch.js" +import { hypermind } from "./platforms/hypermind-fetch.js" +import { kalshi } from "./platforms/kalshi-fetch.js" +import { ladbrokes } from "./platforms/ladbrokes-fetch.js" +import { manifoldmarkets } from "./platforms/manifoldmarkets-fetch.js" +import { metaculus } from "./platforms/metaculus-fetch.js" +import { omen } from "./platforms/omen-fetch.js" +import { polymarket } from "./platforms/polymarket-fetch.js" +import { predictit } from "./platforms/predictit-fetch.js" +import { rootclaim } from "./platforms/rootclaim-fetch.js" +import { smarkets } from "./platforms/smarkets-fetch.js" +import { wildeford } from "./platforms/wildeford-fetch.js" +import { williamhill } from "./platforms/williamhill-fetch.js" +import { mergeEverything } from "./utils/mergeEverything.js" +import { updateHistory } from "./utils/history/updateHistory.js" +import { rebuildAlgoliaDatabase } from "./utils/algolia.js" +import { rebuildNetlifySiteWithNewData } from "./utils/rebuildNetliftySiteWithNewData.js" +import { doEverything, tryCatchTryAgain } from "./utils/doEverything.js" /* Support functions */ -let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything] -let functionNames = functions.map(fun => fun.name) +let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything] +let functionNames = functions.map(fun => fun.name) let whattodoMessage = functionNames - .slice(0,functionNames.length-5) - .map((functionName,i) => `[${i}]: Download predictions from ${functionName}`) + .slice(0, functionNames.length - 5) + .map((functionName, i) => `[${i}]: Download predictions from ${functionName}`) .join('\n') + - `\n[${functionNames.length-5}]: Merge jsons them into one big json (and push it to mongodb database)` + - `\n[${functionNames.length-4}]: Update history` + - `\n[${functionNames.length-3}]: Rebuild algolia database ("index")` + - `\n[${functionNames.length-2}]: Rebuild netlify site with new data` + + `\n[${functionNames.length - 5}]: Merge jsons them into one big json (and push it to mongodb database)` + + `\n[${functionNames.length - 4}]: Update history` + + `\n[${functionNames.length - 3}]: Rebuild algolia database ("index")` + + `\n[${functionNames.length - 2}]: Rebuild netlify site with new data` + // `\n[${functionNames.length-1}]: Add to history` + - `\n[${functionNames.length-1}]: All of the above` + + `\n[${functionNames.length - 1}]: All of the above` + `\nChoose one option, wisely: #` /* BODY */ -let commandLineUtility = async () => { - let whattodo = async (message,callback) => { +let commandLineUtility = async () => { + let whattodo = async (message, callback) => { const rl = readline.createInterface({ input: process.stdin, output: process.stdout @@ -61,24 +62,24 @@ let commandLineUtility = async () => { let executeoption = async (option) => { option = Number(option) //console.log(functionNames[option]) - if(option < 0){ + if (option < 0) { console.log(`Error, ${option} < 0 or ${option} < 0`) - }else if(option < functions.length){ + } else if (option < functions.length) { await tryCatchTryAgain(functions[option]) } } - if(process.argv.length==3){ + if (process.argv.length == 3) { const option = process.argv[2] // e.g., npm start 15 <- const optionNum = Number(option) - if(!isNaN(optionNum)){ + if (!isNaN(optionNum)) { await executeoption(optionNum) - }else if(option == "all"){ - await executeoption(functions.length-1) // 15 = execute all fetchers - }else{ + } else if (option == "all") { + await executeoption(functions.length - 1) // 15 = execute all fetchers + } else { await whattodo(whattodoMessage, executeoption) } - }else( + } else ( await whattodo(whattodoMessage, executeoption) ) } diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js index ade253f..15c9bcd 100644 --- a/src/platforms/csetforetell-fetch.js +++ b/src/platforms/csetforetell-fetch.js @@ -10,6 +10,8 @@ import { upsert } from "../utils/mongo-wrapper.js" let htmlEndPoint = 'https://www.cset-foretell.com/questions?page=' String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } const DEBUG_MODE = "on"// "off" +const SLEEP_TIME_RANDOM=100//5000 // miliseconds +const SLEEP_TIME_EXTRA=0//1000 /* Support functions */ async function fetchPage(page, cookie) { @@ -171,27 +173,28 @@ async function csetforetell_inner(cookie) { while (!isEnd(response) && isSignedIn(response)) { let htmlLines = response.split("\n") - let h4elements = htmlLines.filter(str => str.includes("
str.includes("
') - let url = h4elementSplit[0].split('
', "") - await sleep(1000 + Math.random() * 1000) // don't be as noticeable + let elementSplit = questionHref.split('">') + let url = elementSplit[0].split('', "").replace("", "") + await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable try { let moreinfo = await fetchStats(url, cookie) @@ -221,7 +224,7 @@ async function csetforetell_inner(cookie) { //i=Number(i)+1 console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers") - await sleep(5000 + Math.random() * 1000) // don't be as noticeable + await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable try { response = await fetchPage(i, cookie) diff --git a/src/platforms/manifoldmarkets-fetch.js b/src/platforms/manifoldmarkets-fetch.js new file mode 100644 index 0000000..2242110 --- /dev/null +++ b/src/platforms/manifoldmarkets-fetch.js @@ -0,0 +1,76 @@ +/* Imports */ +import fs from 'fs' +import axios from "axios" +import { calculateStars } from "../utils/stars.js" +import { upsert } from "../utils/mongo-wrapper.js" + +/* Definitions */ +let endpoint = 'https://us-central1-mantic-markets.cloudfunctions.net/markets' + +/* Support functions */ + +async function fetchData() { + let response = await axios({ + url: endpoint, + method: 'GET', + headers: ({ + 'Content-Type': 'text/html', + }), + }) + .then(response => response.data) + // console.log(response) + return response +} + +async function processPredictions(predictions) { + let results = await predictions.map(prediction => { + let probability = prediction.probability + let options = [ + { + "name": "Yes", + "probability": probability, + "type": "PROBABILITY" + }, + { + "name": "No", + "probability": 1 - probability, + "type": "PROBABILITY" + } + ] + let result = ({ + "title": prediction.question, + "url": prediction.url, + "platform": "Manifold Markets", + "description": prediction.description, + "options": options, + "timestamp": new Date().toISOString(), + "qualityindicators": { + "stars": calculateStars("Manifold Markets", ({ + volume7days: prediction.volume7days, + volume24Hours: prediction.volume24Hours, + liquidity: prediction.pool + })), + "createdTime": prediction.createdTime, + "volume7Days": prediction.volume7Days, + "volume24Hours": prediction.volume24Hours, + "liquidity": prediction.pool + } + }) + return result + }) + let unresolvedResults = results.filter(result => !result.isResolved) + return unresolvedResults //resultsProcessed +} + +/* Body */ + +export async function manifoldmarkets() { + let data = await fetchData() + let results = await processPredictions(data) // somehow needed + // console.log(results) + // let string = JSON.stringify(results, null, 2) + // fs.writeFileSync('polyprediction-questions.json', string); + await upsert(results, "manifoldmarkets-questions") + console.log("Done") +} +//manifoldmarkets() diff --git a/src/utils/doEverything.js b/src/utils/doEverything.js index 5395a49..6b7171d 100644 --- a/src/utils/doEverything.js +++ b/src/utils/doEverything.js @@ -1,52 +1,53 @@ -import {astralcodexten} from "../platforms/astralcodexten-fetch.js" -import {betfair} from "../platforms/betfair-fetch.js" -import {coupcast} from "../platforms/coupcast-fetch.js" -import {csetforetell} from "../platforms/csetforetell-fetch.js" -import {elicit} from "../platforms/elicit-fetch.js" -import {estimize} from "../platforms/estimize-fetch.js" -import {fantasyscotus} from "../platforms/fantasyscotus-fetch.js" -import {foretold} from "../platforms/foretold-fetch.js" -import {goodjudgment} from "../platforms/goodjudgment-fetch.js" -import {goodjudgmentopen} from "../platforms/goodjudmentopen-fetch.js" -import {hypermind} from "../platforms/hypermind-fetch.js" -import {kalshi} from "../platforms/kalshi-fetch.js" -import {ladbrokes} from "../platforms/ladbrokes-fetch.js" -import {metaculus} from "../platforms/metaculus-fetch.js" -import {omen} from "../platforms/omen-fetch.js" -import {polymarket} from "../platforms/polymarket-fetch.js" -import {predictit} from "../platforms/predictit-fetch.js" -import {rootclaim} from "../platforms/rootclaim-fetch.js" -import {smarkets} from "../platforms/smarkets-fetch.js" -import {wildeford} from "../platforms/wildeford-fetch.js" -import {williamhill} from "../platforms/williamhill-fetch.js" -import {mergeEverything} from "./mergeEverything.js" -import {updateHistory} from "./history/updateHistory.js" -import {rebuildAlgoliaDatabase} from "./algolia.js" -import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js" +import { astralcodexten } from "../platforms/astralcodexten-fetch.js" +import { betfair } from "../platforms/betfair-fetch.js" +import { coupcast } from "../platforms/coupcast-fetch.js" +import { csetforetell } from "../platforms/csetforetell-fetch.js" +import { elicit } from "../platforms/elicit-fetch.js" +import { estimize } from "../platforms/estimize-fetch.js" +import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js" +import { foretold } from "../platforms/foretold-fetch.js" +import { goodjudgment } from "../platforms/goodjudgment-fetch.js" +import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js" +import { hypermind } from "../platforms/hypermind-fetch.js" +import { kalshi } from "../platforms/kalshi-fetch.js" +import { ladbrokes } from "../platforms/ladbrokes-fetch.js" +import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js" +import { metaculus } from "../platforms/metaculus-fetch.js" +import { omen } from "../platforms/omen-fetch.js" +import { polymarket } from "../platforms/polymarket-fetch.js" +import { predictit } from "../platforms/predictit-fetch.js" +import { rootclaim } from "../platforms/rootclaim-fetch.js" +import { smarkets } from "../platforms/smarkets-fetch.js" +import { wildeford } from "../platforms/wildeford-fetch.js" +import { williamhill } from "../platforms/williamhill-fetch.js" +import { mergeEverything } from "./mergeEverything.js" +import { updateHistory } from "./history/updateHistory.js" +import { rebuildAlgoliaDatabase } from "./algolia.js" +import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData.js" /* Do everything */ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); -} +} -export async function tryCatchTryAgain (fun) { - try{ +export async function tryCatchTryAgain(fun) { + try { console.log("Initial try") await fun() - }catch (error) { + } catch (error) { sleep(10000) console.log("Second try") console.log(error) - try{ + try { await fun() - }catch (error){ + } catch (error) { console.log(error) } } } -export async function doEverything(){ - let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] +export async function doEverything() { + let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku. console.log("") @@ -61,7 +62,7 @@ export async function doEverything(){ console.log("") console.log("") - for(let fun of functions){ + for (let fun of functions) { console.log("") console.log("") console.log("****************************") diff --git a/src/utils/downloadFrontpage.js b/src/utils/downloadFrontpage.js index 19b0fe0..949922d 100644 --- a/src/utils/downloadFrontpage.js +++ b/src/utils/downloadFrontpage.js @@ -2,7 +2,7 @@ import fs from "fs" import { mongoReadWithReadCredentials } from "./mongo-wrapper.js" -let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json' +let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-backend/data/frontpage.json' let shuffle = (array) => { // https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array let currentIndex = array.length, randomIndex; diff --git a/src/utils/mergeEverything.js b/src/utils/mergeEverything.js index e30d2f3..512f778 100644 --- a/src/utils/mergeEverything.js +++ b/src/utils/mergeEverything.js @@ -1,14 +1,14 @@ import { mongoRead, upsert } from "./mongo-wrapper.js" /* Merge everything */ -let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"] +let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"] let suffix = "-questions" export async function mergeEverythingInner(){ let merged = [] for(let set of sets){ let json = await mongoRead(set+suffix) - console.log(`${set} has ${json.length} questions`) + console.log(`${set} has ${json.length} questions\n`) merged = merged.concat(json) } let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")})) diff --git a/src/utils/stars.js b/src/utils/stars.js index c5b8df9..fd4cc71 100644 --- a/src/utils/stars.js +++ b/src/utils/stars.js @@ -156,6 +156,15 @@ function calculateStarsLadbrokes(data) { return starsInteger } +function calculateStarsManifoldMarkets(data) { + let nuno = data => 2 + let eli = (data) => null + let misha = (data) => null + let starsDecimal = average([nuno(data)]) //, eli(data), misha(data)]) + let starsInteger = Math.round(starsDecimal) + return starsInteger +} + function calculateStarsMetaculus(data) { let nuno = data => data.numforecasts > 300 ? 4 : (data.numforecasts > 100 ? 3 : 2) let eli = (data) => 3 @@ -278,6 +287,9 @@ export function calculateStars(platform, data) { case "Ladbrokes": stars = calculateStarsLadbrokes(data) break; + case "Manifold Markets": + stars = calculateStarsManifoldMarkets(data) + break; case "Metaculus": stars = calculateStarsMetaculus(data) break;