quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Added Manifold Markets

Browse Source
This commit is contained in:
NunoSempere 2022-01-11 22:55:46 +01:00
parent 62b0f8e210
commit 08715357fd
7 changed files with 185 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/index.js
View File

@ -2,52 +2,53 @@
import fs from 'fs' import fs from 'fs'
import readline from "readline" import readline from "readline"
import {astralcodexten} from "./platforms/astralcodexten-fetch.js" import { astralcodexten } from "./platforms/astralcodexten-fetch.js"
import {betfair} from "./platforms/betfair-fetch.js" import { betfair } from "./platforms/betfair-fetch.js"
import {coupcast} from "./platforms/coupcast-fetch.js" import { coupcast } from "./platforms/coupcast-fetch.js"
import {csetforetell} from "./platforms/csetforetell-fetch.js" import { csetforetell } from "./platforms/csetforetell-fetch.js"
import {elicit} from "./platforms/elicit-fetch.js" import { elicit } from "./platforms/elicit-fetch.js"
import {estimize} from "./platforms/estimize-fetch.js" import { estimize } from "./platforms/estimize-fetch.js"
import {fantasyscotus} from "./platforms/fantasyscotus-fetch.js" import { fantasyscotus } from "./platforms/fantasyscotus-fetch.js"
import {foretold} from "./platforms/foretold-fetch.js" import { foretold } from "./platforms/foretold-fetch.js"
import {goodjudgment} from "./platforms/goodjudgment-fetch.js" import { goodjudgment } from "./platforms/goodjudgment-fetch.js"
import {goodjudgmentopen} from "./platforms/goodjudmentopen-fetch.js" import { goodjudgmentopen } from "./platforms/goodjudmentopen-fetch.js"
import {hypermind} from "./platforms/hypermind-fetch.js" import { hypermind } from "./platforms/hypermind-fetch.js"
import {kalshi} from "./platforms/kalshi-fetch.js" import { kalshi } from "./platforms/kalshi-fetch.js"
import {ladbrokes} from "./platforms/ladbrokes-fetch.js" import { ladbrokes } from "./platforms/ladbrokes-fetch.js"
import {metaculus} from "./platforms/metaculus-fetch.js" import { manifoldmarkets } from "./platforms/manifoldmarkets-fetch.js"
import {omen} from "./platforms/omen-fetch.js" import { metaculus } from "./platforms/metaculus-fetch.js"
import {polymarket} from "./platforms/polymarket-fetch.js" import { omen } from "./platforms/omen-fetch.js"
import {predictit} from "./platforms/predictit-fetch.js" import { polymarket } from "./platforms/polymarket-fetch.js"
import {rootclaim} from "./platforms/rootclaim-fetch.js" import { predictit } from "./platforms/predictit-fetch.js"
import {smarkets} from "./platforms/smarkets-fetch.js" import { rootclaim } from "./platforms/rootclaim-fetch.js"
import {wildeford} from "./platforms/wildeford-fetch.js" import { smarkets } from "./platforms/smarkets-fetch.js"
import {williamhill} from "./platforms/williamhill-fetch.js" import { wildeford } from "./platforms/wildeford-fetch.js"
import {mergeEverything} from "./utils/mergeEverything.js" import { williamhill } from "./platforms/williamhill-fetch.js"
import {updateHistory} from "./utils/history/updateHistory.js" import { mergeEverything } from "./utils/mergeEverything.js"
import {rebuildAlgoliaDatabase} from "./utils/algolia.js" import { updateHistory } from "./utils/history/updateHistory.js"
import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js" import { rebuildAlgoliaDatabase } from "./utils/algolia.js"
import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js" import { rebuildNetlifySiteWithNewData } from "./utils/rebuildNetliftySiteWithNewData.js"
import { doEverything, tryCatchTryAgain } from "./utils/doEverything.js"
/* Support functions */ /* Support functions */
let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything] let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
let functionNames = functions.map(fun => fun.name) let functionNames = functions.map(fun => fun.name)
let whattodoMessage = functionNames let whattodoMessage = functionNames
.slice(0,functionNames.length-5) .slice(0, functionNames.length - 5)
.map((functionName,i) => `[${i}]: Download predictions from ${functionName}`) .map((functionName, i) => `[${i}]: Download predictions from ${functionName}`)
.join('\n') + .join('\n') +
`\n[${functionNames.length-5}]: Merge jsons them into one big json (and push it to mongodb database)` + `\n[${functionNames.length - 5}]: Merge jsons them into one big json (and push it to mongodb database)` +
`\n[${functionNames.length-4}]: Update history` + `\n[${functionNames.length - 4}]: Update history` +
`\n[${functionNames.length-3}]: Rebuild algolia database ("index")` + `\n[${functionNames.length - 3}]: Rebuild algolia database ("index")` +
`\n[${functionNames.length-2}]: Rebuild netlify site with new data` + `\n[${functionNames.length - 2}]: Rebuild netlify site with new data` +
// `\n[${functionNames.length-1}]: Add to history` + // `\n[${functionNames.length-1}]: Add to history` +
`\n[${functionNames.length-1}]: All of the above` + `\n[${functionNames.length - 1}]: All of the above` +
`\nChoose one option, wisely: #` `\nChoose one option, wisely: #`
/* BODY */ /* BODY */
let commandLineUtility = async () => { let commandLineUtility = async () => {
let whattodo = async (message,callback) => { let whattodo = async (message, callback) => {
const rl = readline.createInterface({ const rl = readline.createInterface({
input: process.stdin, input: process.stdin,
output: process.stdout output: process.stdout
@ -61,24 +62,24 @@ let commandLineUtility = async () => {
let executeoption = async (option) => { let executeoption = async (option) => {
option = Number(option) option = Number(option)
//console.log(functionNames[option]) //console.log(functionNames[option])
if(option < 0){ if (option < 0) {
console.log(`Error, ${option} < 0 or ${option} < 0`) console.log(`Error, ${option} < 0 or ${option} < 0`)
}else if(option < functions.length){ } else if (option < functions.length) {
await tryCatchTryAgain(functions[option]) await tryCatchTryAgain(functions[option])
} }
} }
if(process.argv.length==3){ if (process.argv.length == 3) {
const option = process.argv[2] // e.g., npm start 15 <- const option = process.argv[2] // e.g., npm start 15 <-
const optionNum = Number(option) const optionNum = Number(option)
if(!isNaN(optionNum)){ if (!isNaN(optionNum)) {
await executeoption(optionNum) await executeoption(optionNum)
}else if(option == "all"){ } else if (option == "all") {
await executeoption(functions.length-1) // 15 = execute all fetchers await executeoption(functions.length - 1) // 15 = execute all fetchers
}else{ } else {
await whattodo(whattodoMessage, executeoption) await whattodo(whattodoMessage, executeoption)
} }
}else( } else (
await whattodo(whattodoMessage, executeoption) await whattodo(whattodoMessage, executeoption)
) )
} }

21
src/platforms/csetforetell-fetch.js
View File

@ -10,6 +10,8 @@ import { upsert } from "../utils/mongo-wrapper.js"
let htmlEndPoint = 'https://www.cset-foretell.com/questions?page=' let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
const DEBUG_MODE = "on"// "off" const DEBUG_MODE = "on"// "off"
const SLEEP_TIME_RANDOM=100//5000 // miliseconds
const SLEEP_TIME_EXTRA=0//1000
/* Support functions */ /* Support functions */
async function fetchPage(page, cookie) { async function fetchPage(page, cookie) {
@ -171,27 +173,28 @@ async function csetforetell_inner(cookie) {
while (!isEnd(response) && isSignedIn(response)) { while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n") let htmlLines = response.split("\n")
let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href=")) // let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
//let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/")) let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
// console.log(questionHrefs) // console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log(response) //console.log(response)
console.log(h4elements) console.log("questionHrefs: ")
console.log(questionHrefs)
} }
//console.log("") //console.log("")
//console.log("") //console.log("")
//console.log(h4elements) //console.log(h4elements)
for (let h4element of h4elements) { for (let questionHref of questionHrefs) {
//console.log(h4element) //console.log(h4element)
let h4elementSplit = h4element.split('"><span>') let elementSplit = questionHref.split('"><span>')
let url = h4elementSplit[0].split('<a href="')[1] let url = elementSplit[0].split('<a href="')[1]
let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "") let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "")
await sleep(1000 + Math.random() * 1000) // don't be as noticeable await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
try { try {
let moreinfo = await fetchStats(url, cookie) let moreinfo = await fetchStats(url, cookie)
@ -221,7 +224,7 @@ async function csetforetell_inner(cookie) {
//i=Number(i)+1 //i=Number(i)+1
console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers") console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
await sleep(5000 + Math.random() * 1000) // don't be as noticeable await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
try { try {
response = await fetchPage(i, cookie) response = await fetchPage(i, cookie)

76
src/platforms/manifoldmarkets-fetch.js Normal file
View File

@ -0,0 +1,76 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
/* Definitions */
let endpoint = 'https://us-central1-mantic-markets.cloudfunctions.net/markets'
/* Support functions */
async function fetchData() {
let response = await axios({
url: endpoint,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
})
.then(response => response.data)
// console.log(response)
return response
}
async function processPredictions(predictions) {
let results = await predictions.map(prediction => {
let probability = prediction.probability
let options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
let result = ({
"title": prediction.question,
"url": prediction.url,
"platform": "Manifold Markets",
"description": prediction.description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Manifold Markets", ({
volume7days: prediction.volume7days,
volume24Hours: prediction.volume24Hours,
liquidity: prediction.pool
})),
"createdTime": prediction.createdTime,
"volume7Days": prediction.volume7Days,
"volume24Hours": prediction.volume24Hours,
"liquidity": prediction.pool
}
})
return result
})
let unresolvedResults = results.filter(result => !result.isResolved)
return unresolvedResults //resultsProcessed
}
/* Body */
export async function manifoldmarkets() {
let data = await fetchData()
let results = await processPredictions(data) // somehow needed
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "manifoldmarkets-questions")
console.log("Done")
}
//manifoldmarkets()

67
src/utils/doEverything.js
View File

@ -1,52 +1,53 @@
import {astralcodexten} from "../platforms/astralcodexten-fetch.js" import { astralcodexten } from "../platforms/astralcodexten-fetch.js"
import {betfair} from "../platforms/betfair-fetch.js" import { betfair } from "../platforms/betfair-fetch.js"
import {coupcast} from "../platforms/coupcast-fetch.js" import { coupcast } from "../platforms/coupcast-fetch.js"
import {csetforetell} from "../platforms/csetforetell-fetch.js" import { csetforetell } from "../platforms/csetforetell-fetch.js"
import {elicit} from "../platforms/elicit-fetch.js" import { elicit } from "../platforms/elicit-fetch.js"
import {estimize} from "../platforms/estimize-fetch.js" import { estimize } from "../platforms/estimize-fetch.js"
import {fantasyscotus} from "../platforms/fantasyscotus-fetch.js" import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js"
import {foretold} from "../platforms/foretold-fetch.js" import { foretold } from "../platforms/foretold-fetch.js"
import {goodjudgment} from "../platforms/goodjudgment-fetch.js" import { goodjudgment } from "../platforms/goodjudgment-fetch.js"
import {goodjudgmentopen} from "../platforms/goodjudmentopen-fetch.js" import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js"
import {hypermind} from "../platforms/hypermind-fetch.js" import { hypermind } from "../platforms/hypermind-fetch.js"
import {kalshi} from "../platforms/kalshi-fetch.js" import { kalshi } from "../platforms/kalshi-fetch.js"
import {ladbrokes} from "../platforms/ladbrokes-fetch.js" import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
import {metaculus} from "../platforms/metaculus-fetch.js" import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
import {omen} from "../platforms/omen-fetch.js" import { metaculus } from "../platforms/metaculus-fetch.js"
import {polymarket} from "../platforms/polymarket-fetch.js" import { omen } from "../platforms/omen-fetch.js"
import {predictit} from "../platforms/predictit-fetch.js" import { polymarket } from "../platforms/polymarket-fetch.js"
import {rootclaim} from "../platforms/rootclaim-fetch.js" import { predictit } from "../platforms/predictit-fetch.js"
import {smarkets} from "../platforms/smarkets-fetch.js" import { rootclaim } from "../platforms/rootclaim-fetch.js"
import {wildeford} from "../platforms/wildeford-fetch.js" import { smarkets } from "../platforms/smarkets-fetch.js"
import {williamhill} from "../platforms/williamhill-fetch.js" import { wildeford } from "../platforms/wildeford-fetch.js"
import {mergeEverything} from "./mergeEverything.js" import { williamhill } from "../platforms/williamhill-fetch.js"
import {updateHistory} from "./history/updateHistory.js" import { mergeEverything } from "./mergeEverything.js"
import {rebuildAlgoliaDatabase} from "./algolia.js" import { updateHistory } from "./history/updateHistory.js"
import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js" import { rebuildAlgoliaDatabase } from "./algolia.js"
import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData.js"
/* Do everything */ /* Do everything */
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
} }
export async function tryCatchTryAgain (fun) { export async function tryCatchTryAgain(fun) {
try{ try {
console.log("Initial try") console.log("Initial try")
await fun() await fun()
}catch (error) { } catch (error) {
sleep(10000) sleep(10000)
console.log("Second try") console.log("Second try")
console.log(error) console.log(error)
try{ try {
await fun() await fun()
}catch (error){ } catch (error) {
console.log(error) console.log(error)
} }
} }
} }
export async function doEverything(){ export async function doEverything() {
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
// Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku. // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
console.log("") console.log("")
@ -61,7 +62,7 @@ export async function doEverything(){
console.log("") console.log("")
console.log("") console.log("")
for(let fun of functions){ for (let fun of functions) {
console.log("") console.log("")
console.log("") console.log("")
console.log("****************************") console.log("****************************")

2
src/utils/downloadFrontpage.js
View File

@ -2,7 +2,7 @@ import fs from "fs"
import { mongoReadWithReadCredentials } from "./mongo-wrapper.js" import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json' let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-backend/data/frontpage.json'
let shuffle = (array) => { let shuffle = (array) => {
// https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array // https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
let currentIndex = array.length, randomIndex; let currentIndex = array.length, randomIndex;

4
src/utils/mergeEverything.js
View File

@ -1,14 +1,14 @@
import { mongoRead, upsert } from "./mongo-wrapper.js" import { mongoRead, upsert } from "./mongo-wrapper.js"
/* Merge everything */ /* Merge everything */
let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"] let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
let suffix = "-questions" let suffix = "-questions"
export async function mergeEverythingInner(){ export async function mergeEverythingInner(){
let merged = [] let merged = []
for(let set of sets){ for(let set of sets){
let json = await mongoRead(set+suffix) let json = await mongoRead(set+suffix)
console.log(`${set} has ${json.length} questions`) console.log(`${set} has ${json.length} questions\n`)
merged = merged.concat(json) merged = merged.concat(json)
} }
let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")})) let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")}))

12
src/utils/stars.js
View File

@ -156,6 +156,15 @@ function calculateStarsLadbrokes(data) {
return starsInteger return starsInteger
} }
function calculateStarsManifoldMarkets(data) {
let nuno = data => 2
let eli = (data) => null
let misha = (data) => null
let starsDecimal = average([nuno(data)]) //, eli(data), misha(data)])
let starsInteger = Math.round(starsDecimal)
return starsInteger
}
function calculateStarsMetaculus(data) { function calculateStarsMetaculus(data) {
let nuno = data => data.numforecasts > 300 ? 4 : (data.numforecasts > 100 ? 3 : 2) let nuno = data => data.numforecasts > 300 ? 4 : (data.numforecasts > 100 ? 3 : 2)
let eli = (data) => 3 let eli = (data) => 3
@ -278,6 +287,9 @@ export function calculateStars(platform, data) {
case "Ladbrokes": case "Ladbrokes":
stars = calculateStarsLadbrokes(data) stars = calculateStarsLadbrokes(data)
break; break;
case "Manifold Markets":
stars = calculateStarsManifoldMarkets(data)
break;
case "Metaculus": case "Metaculus":
stars = calculateStarsMetaculus(data) stars = calculateStarsMetaculus(data)
break; break;