quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Added missing Hypermind tournament

Browse Source 
- Also made the Hypermind code more maintainable.
This commit is contained in:
NunoSempere 2021-08-11 20:21:48 +02:00
parent ec28944178
commit 6144675a41
2 changed files with 70 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
src/platforms/hypermind-fetch.js
View File

@ -1,6 +1,8 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from 'fs'
import axios from "axios" import axios from "axios"
import https from "https"
import fetch from "isomorphic-fetch"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js"
@ -8,104 +10,80 @@ import { upsert } from "../utils/mongo-wrapper.js"
/* Definitions */ /* Definitions */
let hypermindEnpoint1 = 'https://predict.hypermind.com/dash/jsx.json' let hypermindEnpoint1 = 'https://predict.hypermind.com/dash/jsx.json'
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } let hypermindEnpoint2 = 'https://prod.hypermind.com/ngdp-jsx/jsx.json'
const insecureHttpsAgent = new https.Agent({
rejectUnauthorized: false, // (NOTE: this will disable client verification)
})
/* Support Functions */ /* Support Functions */
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
} }
/* Fetchers */
async function fetchHypermindData1(slug) { async function fetchHypermindData1(slug) {
let response = await axios("https://predict.hypermind.com/dash/jsx.json", { let jsx = `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`
// console.log(jsx)
let response = await await axios(hypermindEnpoint1, {
"credentials": "omit", "credentials": "omit",
"headers": { "headers": {
"User-Agent": "", "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
"Accept": "*/*", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5", "Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8" "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Cache-Control": "max-age=0"
}, },
"referrer": `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`, "referrer": `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`,
"data": `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`, "data": jsx,
"method": "POST", "method": "POST",
"mode": "cors" "mode": "cors",
}).then(resp => resp.data[0].questions) httpsAgent: insecureHttpsAgent
return response }).then(response => response.data[0].questions)
}
async function fetchHypermindData2(cookie) {
let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
"credentials": "include",
"headers": {
"User-Agent": "",
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8",
"Cookie": cookie
},
"referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
"data": `[["showcase","queryIFPs",{"query":{"showcaseOnly":true},"fmt":{"stats":true,"crowdFcstHist":true}}]]`,
"method": "POST",
"mode": "cors"
}).then(resp => resp.data[0])
return response
}
async function fetchHypermindData3(cookie) {
let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
"credentials": "include",
"headers": {
"User-Agent": "",
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8",
"Cookie": cookie
},
"referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
"data": `[["showcase","getShowcase",{"showcase":"Covid19","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`,
"method": "POST",
"mode": "cors"
})
.then(resp => resp.data[0].items)
.then(items => items.filter(item => item.type == "IFP"))
.then(items => items.map(item => item.IFP))
//console.log(response) //console.log(response)
return response return response
} }
async function fetchHypermindData4(cookie) { async function fetchHypermindDataShowcases(slug, cookie) {
let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { let response = await axios(hypermindEnpoint2, {
"credentials": "include", "credentials": "include",
"headers": { "headers": {
"User-Agent": "", "User-Agent": "",
"Accept": "*/*", "Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5", "Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8", "Content-Type": "application/json; charset=UTF-8",
"Cookie": cookie //"Cookie": cookie
}, },
"referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
"data": `[["showcase","getShowcase",{"showcase":"AI2023","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`, "data": `[["showcase","getShowcase",{"showcase":"${slug}","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`,
"method": "POST", "method": "POST",
"mode": "cors" "mode": "cors",
}) httpsAgent: insecureHttpsAgent
.then(resp => resp.data[0].items) }).then(resp => resp.data[0].items)
.then(items => items.filter(item => item.type == "IFP")) .then(items => items.filter(item => item.type == "IFP"))
.then(items => items.map(item => item.IFP)) .then(items => items.map(item => item.IFP))
//console.log(response)
return response return response
} }
/* Body */ /* Body */
async function hypermind_inner(cookie) { async function hypermind_inner(cookie) {
let slugs = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]
// Prediction markets; dashboard type one.
// https://predict.hypermind.com/dash/dash/dash.html?list=SLUG
// e.g., https://predict.hypermind.com/dash/dash/dash.html?list=POL
let slugs1 = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]
let results1 = [] let results1 = []
for (let slug of slugs) { for (let slug of slugs1) {
console.log(slug) console.log(slug)
await sleep(2000 + Math.random() * 2000) await sleep(2000 + Math.random() * 2000)
let result = await fetchHypermindData1(slug) let result = await fetchHypermindData1(slug)
//console.log(result)
let objs = result.map(res => { let objs = result.map(res => {
let descriptionraw = res.props.details let descriptionraw = res.props.details
let descriptionprocessed1 = descriptionraw.split("%%fr")[0] let descriptionprocessed1 = descriptionraw.split("%%fr")[0]
@ -135,72 +113,40 @@ async function hypermind_inner(cookie) {
} }
}) })
}) })
// console.log(objs)
results1.push(...objs) results1.push(...objs)
} }
console.log("GDP") // Hypermind panelists and competitors; dashboard type two: "showcase"
await sleep(1000 + Math.random() * 1000) // https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=SLUG
let results2 = await fetchHypermindData2(cookie) // E.g., https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=AI2023
let results2processed = results2.map(res => { let slugs2 = [ "Covid19", "DOSES", "H5N8", "NGDP", "JSAI", "AI2023", "AI2030"]
//console.log(res.props.details) let results2 = []
let descriptionraw = res.props.details.split("<hr size=1>")[0] for(let slug of slugs2){
let descriptionprocessed1 = toMarkdown(descriptionraw) console.log(slug)
let descriptionprocessed2 = descriptionprocessed1.split("![image]")[0] await sleep(1000 + Math.random() * 1000)
let description = descriptionprocessed2 let response = await fetchHypermindDataShowcases(slug)
//console.log(description) let objs = response.map(result => {
return ({ let descriptionraw = result.props.details.split("<hr size=1>")[0]
"title": res.props.title, let descriptionprocessed1 = toMarkdown(descriptionraw)
"url": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html", let descriptionprocessed2 = descriptionprocessed1.split("![image]")[0]
"platform": "Hypermind", let description = descriptionprocessed2
"description": description, return ({
"options": [], "title": result.props.title,
"qualityindicators": { "url": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html",
"stars": calculateStars("Hypermind", ({})), "platform": "Hypermind",
} "description": description,
"options": [],
"qualityindicators": {
"stars": calculateStars("Hypermind", ({})),
}
})
}) })
}) // console.log(objs)
results2.push(...objs)
}
let resultsTotal = [...results1, ...results2]
console.log("COVID-19 OpenPhil")
await sleep(1000 + Math.random() * 1000)
let results3 = await fetchHypermindData3(cookie)
// console.log(results3)
let results3processed = results3.map(res => {
let descriptionraw = res.props.details.split("<hr size=1>")[0]
let descriptionprocessed1 = toMarkdown(descriptionraw)
let descriptionprocessed2 = descriptionprocessed1.split("![image]")[0]
let description = descriptionprocessed2
return ({
"title": res.props.title,
"url": "https://prod.hypermind.com/ngdp/en/showcase2/showcase.html?sc=Covid19",
"platform": "Hypermind",
"description": description,
"options": [],
"qualityindicators": {
"stars": calculateStars("Hypermind", ({})),
}
})
})
console.log("AI in 2023")
await sleep(1000 + Math.random() * 1000)
let results4 = await fetchHypermindData4(cookie)
// console.log(results4)
let results4processed = results2.map(res => {
let description = res.props.details
return ({
"title": res.props.title,
"url": "https://prod.hypermind.com/ngdp/en/showcase2/showcase.html?sc=AI2023",
"platform": "Hypermind",
"description": description,
"options": [],
"qualityindicators": {
"stars": calculateStars("Hypermind", ({})),
}
})
})
let resultsTotal = [...results1, ...results2processed, ...results3processed, ...results4processed]
let distinctTitles = [] let distinctTitles = []
let resultsTotalUnique = [] let resultsTotalUnique = []
@ -210,7 +156,8 @@ async function hypermind_inner(cookie) {
distinctTitles.push(result["title"]) distinctTitles.push(result["title"])
} }
} }
//console.log(resultsTotalUnique) // console.log(resultsTotal)
console.log(resultsTotalUnique)
// let string = JSON.stringify(resultsTotalUnique, null, 2) // let string = JSON.stringify(resultsTotalUnique, null, 2)
// fs.writeFileSync('./data/hypermind-questions.json', string); // fs.writeFileSync('./data/hypermind-questions.json', string);
await upsert(resultsTotalUnique, "hypermind-questions") await upsert(resultsTotalUnique, "hypermind-questions")

1
src/utils/doEverything.js
View File

@ -39,6 +39,7 @@ export async function tryCatchTryAgain (fun) {
} }
} }
} }
export async function doEverything(){ export async function doEverything(){
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
// Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku. // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.