diff --git a/src/platforms/hypermind-fetch.js b/src/platforms/hypermind-fetch.js index 054d0d8..0905d74 100644 --- a/src/platforms/hypermind-fetch.js +++ b/src/platforms/hypermind-fetch.js @@ -1,6 +1,8 @@ /* Imports */ import fs from 'fs' import axios from "axios" +import https from "https" +import fetch from "isomorphic-fetch" import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import toMarkdown from "../utils/toMarkdown.js" import { calculateStars } from "../utils/stars.js" @@ -8,104 +10,80 @@ import { upsert } from "../utils/mongo-wrapper.js" /* Definitions */ let hypermindEnpoint1 = 'https://predict.hypermind.com/dash/jsx.json' -String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } +let hypermindEnpoint2 = 'https://prod.hypermind.com/ngdp-jsx/jsx.json' +const insecureHttpsAgent = new https.Agent({ + rejectUnauthorized: false, // (NOTE: this will disable client verification) +}) /* Support Functions */ +String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } + function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } +/* Fetchers */ async function fetchHypermindData1(slug) { - let response = await axios("https://predict.hypermind.com/dash/jsx.json", { + let jsx = `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D` + // console.log(jsx) + let response = await await axios(hypermindEnpoint1, { "credentials": "omit", "headers": { - "User-Agent": "", - "Accept": "*/*", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8" + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Cache-Control": "max-age=0" }, "referrer": `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`, - "data": `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`, + "data": jsx, "method": "POST", - "mode": "cors" - }).then(resp => resp.data[0].questions) - return response -} - -async function fetchHypermindData2(cookie) { - let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { - "credentials": "include", - "headers": { - "User-Agent": "", - "Accept": "*/*", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/json; charset=UTF-8", - "Cookie": cookie - }, - "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", - "data": `[["showcase","queryIFPs",{"query":{"showcaseOnly":true},"fmt":{"stats":true,"crowdFcstHist":true}}]]`, - "method": "POST", - "mode": "cors" - }).then(resp => resp.data[0]) - return response -} - -async function fetchHypermindData3(cookie) { - let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { - "credentials": "include", - "headers": { - "User-Agent": "", - "Accept": "*/*", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/json; charset=UTF-8", - "Cookie": cookie - }, - "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", - "data": `[["showcase","getShowcase",{"showcase":"Covid19","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`, - "method": "POST", - "mode": "cors" - }) - .then(resp => resp.data[0].items) - .then(items => items.filter(item => item.type == "IFP")) - .then(items => items.map(item => item.IFP)) + "mode": "cors", + httpsAgent: insecureHttpsAgent + }).then(response => response.data[0].questions) //console.log(response) return response } -async function fetchHypermindData4(cookie) { - let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { +async function fetchHypermindDataShowcases(slug, cookie) { + let response = await axios(hypermindEnpoint2, { "credentials": "include", "headers": { "User-Agent": "", "Accept": "*/*", "Accept-Language": "en-US,en;q=0.5", "Content-Type": "application/json; charset=UTF-8", - "Cookie": cookie + //"Cookie": cookie }, "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", - "data": `[["showcase","getShowcase",{"showcase":"AI2023","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`, + "data": `[["showcase","getShowcase",{"showcase":"${slug}","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`, "method": "POST", - "mode": "cors" - }) - .then(resp => resp.data[0].items) + "mode": "cors", + httpsAgent: insecureHttpsAgent + }).then(resp => resp.data[0].items) .then(items => items.filter(item => item.type == "IFP")) .then(items => items.map(item => item.IFP)) - //console.log(response) return response } /* Body */ async function hypermind_inner(cookie) { - let slugs = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"] + // Prediction markets; dashboard type one. + // https://predict.hypermind.com/dash/dash/dash.html?list=SLUG + // e.g., https://predict.hypermind.com/dash/dash/dash.html?list=POL + let slugs1 = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"] let results1 = [] - for (let slug of slugs) { + for (let slug of slugs1) { console.log(slug) await sleep(2000 + Math.random() * 2000) let result = await fetchHypermindData1(slug) - //console.log(result) - let objs = result.map(res => { let descriptionraw = res.props.details let descriptionprocessed1 = descriptionraw.split("%%fr")[0] @@ -135,72 +113,40 @@ async function hypermind_inner(cookie) { } }) }) + // console.log(objs) results1.push(...objs) } - console.log("GDP") - await sleep(1000 + Math.random() * 1000) - let results2 = await fetchHypermindData2(cookie) - let results2processed = results2.map(res => { - //console.log(res.props.details) - let descriptionraw = res.props.details.split("
")[0] - let descriptionprocessed1 = toMarkdown(descriptionraw) - let descriptionprocessed2 = descriptionprocessed1.split("![image]")[0] - let description = descriptionprocessed2 - //console.log(description) - return ({ - "title": res.props.title, - "url": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html", - "platform": "Hypermind", - "description": description, - "options": [], - "qualityindicators": { - "stars": calculateStars("Hypermind", ({})), - } + // Hypermind panelists and competitors; dashboard type two: "showcase" + // https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=SLUG + // E.g., https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=AI2023 + let slugs2 = [ "Covid19", "DOSES", "H5N8", "NGDP", "JSAI", "AI2023", "AI2030"] + let results2 = [] + for(let slug of slugs2){ + console.log(slug) + await sleep(1000 + Math.random() * 1000) + let response = await fetchHypermindDataShowcases(slug) + let objs = response.map(result => { + let descriptionraw = result.props.details.split("
")[0] + let descriptionprocessed1 = toMarkdown(descriptionraw) + let descriptionprocessed2 = descriptionprocessed1.split("![image]")[0] + let description = descriptionprocessed2 + return ({ + "title": result.props.title, + "url": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html", + "platform": "Hypermind", + "description": description, + "options": [], + "qualityindicators": { + "stars": calculateStars("Hypermind", ({})), + } + }) }) - }) + // console.log(objs) + results2.push(...objs) + } - - console.log("COVID-19 OpenPhil") - await sleep(1000 + Math.random() * 1000) - let results3 = await fetchHypermindData3(cookie) - // console.log(results3) - let results3processed = results3.map(res => { - let descriptionraw = res.props.details.split("
")[0] - let descriptionprocessed1 = toMarkdown(descriptionraw) - let descriptionprocessed2 = descriptionprocessed1.split("![image]")[0] - let description = descriptionprocessed2 - return ({ - "title": res.props.title, - "url": "https://prod.hypermind.com/ngdp/en/showcase2/showcase.html?sc=Covid19", - "platform": "Hypermind", - "description": description, - "options": [], - "qualityindicators": { - "stars": calculateStars("Hypermind", ({})), - } - }) - }) - - console.log("AI in 2023") - await sleep(1000 + Math.random() * 1000) - let results4 = await fetchHypermindData4(cookie) - // console.log(results4) - let results4processed = results2.map(res => { - let description = res.props.details - return ({ - "title": res.props.title, - "url": "https://prod.hypermind.com/ngdp/en/showcase2/showcase.html?sc=AI2023", - "platform": "Hypermind", - "description": description, - "options": [], - "qualityindicators": { - "stars": calculateStars("Hypermind", ({})), - } - }) - }) - - let resultsTotal = [...results1, ...results2processed, ...results3processed, ...results4processed] + let resultsTotal = [...results1, ...results2] let distinctTitles = [] let resultsTotalUnique = [] @@ -210,7 +156,8 @@ async function hypermind_inner(cookie) { distinctTitles.push(result["title"]) } } - //console.log(resultsTotalUnique) + // console.log(resultsTotal) + console.log(resultsTotalUnique) // let string = JSON.stringify(resultsTotalUnique, null, 2) // fs.writeFileSync('./data/hypermind-questions.json', string); await upsert(resultsTotalUnique, "hypermind-questions") diff --git a/src/utils/doEverything.js b/src/utils/doEverything.js index 2b34cd5..8f4b464 100644 --- a/src/utils/doEverything.js +++ b/src/utils/doEverything.js @@ -39,6 +39,7 @@ export async function tryCatchTryAgain (fun) { } } } + export async function doEverything(){ let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.