diff --git a/src/platforms/betfair-fetch.js b/src/platforms/betfair-fetch.js index 4d715fb..814e2e7 100644 --- a/src/platforms/betfair-fetch.js +++ b/src/platforms/betfair-fetch.js @@ -1,13 +1,13 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import https from 'https'; -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import https from "https"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let endpoint = process.env.SECRET_BETFAIR_ENDPOINT +let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; /* Utilities */ let arraysEqual = (a, b) => { @@ -24,74 +24,81 @@ let arraysEqual = (a, b) => { if (a[i] !== b[i]) return false; } return true; -} +}; let mergeRunners = (runnerCatalog, runnerBook) => { - let keys = Object.keys(runnerCatalog) - let result = [] - for(let key of keys){ - result.push({...runnerCatalog[key], ...runnerBook[key]}) + let keys = Object.keys(runnerCatalog); + let result = []; + for (let key of keys) { + result.push({ ...runnerCatalog[key], ...runnerBook[key] }); } - return result -} + return result; +}; /* Support functions */ async function fetchPredictions() { - const agent = new https.Agent({ - rejectUnauthorized: false + const agent = new https.Agent({ + rejectUnauthorized: false, }); let response = await axios({ url: endpoint, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - }), - httpsAgent: agent - }).then(response => response.data) - - return response + method: "GET", + headers: { + "Content-Type": "text/html", + }, + httpsAgent: agent, + }).then((response) => response.data); + + return response; } -async function whipIntoShape(data){ - - - let catalogues = data.market_catalogues - let books = data.market_books - let keys1 = Object.keys(catalogues).sort() - let keys2 = Object.keys(books).sort() +async function whipIntoShape(data) { + let catalogues = data.market_catalogues; + let books = data.market_books; + let keys1 = Object.keys(catalogues).sort(); + let keys2 = Object.keys(books).sort(); // console.log(keys1) // console.log(keys2) - let results = [] - if(!arraysEqual(keys1, keys2)){ - throw new Error("Betfair: Error in endpoint; Betfair catalogues and books do not match") - }else{ - for(let key of keys1){ - results.push({...catalogues[key], ...books[key], options: mergeRunners(catalogues[key].runners, books[key].runners)}) + let results = []; + if (!arraysEqual(keys1, keys2)) { + throw new Error( + "Betfair: Error in endpoint; Betfair catalogues and books do not match" + ); + } else { + for (let key of keys1) { + results.push({ + ...catalogues[key], + ...books[key], + options: mergeRunners(catalogues[key].runners, books[key].runners), + }); } } - return results + return results; } async function processPredictions(data) { - let predictions = await whipIntoShape(data) + let predictions = await whipIntoShape(data); // console.log(JSON.stringify(predictions, null, 4)) - let results = predictions.map(prediction => { + let results = predictions.map((prediction) => { /* if(Math.floor(Math.random() * 10) % 20 ==0){ console.log(JSON.stringify(prediction, null, 4)) } */ - - let normalizationFactor = (prediction.options - .filter(option => option.status == "ACTIVE" && option.totalMatched > 0) - .map(option => option.lastPriceTraded)) - .map(x => 1/x) - .reduce((a, b) => a + b, 0) + let id = `betfair-${prediction.marketId}`; + let normalizationFactor = prediction.options + .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0) + .map((option) => option.lastPriceTraded) + .map((x) => 1 / x) + .reduce((a, b) => a + b, 0); let options = prediction.options - .filter(option => option.status == "ACTIVE" && option.totalMatched > 0) - .map(option => ({ - "name": option.runnerName, - "probability": option.lastPriceTraded !=0 ? (1/option.lastPriceTraded)/normalizationFactor : 0, // https://www.aceodds.com/bet-calculator/odds-converter.html - "type": "PROBABILITY" - })) + .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0) + .map((option) => ({ + name: option.runnerName, + probability: + option.lastPriceTraded != 0 + ? 1 / option.lastPriceTraded / normalizationFactor + : 0, // https://www.aceodds.com/bet-calculator/odds-converter.html + type: "PROBABILITY", + })); // console.log(prediction.options) @@ -102,41 +109,42 @@ async function processPredictions(data) { .replace(//g, " ") .replace(/<\/b>/g, " ") .replace(/\n/g, " ") - .trim() - if(rules == undefined){ + .trim(); + if (rules == undefined) { // console.log(prediction.description) } - let title = rules.split("? ")[0] + "?" - let description = rules.split("? ")[1].trim() - if(title.includes("of the named")){ - title = prediction.marketName + ": "+ title + let title = rules.split("? ")[0] + "?"; + let description = rules.split("? ")[1].trim(); + if (title.includes("of the named")) { + title = prediction.marketName + ": " + title; } - let result = ({ - "title": title, - "url": `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`, - "platform": "Betfair", - "description": description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "stars": calculateStars("Betfair", ({volume: prediction.totalMatched})), - "volume": prediction.totalMatched, - } - }) - return result - }) - return results //resultsProcessed + let result = { + id: id, + title: title, + url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`, + platform: "Betfair", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Betfair", { volume: prediction.totalMatched }), + volume: prediction.totalMatched, + }, + }; + return result; + }); + return results; //resultsProcessed } /* Body */ export async function betfair() { - let data = await fetchPredictions() - let results = await processPredictions(data) // somehow needed + let data = await fetchPredictions(); + let results = await processPredictions(data); // somehow needed // console.log(results.map(result => ({title: result.title, description: result.description}))) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('polyprediction-questions.json', string); - await upsert(results, "betfair-questions") - console.log("Done") + await upsert(results, "betfair-questions"); + console.log("Done"); } // betfair() diff --git a/src/platforms/deprecated/ladbrokes-fetch.js b/src/platforms/deprecated/ladbrokes-fetch.js new file mode 100644 index 0000000..da091f5 --- /dev/null +++ b/src/platforms/deprecated/ladbrokes-fetch.js @@ -0,0 +1,146 @@ +/* Imports */ +import axios from "axios"; +import fs from "fs"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; + +/* Definitions */ +let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`; +let enpointDrillDown = (id) => + `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`; + +//
response.data); + return response; +} + +let processResults = async (json) => { + let results = []; + let children = json.SSResponse.children; + children.pop(); + let ids = children.map((child) => child.event.id); + let markets = []; + for (let id of ids) { + let marketsContainer = await fetchUrl(enpointDrillDown(id)); + let marketsObj = marketsContainer.SSResponse.children[0].event; + let newMarkets = marketsObj.children; + newMarkets = newMarkets.map((market) => ({ + ...market.market, + parent: marketsObj.name, + })); + markets.push(...newMarkets); + } + let normalMarkets = markets.filter( + (market) => !market.name.includes("Specials") + ); + //console.log(normalMarkets) + + for (let normalMarket of normalMarkets) { + let title = normalMarket.parent + ": " + normalMarket.name; + title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge + let options = normalMarket.children.map((child) => { + let name = child.outcome.name; + + let priceData = child.outcome.children[0].price; + let priceDecimal = Number(priceData.priceDec); + let probability = 1 / priceDecimal; + let option = { + name: name, + probability: probability, + type: "PROBABILITY", + }; + return option; + }); + // normalize probabilities + let totalValue = options + .map((element) => Number(element.probability)) + .reduce((a, b) => a + b, 0); + options = options.map((element) => ({ + ...element, + probability: Number(element.probability) / totalValue, + })); + + // Filter very unlikely probabilities: Not here, but on the front end + // options = options.filter(element => element.probability > 0.02) + + let obj = { + title: title, + url: "https://sports.ladbrokes.com/sport/politics/outrights", + platform: "Ladbrokes", + description: "", + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Ladbrokes", {}), + }, + }; + results.push(obj); + } + + let specialMarkets = markets.filter((market) => + market.name.includes("Specials") + ); + for (let specialMarket of specialMarkets) { + //let title = specialMarket.parent + ": " + specialMarket.name + //console.log(title) + specialMarket.children.forEach((child) => { + let name = specialMarket.parent.includes("Specials") + ? child.outcome.name + : specialMarket.parent + ": " + child.outcome.name; + name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge + let priceData = child.outcome.children[0].price; + let priceDecimal = Number(priceData.priceDec); + let probability = 1 / priceDecimal; + let obj = { + title: name, + url: "https://sports.ladbrokes.com/sport/politics/outrights", + platform: "LadBrokes", + options: [ + { + name: "Yes", + probability: probability, + type: "PROBABILITY", + }, + { + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ], + qualityindicators: { + stars: calculateStars("Ladbrokes", {}), + }, + }; + results.push(obj); + }); + } + return results; +}; + +/* Body */ +export async function ladbrokes() { + let response = await fetchUrl(endpointPolitics); + let results = await processResults(response); + // console.log(results) + // let string = JSON.stringify(results, null, 2) + // fs.writeFileSync('./data/ladbrokes-questions.json', string); + await upsert(results, "ladbrokes-questions"); + console.log("Done"); +} +//ladbrokes() diff --git a/src/platforms/williamhill-fetch.js b/src/platforms/deprecated/williamhill-fetch.js similarity index 100% rename from src/platforms/williamhill-fetch.js rename to src/platforms/deprecated/williamhill-fetch.js diff --git a/src/platforms/example-fetch.js b/src/platforms/example-fetch.js index 36a7624..4132772 100644 --- a/src/platforms/example-fetch.js +++ b/src/platforms/example-fetch.js @@ -1,70 +1,70 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let endpoint = 'https://example.com/' +let endpoint = "https://example.com/"; /* Support functions */ async function fetchData() { let response = await axios({ url: endpoint, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - }), - }) - .then(response => response.data) + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((response) => response.data); // console.log(response) - return response + return response; } async function processPredictions(predictions) { - let results = await predictions.map(prediction => { - let probability = prediction.probability + let results = await predictions.map((prediction) => { + let id = `platform-${prediction.id}`; + let probability = prediction.probability; let options = [ { - "name": "Yes", - "probability": probability, - "type": "PROBABILITY" + name: "Yes", + probability: probability, + type: "PROBABILITY", }, { - "name": "No", - "probability": 1 - probability, - "type": "PROBABILITY" - } - ] - let result = ({ - "title": prediction.title, - "url": `https://example.com`, - "platform": "Example", - "description": prediction.description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "stars": calculateStars("Example", ({some: somex, factors: factors})), - "other": prediction.otherx, - "indicators": prediction.indicatorx - } - }) - return result - }) - return results //resultsProcessed + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; + let result = { + title: prediction.title, + url: `https://example.com`, + platform: "Example", + description: prediction.description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Example", { some: somex, factors: factors }), + other: prediction.otherx, + indicators: prediction.indicatorx, + }, + }; + return result; + }); + return results; //resultsProcessed } /* Body */ export async function example() { - let data = await fetchData() - let results = await processPredictions(data) // somehow needed + let data = await fetchData(); + let results = await processPredictions(data); // somehow needed // console.log(results) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('polyprediction-questions.json', string); - await upsert(results, "example-questions") - console.log("Done") + await upsert(results, "example-questions"); + console.log("Done"); } //example() diff --git a/src/platforms/fantasyscotus-fetch.js b/src/platforms/fantasyscotus-fetch.js index 75e0bc6..723c66d 100644 --- a/src/platforms/fantasyscotus-fetch.js +++ b/src/platforms/fantasyscotus-fetch.js @@ -1,113 +1,124 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import { calculateStars } from "../utils/stars.js" -import { upsert } from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let unixtime = new Date() .getTime() -let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=0&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=12&_=${unixtime}` +let unixtime = new Date().getTime(); +let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=0&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=12&_=${unixtime}`; async function fetchData() { let response = await axios({ - "url": endpoint, - "credentials": "omit", - "headers": { - "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", - "Accept": "application/json, text/javascript, */*; q=0.01", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/x-www-form-urlencoded", - "X-Requested-With": "XMLHttpRequest" - }, - "referrer": "https://fantasyscotus.net/case/list/", - "method": "GET", - "mode": "cors" - }) - .then(res => res.data) + url: endpoint, + credentials: "omit", + headers: { + "User-Agent": + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", + Accept: "application/json, text/javascript, */*; q=0.01", + "Accept-Language": "en-US,en;q=0.5", + "Content-Type": "application/x-www-form-urlencoded", + "X-Requested-With": "XMLHttpRequest", + }, + referrer: "https://fantasyscotus.net/case/list/", + method: "GET", + mode: "cors", + }).then((res) => res.data); //console.log(response) - return response + return response; } async function getPredictionsData(caseUrl) { - let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}` + let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`; //console.log(newCaseUrl) let predictions = await axios({ - "url": newCaseUrl, - "credentials": "include", - "headers": { - "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", - "Accept": "application/json, text/javascript, */*; q=0.01", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/x-www-form-urlencoded", - "X-Requested-With": "XMLHttpRequest" - }, - "referrer": newCaseUrl, - "method": "GET", - "mode": "cors" - }) - .then(res => res.data) - - let predictionsAffirm = predictions.filter(prediction => prediction.percent_affirm > 50) + url: newCaseUrl, + credentials: "include", + headers: { + "User-Agent": + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", + Accept: "application/json, text/javascript, */*; q=0.01", + "Accept-Language": "en-US,en;q=0.5", + "Content-Type": "application/x-www-form-urlencoded", + "X-Requested-With": "XMLHttpRequest", + }, + referrer: newCaseUrl, + method: "GET", + mode: "cors", + }).then((res) => res.data); + + let predictionsAffirm = predictions.filter( + (prediction) => prediction.percent_affirm > 50 + ); //console.log(predictions) //console.log(predictionsAffirm.length/predictions.length) - - return ({ + + return { numAffirm: predictionsAffirm.length, - proportionAffirm: predictionsAffirm.length/predictions.length, - numForecasts: predictions.length - }) + proportionAffirm: predictionsAffirm.length / predictions.length, + numForecasts: predictions.length, + }; } async function processData(data) { - let events = data.object_list - let historicalPercentageCorrect = data.stats.pcnt_correct - let historicalProbabilityCorrect = Number(historicalPercentageCorrect.replace("%",""))/100 - let results = [] - for(let event of events){ - if(event.accuracy == ''){ // if the thing hasn't already resolved - let predictionData = await getPredictionsData(event.docket_url) - let pAffirm = predictionData.proportionAffirm + let events = data.object_list; + let historicalPercentageCorrect = data.stats.pcnt_correct; + let historicalProbabilityCorrect = + Number(historicalPercentageCorrect.replace("%", "")) / 100; + let results = []; + for (let event of events) { + if (event.accuracy == "") { + let id = `fantasyscotus-${event.id}`; + // if the thing hasn't already resolved + let predictionData = await getPredictionsData(event.docket_url); + let pAffirm = predictionData.proportionAffirm; //let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect - let eventObject = ({ - "title": `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`, - "url": `https://fantasyscotus.net/user-predictions${event.docket_url}`, - "platform": "FantasySCOTUS", - "description": `${(pAffirm*100).toFixed(2)}% (${predictionData.numAffirm} out of ${predictionData.numForecasts}) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of ${event.prediction}. Historically, FantasySCOTUS has chosen the correct side ${historicalPercentageCorrect} of the time.`, - "options": [ - { - "name": "Yes", - "probability": pAffirm, - "type": "PROBABILITY" - }, - { - "name": "No", - "probability": 1-pAffirm, - "type": "PROBABILITY" - } - ], - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": Number(predictionData.numForecasts), - "stars": calculateStars("FantasySCOTUS", ({})) - } - }) + let eventObject = { + id: id, + title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`, + url: `https://fantasyscotus.net/user-predictions${event.docket_url}`, + platform: "FantasySCOTUS", + description: `${(pAffirm * 100).toFixed(2)}% (${ + predictionData.numAffirm + } out of ${ + predictionData.numForecasts + }) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of ${ + event.prediction + }. Historically, FantasySCOTUS has chosen the correct side ${historicalPercentageCorrect} of the time.`, + options: [ + { + name: "Yes", + probability: pAffirm, + type: "PROBABILITY", + }, + { + name: "No", + probability: 1 - pAffirm, + type: "PROBABILITY", + }, + ], + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Number(predictionData.numForecasts), + stars: calculateStars("FantasySCOTUS", {}), + }, + }; // console.log(eventObject) - results.push(eventObject) + results.push(eventObject); } - } - return results -} + return results; +} /* Body */ export async function fantasyscotus() { - let rawData = await fetchData() - let results = await processData(rawData) + let rawData = await fetchData(); + let results = await processData(rawData); //console.log(results) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/fantasyscotus-questions.json', string); - await upsert(results, "fantasyscotus-questions") - console.log("Done") + await upsert(results, "fantasyscotus-questions"); + console.log("Done"); } //fantasyscotus() diff --git a/src/platforms/foretold-fetch.js b/src/platforms/foretold-fetch.js index b86afe2..ae76da6 100644 --- a/src/platforms/foretold-fetch.js +++ b/src/platforms/foretold-fetch.js @@ -1,20 +1,26 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import { calculateStars } from "../utils/stars.js" -import { upsert } from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let graphQLendpoint = "https://api.foretold.io/graphql" -let highQualityCommunities = ["0104d8e8-07e4-464b-8b32-74ef22b49f21", "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", "cf663021-f87f-4632-ad82-962d889a2d39", "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", "b2412a1d-0aa4-4e37-a12a-0aca9e440a96"] +let graphQLendpoint = "https://api.foretold.io/graphql"; +let highQualityCommunities = [ + "0104d8e8-07e4-464b-8b32-74ef22b49f21", + "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", + "cf663021-f87f-4632-ad82-962d889a2d39", + "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", + "b2412a1d-0aa4-4e37-a12a-0aca9e440a96", +]; /* Support functions */ async function fetchAllCommunityQuestions(communityId) { let response = await axios({ url: graphQLendpoint, - method: 'POST', - headers: ({ 'Content-Type': 'application/json' }), - data: JSON.stringify(({ + method: "POST", + headers: { "Content-Type": "application/json" }, + data: JSON.stringify({ query: ` query { measurables( @@ -38,62 +44,64 @@ async function fetchAllCommunityQuestions(communityId) { } } } - ` - })), + `, + }), }) - .then(res => res.data) - .then(res => res.data.measurables.edges) + .then((res) => res.data) + .then((res) => res.data.measurables.edges); //console.log(response) - return response + return response; } /* Body */ -export async function foretold(){ - let results = [] - for(let community of highQualityCommunities){ - let questions = await fetchAllCommunityQuestions(community) - questions = questions.map(question => question.node) - questions = questions.filter(question => question.previousAggregate) // Questions without any predictions - questions.forEach(question => { - let options = [] - if(question.valueType == "PERCENTAGE"){ - let probability = question.previousAggregate.value.percentage +export async function foretold() { + let results = []; + for (let community of highQualityCommunities) { + let questions = await fetchAllCommunityQuestions(community); + questions = questions.map((question) => question.node); + questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions + questions.forEach((question) => { + let id = `foretold-${question.id}`; + let options = []; + if (question.valueType == "PERCENTAGE") { + let probability = question.previousAggregate.value.percentage; options = [ { - "name": "Yes", - "probability": probability/100, - "type": "PROBABILITY" + name: "Yes", + probability: probability / 100, + type: "PROBABILITY", }, { - "name": "No", - "probability": 1-probability/100, - "type": "PROBABILITY" - } - ] + name: "No", + probability: 1 - probability / 100, + type: "PROBABILITY", + }, + ]; } let result = { - "title": question.name, - "url": `https://www.foretold.io/c/${community}/m/${question.id}`, - "platform": "Foretold", - "description": "", - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": Math.floor(Number(question.measurementCount) / 2), - "stars": calculateStars("Foretold", ({ })) - } - /*liquidity: liquidity.toFixed(2), + id: id, + title: question.name, + url: `https://www.foretold.io/c/${community}/m/${question.id}`, + platform: "Foretold", + description: "", + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Math.floor(Number(question.measurementCount) / 2), + stars: calculateStars("Foretold", {}), + }, + /*liquidity: liquidity.toFixed(2), tradevolume: tradevolume.toFixed(2), address: obj.address*/ - } + }; // console.log(result) - results.push(result) - }) + results.push(result); + }); } // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/foretold-questions.json', string); - await upsert(results, "foretold-questions") - console.log("Done") + await upsert(results, "foretold-questions"); + console.log("Done"); } // foretold() diff --git a/src/platforms/givewellopenphil-fetch.js b/src/platforms/givewellopenphil-fetch.js index 20ac406..4839947 100644 --- a/src/platforms/givewellopenphil-fetch.js +++ b/src/platforms/givewellopenphil-fetch.js @@ -1,61 +1,69 @@ /* Imports */ -import fs from "fs" -import axios from "axios" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import { upsert } from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let locationData = "./data/" +let locationData = "./data/"; /* Support functions */ async function fetchPage(url) { let response = await axios({ url: url, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); //console.log(response) - return response + return response; } /* Body */ async function main() { - let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt") - let data = rawdata.toString().split("\n").filter(url => url != ""); + let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt"); + let data = rawdata + .toString() + .split("\n") + .filter((url) => url != ""); // console.log(data) - let results = [] + let results = []; for (let url of data) { // console.log(url) - let page = await fetchPage(url) + let page = await fetchPage(url); // Title - let titleraw = page.split('')[0] + let titleraw = page.split('')[0]; // Description - let internalforecasts = page.split(" section.includes("Internal forecast") || section.includes("internal forecast")) - let description = "

+ section.includes("Internal forecast") || + section.includes("internal forecast") + ); + let description = "

row["0"].includes("BACKGROUND:")) - .map((row) => row["0"]) - .map((text) => - text - .split("BACKGROUND:")[1] - .split("Examples of Superforecaster")[0] - .split("AT A GLANCE")[0] - .replaceAll("\n\n", "\n") - .split("\n") - .slice(3) - .join(" ") - .replaceAll(" ", "") - .replaceAll("
", "") - )[0]; - let options = table - .filter((row) => "4" in row) - .map((row) => ({ - name: row["2"].split('')[1].replace("", ""), - probability: Number(row["3"].split("%")[0]) / 100, - type: "PROBABILITY", - })); - let analysis = table.filter((row) => - row[0] ? row[0].toLowerCase().includes("commentary") : false - ); - // "Examples of Superforecaster Commentary" / Analysis - // The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list. - analysis = analysis ? analysis[0] : ""; - analysis = analysis ? analysis[0] : ""; - // console.log(analysis) - let standardObj = { - title: title, - url: endpoint, - platform: "Good Judgment", - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Good Judgment", {}), - }, - extra: { - superforecastercommentary: analysis || "", - }, - }; - if (standardObj.title != undefined) { + if (title != undefined) { + title = title.replaceAll("", ""); + let id = `goodjudgment-${hash(title)}`; + let description = table + .filter((row) => row["0"].includes("BACKGROUND:")) + .map((row) => row["0"]) + .map((text) => + text + .split("BACKGROUND:")[1] + .split("Examples of Superforecaster")[0] + .split("AT A GLANCE")[0] + .replaceAll("\n\n", "\n") + .split("\n") + .slice(3) + .join(" ") + .replaceAll(" ", "") + .replaceAll("
", "") + )[0]; + let options = table + .filter((row) => "4" in row) + .map((row) => ({ + name: row["2"] + .split('')[1] + .replace("", ""), + probability: Number(row["3"].split("%")[0]) / 100, + type: "PROBABILITY", + })); + let analysis = table.filter((row) => + row[0] ? row[0].toLowerCase().includes("commentary") : false + ); + // "Examples of Superforecaster Commentary" / Analysis + // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list. + analysis = analysis ? analysis[0] : ""; + analysis = analysis ? analysis[0] : ""; // not a duplicate + // console.log(analysis) + let standardObj = { + id: id, + title: title, + url: endpoint, + platform: "Good Judgment", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Good Judgment", {}), + }, + extra: { + superforecastercommentary: analysis || "", + }, + }; results.push(standardObj); } } diff --git a/src/platforms/goodjudmentopen-fetch.js b/src/platforms/goodjudmentopen-fetch.js index d9691f0..de118ce 100644 --- a/src/platforms/goodjudmentopen-fetch.js +++ b/src/platforms/goodjudmentopen-fetch.js @@ -1,211 +1,242 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" -import { Tabletojson } from "tabletojson" -import { calculateStars } from "../utils/stars.js" -import toMarkdown from "../utils/toMarkdown.js" -import { upsert } from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import { getCookie, applyIfCookieExists } from "../utils/getCookies.js"; +import { Tabletojson } from "tabletojson"; +import { calculateStars } from "../utils/stars.js"; +import toMarkdown from "../utils/toMarkdown.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let htmlEndPoint = 'https://www.gjopen.com/questions?page=' -let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"] -const DEBUG_MODE = "off" // "on" -const id = x => x +let htmlEndPoint = "https://www.gjopen.com/questions?page="; +let annoyingPromptUrls = [ + "https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", + "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters", + "https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread", + "https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", +]; +const DEBUG_MODE = "off"; // "on" +const id = (x) => x; /* Support functions */ async function fetchPage(page, cookie) { let response = await axios({ url: htmlEndPoint + page, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - 'Cookie': cookie - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + Cookie: cookie, + }, + }).then((res) => res.data); //console.log(response) - return response + return response; } async function fetchStats(questionUrl, cookie) { let response = await axios({ url: questionUrl + "/stats", - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - 'Cookie': cookie, - 'Referer': questionUrl, - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + Cookie: cookie, + Referer: questionUrl, + }, + }).then((res) => res.data); //console.log(response) // Is binary? - let isbinary = response.includes("binary?":true") + let isbinary = response.includes("binary?":true"); - let options = [] + let options = []; if (isbinary) { // Crowd percentage - let htmlElements = response.split("\n") - let h3Element = htmlElements.filter(str => str.includes("

"))[0] + let htmlElements = response.split("\n"); + let h3Element = htmlElements.filter((str) => str.includes("

"))[0]; // console.log(h3Element) - let crowdpercentage = h3Element.split(">")[1].split("<")[0] - let probability = Number(crowdpercentage.replace("%", "")) / 100 - options.push(({ - name: "Yes", - probability: probability, - type: "PROBABILITY" - }), ({ - name: "No", - probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans - type: "PROBABILITY" - })) + let crowdpercentage = h3Element.split(">")[1].split("<")[0]; + let probability = Number(crowdpercentage.replace("%", "")) / 100; + options.push( + { + name: "Yes", + probability: probability, + type: "PROBABILITY", + }, + { + name: "No", + probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans + type: "PROBABILITY", + } + ); } else { - let optionsHtmlElement = "" - let tablesAsJson = Tabletojson.convert(optionsHtmlElement) - let firstTable = tablesAsJson[0] - options = firstTable.map(element => ({ - name: element['0'], - probability: Number(element['1'].replace("%", "")) / 100, - type: "PROBABILITY" - })) + let optionsHtmlElement = ""; + let tablesAsJson = Tabletojson.convert(optionsHtmlElement); + let firstTable = tablesAsJson[0]; + options = firstTable.map((element) => ({ + name: element["0"], + probability: Number(element["1"].replace("%", "")) / 100, + type: "PROBABILITY", + })); //console.log(optionsHtmlElement) //console.log(options) } // Description - let descriptionraw = response.split(`
`)[1] - let descriptionprocessed1 = descriptionraw.split(`
`)[0] - let descriptionprocessed2 = toMarkdown(descriptionprocessed1) - let descriptionprocessed3 = descriptionprocessed2.split("\n") - .filter(string => !string.includes("Confused? Check our")) - .join("\n") - let description = descriptionprocessed3 + let descriptionraw = response.split( + `
` + )[1]; + let descriptionprocessed1 = descriptionraw.split(`
`)[0]; + let descriptionprocessed2 = toMarkdown(descriptionprocessed1); + let descriptionprocessed3 = descriptionprocessed2 + .split("\n") + .filter((string) => !string.includes("Confused? Check our")) + .join("\n"); + let description = descriptionprocessed3; // Number of forecasts - let numforecasts = response.split("prediction_sets_count":")[1].split(",")[0] + let numforecasts = response + .split("prediction_sets_count":")[1] + .split(",")[0]; //console.log(numforecasts) // Number of predictors - let numforecasters = response.split("predictors_count":")[1].split(",")[0] + let numforecasters = response + .split("predictors_count":")[1] + .split(",")[0]; //console.log(numpredictors) // Calculate the stars - let minProbability = Math.min(...options.map(option => option.probability)) - let maxProbability = Math.max(...options.map(option => option.probability)) - + let minProbability = Math.min(...options.map((option) => option.probability)); + let maxProbability = Math.max(...options.map((option) => option.probability)); + let result = { - "description": description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": Number(numforecasts), - "numforecasters": Number(numforecasters), - "stars": calculateStars("Good Judgment Open", ({ numforecasts, minProbability, maxProbability })) - } - } - return result + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Number(numforecasts), + numforecasters: Number(numforecasters), + stars: calculateStars("Good Judgment Open", { + numforecasts, + minProbability, + maxProbability, + }), + }, + }; + return result; } -function isSignedIn(html){ - - let isSignedInBool = !( html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") ) +function isSignedIn(html) { + let isSignedInBool = !( + html.includes("You need to sign in or sign up before continuing") || + html.includes("Sign up") + ); // console.log(html) - if(!isSignedInBool){ - console.log("Error: Not signed in.") + if (!isSignedInBool) { + console.log("Error: Not signed in."); } - console.log(`is signed in? ${isSignedInBool}`) - return isSignedInBool + console.log(`is signed in? ${isSignedInBool}`); + return isSignedInBool; } -function isEnd(html){ - let isEndBool = html.includes("No questions match your filter") - if(isEndBool){ +function isEnd(html) { + let isEndBool = html.includes("No questions match your filter"); + if (isEndBool) { //console.log(html) } - console.log(`IsEnd? ${isEndBool}`) - return isEndBool + console.log(`IsEnd? ${isEndBool}`); + return isEndBool; } function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise((resolve) => setTimeout(resolve, ms)); } /* Body */ async function goodjudgmentopen_inner(cookie) { - let i = 1 - let response = await fetchPage(i, cookie) - - let results = [] - let init = Date.now() + let i = 1; + let response = await fetchPage(i, cookie); + + let results = []; + let init = Date.now(); // console.log("Downloading... This might take a couple of minutes. Results will be shown.") - while(!isEnd(response) && isSignedIn(response)){ - let htmlLines = response.split("\n") - DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id() - let h5elements = htmlLines.filter(str => str.includes("
console.log(line)) : id(); + let h5elements = htmlLines.filter((str) => str.includes("
') - let url = h5elementSplit[0].split(''); + let url = h5elementSplit[0].split('
", ""); + await sleep(1000 + Math.random() * 1000); // don't be as noticeable try { - let moreinfo = await fetchStats(url, cookie) + let moreinfo = await fetchStats(url, cookie); if (moreinfo.isbinary) { - if (!moreinfo.crowdpercentage) { // then request again. - moreinfo = await fetchStats(url, cookie) + if (!moreinfo.crowdpercentage) { + // then request again. + moreinfo = await fetchStats(url, cookie); } } - let question = ({ - "title": title, - "url": url, - "platform": "Good Judgment Open", - ...moreinfo - }) - if(j % 30 == 0 || DEBUG_MODE == "on"){ - console.log(`Page #${i}`) - console.log(question) + let questionNumRegex = new RegExp("questions/([0-9]+)"); + let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; + let id = `goodjudmentopen-${questionNum}`; + let question = { + id: id, + title: title, + url: url, + platform: "Good Judgment Open", + ...moreinfo, + }; + if (j % 30 == 0 || DEBUG_MODE == "on") { + console.log(`Page #${i}`); + console.log(question); } // console.log(question) - results.push(question) + results.push(question); } catch (error) { - console.log(error) - console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`) + console.log(error); + console.log( + `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json` + ); } } - j = j+1 + j = j + 1; } - i = i + 1 + i = i + 1; // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers") - await sleep(5000 + Math.random() * 1000) // don't be a dick to gjopen server + await sleep(5000 + Math.random() * 1000); // don't be a dick to gjopen server try { - response = await fetchPage(i, cookie) + response = await fetchPage(i, cookie); } catch (error) { - console.log(error) - console.log(`We encountered some error when fetching page #${i}, so it won't appear on the final json`) + console.log(error); + console.log( + `We encountered some error when fetching page #${i}, so it won't appear on the final json` + ); } } // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/goodjudmentopen-questions.json', string); - console.log(results) - if(results.length > 0){ - await upsert(results, "goodjudmentopen-questions") - }else{ - console.log("Not updating results, as process was not signed in") + console.log(results); + if (results.length > 0) { + await upsert(results, "goodjudmentopen-questions"); + } else { + console.log("Not updating results, as process was not signed in"); } - let end = Date.now() - let difference = end - init - console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`) + let end = Date.now(); + let difference = end - init; + console.log( + `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` + ); } -export async function goodjudgmentopen(){ - let cookie = process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudmentopen") - await applyIfCookieExists(cookie, goodjudgmentopen_inner) +export async function goodjudgmentopen() { + let cookie = + process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudmentopen"); + await applyIfCookieExists(cookie, goodjudgmentopen_inner); } diff --git a/src/platforms/infer-fetch.js b/src/platforms/infer-fetch.js index cdd6ce0..da41573 100644 --- a/src/platforms/infer-fetch.js +++ b/src/platforms/infer-fetch.js @@ -1,187 +1,201 @@ /* Imports */ -import axios from "axios" -import { getCookie, applyIfCookieExists } from "../utils/getCookies.js" -import { Tabletojson } from "tabletojson" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import { upsert } from "../utils/mongo-wrapper.js" +import axios from "axios"; +import { getCookie, applyIfCookieExists } from "../utils/getCookies.js"; +import { Tabletojson } from "tabletojson"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let htmlEndPoint = 'https://www.infer-pub.com/questions' -String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } -const DEBUG_MODE = "on"// "off" -const SLEEP_TIME_RANDOM = 7000 // miliseconds -const SLEEP_TIME_EXTRA = 2000 +let htmlEndPoint = "https://www.infer-pub.com/questions"; +String.prototype.replaceAll = function replaceAll(search, replace) { + return this.split(search).join(replace); +}; +const DEBUG_MODE = "on"; // "off" +const SLEEP_TIME_RANDOM = 7000; // miliseconds +const SLEEP_TIME_EXTRA = 2000; /* Support functions */ async function fetchPage(page, cookie) { - console.log(`Page #${page}`) + console.log(`Page #${page}`); if (page == 1) { - cookie = cookie.split(";")[0] // Interesting that it otherwise doesn't work :( + cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :( } - let urlEndpoint = `${htmlEndPoint}/?page=${page}` - console.log(urlEndpoint) + let urlEndpoint = `${htmlEndPoint}/?page=${page}`; + console.log(urlEndpoint); let response = await axios({ url: urlEndpoint, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - 'Cookie': cookie - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + Cookie: cookie, + }, + }).then((res) => res.data); // console.log(response) - return response + return response; } async function fetchStats(questionUrl, cookie) { let response = await axios({ url: questionUrl + "/stats", - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - 'Cookie': cookie, - 'Referer': questionUrl, - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + Cookie: cookie, + Referer: questionUrl, + }, + }).then((res) => res.data); if (response.includes("Sign up or sign in to forecast")) { - throw Error("Not logged in") + throw Error("Not logged in"); } // Is binary? - let isbinary = response.includes("binary?":true") + let isbinary = response.includes("binary?":true"); // console.log(`is binary? ${isbinary}`) - let options = [] + let options = []; if (isbinary) { // Crowd percentage - let htmlElements = response.split("\n") + let htmlElements = response.split("\n"); // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id() - let h3Element = htmlElements.filter(str => str.includes("

"))[0] + let h3Element = htmlElements.filter((str) => str.includes("

"))[0]; // DEBUG_MODE == "on" ? console.log(h5elements) : id() - let crowdpercentage = h3Element.split(">")[1].split("<")[0] - let probability = Number(crowdpercentage.replace("%", "")) / 100 - options.push(({ - name: "Yes", - probability: probability, - type: "PROBABILITY" - }), ({ - name: "No", - probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans - type: "PROBABILITY" - })) + let crowdpercentage = h3Element.split(">")[1].split("<")[0]; + let probability = Number(crowdpercentage.replace("%", "")) / 100; + options.push( + { + name: "Yes", + probability: probability, + type: "PROBABILITY", + }, + { + name: "No", + probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans + type: "PROBABILITY", + } + ); } else { try { - let optionsBody = response.split("tbody")[1] // Previously [1], then previously [3] but they added a new table. + let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table. // console.log(optionsBody) - let optionsHtmlElement = "" - let tablesAsJson = Tabletojson.convert(optionsHtmlElement) - let firstTable = tablesAsJson[0] - options = firstTable.map(element => ({ - name: element['0'], - probability: Number(element['1'].replace("%", "")) / 100, - type: "PROBABILITY" - })) + let optionsHtmlElement = ""; + let tablesAsJson = Tabletojson.convert(optionsHtmlElement); + let firstTable = tablesAsJson[0]; + options = firstTable.map((element) => ({ + name: element["0"], + probability: Number(element["1"].replace("%", "")) / 100, + type: "PROBABILITY", + })); } catch (error) { - let optionsBody = response.split("tbody")[3] // Catch if the error is related to table position - let optionsHtmlElement = "" - let tablesAsJson = Tabletojson.convert(optionsHtmlElement) - let firstTable = tablesAsJson[0] + let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position + let optionsHtmlElement = ""; + let tablesAsJson = Tabletojson.convert(optionsHtmlElement); + let firstTable = tablesAsJson[0]; if (firstTable) { - options = firstTable.map(element => ({ - name: element['0'], - probability: Number(element['1'].replace("%", "")) / 100, - type: "PROBABILITY" - })) + options = firstTable.map((element) => ({ + name: element["0"], + probability: Number(element["1"].replace("%", "")) / 100, + type: "PROBABILITY", + })); } else { // New type of question, tricky to parse the options // Just leave options = [] for now. // https://www.cset-foretell.com/blog/rolling-question-formats } } - } - // Description - let descriptionraw = response.split(``)[0] - let descriptionprocessed2 = descriptionprocessed1.replace(">", "") - let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``) + // Description + let descriptionraw = response.split(``)[0]; + let descriptionprocessed2 = descriptionprocessed1.replace(">", ""); + let descriptionprocessed3 = descriptionprocessed2.replace( + "To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", + `` + ); // console.log(descriptionprocessed3) - let descriptionprocessed4 = descriptionprocessed3.replaceAll("\r\n\r\n", "\n") - let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n") - let descriptionprocessed6 = descriptionprocessed5.replaceAll(""", `"`) - let descriptionprocessed7 = descriptionprocessed6.replaceAll("'", "'") - let descriptionprocessed8 = toMarkdown(descriptionprocessed7) - let description = descriptionprocessed8 + let descriptionprocessed4 = descriptionprocessed3.replaceAll( + "\r\n\r\n", + "\n" + ); + let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n"); + let descriptionprocessed6 = descriptionprocessed5.replaceAll(""", `"`); + let descriptionprocessed7 = descriptionprocessed6.replaceAll("'", "'"); + let descriptionprocessed8 = toMarkdown(descriptionprocessed7); + let description = descriptionprocessed8; // Number of forecasts //console.log(response) //console.log(response.split("prediction_sets_count":")[1]) - let numforecasts = response.split("prediction_sets_count":")[1].split(",")[0] + let numforecasts = response + .split("prediction_sets_count":")[1] + .split(",")[0]; // console.log(numforecasts) // Number of predictors - let numforecasters = response.split("predictors_count":")[1].split(",")[0] + let numforecasters = response + .split("predictors_count":")[1] + .split(",")[0]; // console.log(numpredictors) let result = { - "description": description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": Number(numforecasts), - "numforecasters": Number(numforecasters), - "stars": calculateStars("Infer", { numforecasts }) - } - } + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Number(numforecasts), + numforecasters: Number(numforecasters), + stars: calculateStars("Infer", { numforecasts }), + }, + }; - return result + return result; } function isSignedIn(html) { - - let isSignedInBool = !(html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up")) + let isSignedInBool = !( + html.includes("You need to sign in or sign up before continuing") || + html.includes("Sign up") + ); if (!isSignedInBool) { - console.log("Error: Not signed in.") + console.log("Error: Not signed in."); } - console.log(`Signed in? ${isSignedInBool}`) - return isSignedInBool + console.log(`Signed in? ${isSignedInBool}`); + return isSignedInBool; } function isEnd(html) { - let isEndBool = html.includes("No questions match your filter") + let isEndBool = html.includes("No questions match your filter"); if (isEndBool) { //console.log(html) } - console.log(`IsEnd? ${isEndBool}`) - return isEndBool + console.log(`IsEnd? ${isEndBool}`); + return isEndBool; } function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise((resolve) => setTimeout(resolve, ms)); } /* Body */ async function infer_inner(cookie) { - let i = 1 - let response = await fetchPage(i, cookie) - let results = [] - let init = Date.now() + let i = 1; + let response = await fetchPage(i, cookie); + let results = []; + let init = Date.now(); // console.log("Downloading... This might take a couple of minutes. Results will be shown.") while (!isEnd(response) && isSignedIn(response)) { - - let htmlLines = response.split("\n") + let htmlLines = response.split("\n"); // let h4elements = htmlLines.filter(str => str.includes("
+ str.includes("https://www.infer-pub.com/questions/") + ); // console.log(questionHrefs) - if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { //console.log(response) - console.log("questionHrefs: ") - console.log(questionHrefs) + console.log("questionHrefs: "); + console.log(questionHrefs); } //console.log("") @@ -191,64 +205,83 @@ async function infer_inner(cookie) { for (let questionHref of questionHrefs) { //console.log(h4element) - let elementSplit = questionHref.split('">') - let url = elementSplit[0].split('', "").replace("", "") - await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable + let elementSplit = questionHref.split('">'); + let url = elementSplit[0].split('", "") + .replace("
", "") + .replace("", ""); + await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable try { - let moreinfo = await fetchStats(url, cookie) - let question = ({ - "title": title, - "url": url, - "platform": "Infer", - ...moreinfo - }) - if (i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) { - console.log(`Page #${i}` && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) - console.log(question) + let moreinfo = await fetchStats(url, cookie); + let questionNumRegex = new RegExp("questions/([0-9]+)"); + let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; + let id = `infer-${questionNum}`; + let question = { + id: id, + title: title, + url: url, + platform: "Infer", + ...moreinfo, + }; + if ( + i % 30 == 0 && + !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") + ) { + console.log( + `Page #${i}` && + !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") + ); + console.log(question); } - results.push(question) + results.push(question); if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { - console.log(url) - console.log(question) + console.log(url); + console.log(question); } - } catch (error) { - console.log(error) - console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`) + console.log(error); + console.log( + `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json` + ); } } - i++ + i++; //i=Number(i)+1 - console.log("Sleeping for ~5secs so as to not be as noticeable to the infer servers") - await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable + console.log( + "Sleeping for ~5secs so as to not be as noticeable to the infer servers" + ); + await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable try { - response = await fetchPage(i, cookie) + response = await fetchPage(i, cookie); } catch (error) { - console.log(error) - console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`) + console.log(error); + console.log( + `The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages` + ); } } // let string = JSON.stringify(results,null, 2) // fs.writeFileSync('./data/infer-questions.json', string); // console.log(results) if (results.length > 0) { - await upsert(results, "infer-questions") + await upsert(results, "infer-questions"); } else { - console.log("Not updating results, as process was not signed in") + console.log("Not updating results, as process was not signed in"); } - let end = Date.now() - let difference = end - init - console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`) + let end = Date.now(); + let difference = end - init; + console.log( + `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` + ); } - export async function infer() { - let cookie = process.env.INFER_COOKIE || getCookie("infer") - await applyIfCookieExists(cookie, infer_inner) + let cookie = process.env.INFER_COOKIE || getCookie("infer"); + await applyIfCookieExists(cookie, infer_inner); } diff --git a/src/platforms/kalshi-fetch.js b/src/platforms/kalshi-fetch.js index 570a38b..12a95e8 100644 --- a/src/platforms/kalshi-fetch.js +++ b/src/platforms/kalshi-fetch.js @@ -1,11 +1,11 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' +let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' /* Support functions async function fetchAllContractInfo(){ // for info which the polymarket graphql API @@ -14,66 +14,76 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql return response } */ -async function fetchAllMarkets() { // for info which the polymarket graphql API - let response = await axios.get(jsonEndpoint).then(response => response.data.markets) +async function fetchAllMarkets() { + // for info which the polymarket graphql API + let response = await axios + .get(jsonEndpoint) + .then((response) => response.data.markets); // console.log(response) - return response + return response; } - async function processMarkets(markets) { - let dateNow = new Date().toISOString() + let dateNow = new Date().toISOString(); // console.log(markets) - markets = markets.filter(market => market.close_date > dateNow) - let results = await markets.map(market => { - let probability = market.last_price/100 + markets = markets.filter((market) => market.close_date > dateNow); + let results = await markets.map((market) => { + let probability = market.last_price / 100; let options = [ { - "name": "Yes", - "probability": probability, - "type": "PROBABILITY" + name: "Yes", + probability: probability, + type: "PROBABILITY", }, { - "name": "No", - "probability": 1 - probability, - "type": "PROBABILITY" - } - ] - let result = ({ - "title": market.title.replaceAll("*", ""), - "url": `https://kalshi.com/markets/${market.ticker_name}`, - "platform": "Kalshi", - "description": `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "stars": calculateStars("Kalshi", ({shares_volume: market.volume, interest: market.open_interest})), - "yes_bid": market.yes_bid, - "yes_ask": market.yes_ask, - "spread": Math.abs(market.yes_bid-market.yes_ask), - "shares_volume": market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if. - // "open_interest": market.open_interest, also in shares - } - }) - return result - }) + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; + let id = `kalshi-${market.id}`; + let result = { + id: id, + title: market.title.replaceAll("*", ""), + url: `https://kalshi.com/markets/${market.ticker_name}`, + platform: "Kalshi", + description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Kalshi", { + shares_volume: market.volume, + interest: market.open_interest, + }), + yes_bid: market.yes_bid, + yes_ask: market.yes_ask, + spread: Math.abs(market.yes_bid - market.yes_ask), + shares_volume: market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if. + // "open_interest": market.open_interest, also in shares + }, + }; + return result; + }); //console.log(results.length) // console.log(results.map(result => result.title)) // console.log(results.map(result => result.title).length) - console.log([...new Set(results.map(result => result.title))]) - console.log("Number of unique questions: ", [...new Set(results.map(result => result.title))].length) + console.log([...new Set(results.map((result) => result.title))]); + console.log( + "Number of unique questions: ", + [...new Set(results.map((result) => result.title))].length + ); // console.log([...new Set(results.map(result => result.title))].length) - return results //resultsProcessed + return results; //resultsProcessed } /* Body */ export async function kalshi() { - let markets = await fetchAllMarkets() - let results = await processMarkets(markets) // somehow needed + let markets = await fetchAllMarkets(); + let results = await processMarkets(markets); // somehow needed // console.log(results) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('polymarket-questions.json', string); - await upsert(results, "kalshi-questions") - console.log("Done") + await upsert(results, "kalshi-questions"); + console.log("Done"); } // kalshi() diff --git a/src/platforms/ladbrokes-fetch.js b/src/platforms/ladbrokes-fetch.js deleted file mode 100644 index 9bbb7f1..0000000 --- a/src/platforms/ladbrokes-fetch.js +++ /dev/null @@ -1,136 +0,0 @@ -/* Imports */ -import axios from "axios" -import fs from "fs" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" - -/* Definitions */ -let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market` -let enpointDrillDown = (id) => `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json` - -//
response.data) - return response - -} - -let processResults = async (json) => { - let results = [] - let children = json.SSResponse.children - children.pop() - let ids = children.map(child => child.event.id) - let markets = [] - for(let id of ids){ - let marketsContainer = await fetchUrl(enpointDrillDown(id)) - let marketsObj = marketsContainer.SSResponse.children[0].event - let newMarkets = marketsObj.children - newMarkets = newMarkets.map(market => ({...market.market, parent: marketsObj.name})) - markets.push(...newMarkets) - } - let normalMarkets = markets.filter(market => !market.name.includes("Specials")) - //console.log(normalMarkets) - - for(let normalMarket of normalMarkets){ - let title = normalMarket.parent + ": " + normalMarket.name - title = title.replace("Boris Johnson Specials", "Boris Johnson") // small kludge - let options = normalMarket.children.map(child => { - let name = child.outcome.name - - let priceData = child.outcome.children[0].price - let priceDecimal = Number(priceData.priceDec) - let probability = 1/priceDecimal - let option = ({ - "name":name, - "probability": probability, - "type": "PROBABILITY" - }) - return option - }) - // normalize probabilities - let totalValue = options - .map(element => Number(element.probability)) - .reduce((a, b) => (a + b), 0) - options = options.map(element => ({ - ...element, - probability: Number(element.probability) / totalValue - })) - - // Filter very unlikely probabilities: Not here, but on the front end - // options = options.filter(element => element.probability > 0.02) - - let obj = ({ - "title": title, - "url": "https://sports.ladbrokes.com/sport/politics/outrights", - "platform": "Ladbrokes", - "description": "", - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "stars": calculateStars("Ladbrokes", ({})) - } - }) - results.push(obj) - } - - - let specialMarkets = markets.filter(market => market.name.includes("Specials")) - for(let specialMarket of specialMarkets){ - //let title = specialMarket.parent + ": " + specialMarket.name - //console.log(title) - specialMarket.children.forEach(child => { - let name = specialMarket.parent.includes("Specials")? child.outcome.name : specialMarket.parent + ": " + child.outcome.name - name = name.replace("Boris Johnson Specials", "Boris Johnson") // small kludge - let priceData = child.outcome.children[0].price - let priceDecimal = Number(priceData.priceDec) - let probability = 1/priceDecimal - let obj = ({ - "title": name, - "url": "https://sports.ladbrokes.com/sport/politics/outrights", - "platform": "LadBrokes", - "options": [ - { - "name": "Yes", - "probability": probability, - "type": "PROBABILITY" - }, - { - "name": "No", - "probability": 1 - probability, - "type": "PROBABILITY" - } - ], - "qualityindicators": { - "stars": calculateStars("Ladbrokes", ({})) - } - }) - results.push(obj) - }) - } - return results -} - -/* Body */ -export async function ladbrokes() { - let response = await fetchUrl(endpointPolitics) - let results = await processResults(response) - // console.log(results) - // let string = JSON.stringify(results, null, 2) - // fs.writeFileSync('./data/ladbrokes-questions.json', string); - await upsert(results, "ladbrokes-questions") - console.log("Done") -} -//ladbrokes() diff --git a/src/platforms/manifoldmarkets-fetch.js b/src/platforms/manifoldmarkets-fetch.js index fa8b392..fbbc36b 100644 --- a/src/platforms/manifoldmarkets-fetch.js +++ b/src/platforms/manifoldmarkets-fetch.js @@ -43,6 +43,7 @@ function showStatistics(results) { async function processPredictions(predictions) { let results = await predictions.map((prediction) => { + let id = `manifold-${prediction.id}`; let probability = prediction.probability; let options = [ { @@ -57,6 +58,7 @@ async function processPredictions(predictions) { }, ]; let result = { + id: id, title: prediction.question, url: prediction.url, platform: "Manifold Markets", diff --git a/src/platforms/metaculus-fetch.js b/src/platforms/metaculus-fetch.js index 54c5e8a..bfacc34 100644 --- a/src/platforms/metaculus-fetch.js +++ b/src/platforms/metaculus-fetch.js @@ -1,158 +1,161 @@ /* Imports */ -import axios from "axios" -import fs from 'fs' -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import axios from "axios"; +import fs from "fs"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let jsonEndPoint = 'https://www.metaculus.com/api2/questions/?page=' +let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; let all_questions = []; -let now = new Date().toISOString() - +let now = new Date().toISOString(); +let DEBUG_MODE = "off"; /* Support functions */ async function fetchMetaculusQuestions(next) { // Numbers about a given address: how many, how much, at what price, etc. - let response = await axios(({ + let response = await axios({ url: next, - method: 'GET', - headers: ({ 'Content-Type': 'application/json' }) - })) - .then(res => res.data) + method: "GET", + headers: { "Content-Type": "application/json" }, + }).then((res) => res.data); // console.log(response) - return response + return response; } - function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise((resolve) => setTimeout(resolve, ms)); } async function fetchMetaculusQuestionDescription(slug) { try { let response = await axios({ - method: 'get', - url: "https://www.metaculus.com" + slug - }).then(response => response.data) - return response + method: "get", + url: "https://www.metaculus.com" + slug, + }).then((response) => response.data); + return response; } catch (error) { - console.log(error) - console.log(`We encountered some error when attempting to fetch a metaculus page. Trying again`) - await sleep(10000) + console.log(error); + console.log( + `We encountered some error when attempting to fetch a metaculus page. Trying again` + ); + await sleep(10000); try { let response = await axios({ - method: 'get', - url: "https://www.metaculus.com" + slug - }).then(response => response.data) + method: "get", + url: "https://www.metaculus.com" + slug, + }).then((response) => response.data); // console.log(response) - return response + return response; } catch (error) { - console.log(`We encountered some error when attempting to fetch a metaculus page.`) - console.log("Error", error) - throw "Giving up" + console.log( + `We encountered some error when attempting to fetch a metaculus page.` + ); + console.log("Error", error); + throw "Giving up"; } } - } /* Body */ export async function metaculus() { - // let metaculusQuestionsInit = await fetchMetaculusQuestions(1) // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test - let next = "https://www.metaculus.com/api2/questions/" - let i = 1 - while(next){ + let next = "https://www.metaculus.com/api2/questions/"; + let i = 1; + while (next) { if (i % 20 == 0) { - console.log("Sleeping for 5secs") - await sleep(5000) + console.log("Sleeping for 5secs"); + await sleep(5000); } - console.log(`\nQuery #${i}`) - let metaculusQuestions = await fetchMetaculusQuestions(next) + console.log(`\nQuery #${i}`); + let metaculusQuestions = await fetchMetaculusQuestions(next); let results = metaculusQuestions.results; - let j=false + let j = false; for (let result of results) { - if ( - (result.publish_time < now) && - (now < result.resolve_time) - ) { - await sleep(5000) - let questionPage = await fetchMetaculusQuestionDescription(result.page_url) - if(!questionPage.includes("A public prediction by")){ + if (result.publish_time < now && now < result.resolve_time) { + await sleep(5000); + let questionPage = await fetchMetaculusQuestionDescription( + result.page_url + ); + if (!questionPage.includes("A public prediction by")) { // console.log(questionPage) - let descriptionraw = questionPage.split(`
`)[1] //.split(`
`)[1] - let descriptionprocessed1 = descriptionraw.split("
")[0] - let descriptionprocessed2 = toMarkdown(descriptionprocessed1) - let description = descriptionprocessed2 - - let isbinary = result.possibilities.type == "binary" - let options = [] + let descriptionraw = questionPage.split( + `
` + )[1]; //.split(`
`)[1] + let descriptionprocessed1 = descriptionraw.split("
")[0]; + let descriptionprocessed2 = toMarkdown(descriptionprocessed1); + let description = descriptionprocessed2; + + let isbinary = result.possibilities.type == "binary"; + let options = []; if (isbinary) { - let probability = Number(result.community_prediction.full.q2) + let probability = Number(result.community_prediction.full.q2); options = [ { - "name": "Yes", - "probability": probability, - "type": "PROBABILITY" + name: "Yes", + probability: probability, + type: "PROBABILITY", }, { - "name": "No", - "probability": 1 - probability, - "type": "PROBABILITY" - } - ] + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; } - let interestingInfo = ({ - "title": result.title, - "url": "https://www.metaculus.com" + result.page_url, - "platform": "Metaculus", - "description": description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": Number(result.number_of_predictions), - "stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions })) - }, - "extra": { - "resolution_data": { - "publish_time": result.publish_time, - "resolution": result.resolution, - "close_time": result.close_time, - "resolve_time": result.resolve_time - } - } + let id = `metaculus-${result.id}`; + let interestingInfo = { + id: id, + title: result.title, + url: "https://www.metaculus.com" + result.page_url, + platform: "Metaculus", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: Number(result.number_of_predictions), + stars: calculateStars("Metaculus", { + numforecasts: result.number_of_predictions, + }), + }, + extra: { + resolution_data: { + publish_time: result.publish_time, + resolution: result.resolution, + close_time: result.close_time, + resolve_time: result.resolve_time, + }, + }, //"status": result.status, //"publish_time": result.publish_time, //"close_time": result.close_time, //"type": result.possibilities.type, // We want binary ones here. //"last_activity_time": result.last_activity_time, - }) + }; if (Number(result.number_of_predictions) >= 10) { - console.log(`- ${interestingInfo.title}`) - all_questions.push(interestingInfo) - if(!j && (i % 20 == 0)){ - console.log(interestingInfo) - j = true + console.log(`- ${interestingInfo.title}`); + all_questions.push(interestingInfo); + if ((!j && i % 20 == 0) || DEBUG_MODE == "on") { + console.log(interestingInfo); + j = true; } } - }else{ - console.log("- [Skipping public prediction]") + } else { + console.log("- [Skipping public prediction]"); } - - } } - next = metaculusQuestions.next - i = i+1 + next = metaculusQuestions.next; + i = i + 1; } // let string = JSON.stringify(all_questions, null, 2) // fs.writeFileSync('./metaculus-questions.json', string); - await upsert(all_questions, "metaculus-questions") + await upsert(all_questions, "metaculus-questions"); - console.log("Done") + console.log("Done"); } //metaculus() diff --git a/src/platforms/polymarket-fetch.js b/src/platforms/polymarket-fetch.js index 8519f4c..3afb158 100644 --- a/src/platforms/polymarket-fetch.js +++ b/src/platforms/polymarket-fetch.js @@ -1,12 +1,13 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"// "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' -let units = 10 ** 6 +let graphQLendpoint = + "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' +let units = 10 ** 6; /* Support functions async function fetchAllContractInfo(){ // for info which the polymarket graphql API @@ -15,20 +16,24 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql return response } */ -async function fetchAllContractInfo() { // for info which the polymarket graphql API - let response = await axios.get('https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1') - .then(query => query.data); - response = response.filter(res => res.closed != true) - return response +async function fetchAllContractInfo() { + // for info which the polymarket graphql API + let response = await axios + .get( + "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1" + ) + .then((query) => query.data); + response = response.filter((res) => res.closed != true); + return response; } async function fetchAllContractData() { - let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7 // last week + let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week let response = await axios({ url: graphQLendpoint, - method: 'POST', - headers: ({ 'Content-Type': 'application/json' }), - data: JSON.stringify(({ + method: "POST", + headers: { "Content-Type": "application/json" }, + data: JSON.stringify({ query: ` { fixedProductMarketMakers(first: 1000 @@ -53,23 +58,23 @@ async function fetchAllContractData() { } } } - ` - })), + `, + }), }) - .then(res => res.data) - .then(res => res.data.fixedProductMarketMakers) + .then((res) => res.data) + .then((res) => res.data.fixedProductMarketMakers); // console.log(response) - return response + return response; } async function fetch_all() { - let allData = await fetchAllContractData() - let allInfo = await fetchAllContractInfo() + let allData = await fetchAllContractData(); + let allInfo = await fetchAllContractInfo(); - let infos = ({}) + let infos = {}; for (let info of allInfo) { - let address = info.marketMakerAddress - let addressLowerCase = address.toLowerCase() + let address = info.marketMakerAddress; + let addressLowerCase = address.toLowerCase(); //delete info.history if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") infos[addressLowerCase] = { @@ -79,67 +84,75 @@ async function fetch_all() { description: info.description, outcomes: info.outcomes, options: [], - category: info.category - } + category: info.category, + }; } - let results = [] + let results = []; for (let data of allData) { - let addressLowerCase = data.id + let addressLowerCase = data.id; // console.log(data) if (infos[addressLowerCase] != undefined) { // console.log(addressLowerCase) - let info = infos[addressLowerCase] - let numforecasts = Number(data.tradesQuantity) - let tradevolume = (Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) / units - let liquidity = Number(data.liquidityParameter) / units + let id = `polymarket-${addressLowerCase.slice(0, 10)}`; + let info = infos[addressLowerCase]; + let numforecasts = Number(data.tradesQuantity); + let tradevolume = + (Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) / + units; + let liquidity = Number(data.liquidityParameter) / units; // let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2 // let percentage = Number(data.outcomeTokenPrices[0]) * 100 // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none" - let options = [] + let options = []; for (let outcome in data.outcomeTokenPrices) { options.push({ - "name": info.outcomes[outcome], - "probability": data.outcomeTokenPrices[outcome], - "type": "PROBABILITY" - }) + name: info.outcomes[outcome], + probability: data.outcomeTokenPrices[outcome], + type: "PROBABILITY", + }); } - let result = ({ - "title": info.title, - "url": info.url, - "platform": "PolyMarket", - "description": info.description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": numforecasts.toFixed(0), - "liquidity": liquidity.toFixed(2), - "tradevolume": tradevolume.toFixed(2), - "stars": calculateStars("Polymarket", ({ liquidity, option: options[0], volume: tradevolume})) - }, - "extra": { - "address": info.address - } + let result = { + id: id, + title: info.title, + url: info.url, + platform: "PolyMarket", + description: info.description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: numforecasts.toFixed(0), + liquidity: liquidity.toFixed(2), + tradevolume: tradevolume.toFixed(2), + stars: calculateStars("Polymarket", { + liquidity, + option: options[0], + volume: tradevolume, + }), + }, + extra: { + address: info.address, + }, /* - */ - }) - if(info.category != "Sports"){ + */ + }; + if (info.category != "Sports") { // console.log(result) - results.push(result) + results.push(result); } } } - return results //resultsProcessed + return results; //resultsProcessed } /* Body */ export async function polymarket() { - let results = await fetch_all() + let results = await fetch_all(); // console.log(results) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('polymarket-questions.json', string); - await upsert(results, "polymarket-questions") - console.log("Done") + await upsert(results, "polymarket-questions"); + console.log("Done"); } // polymarket() diff --git a/src/platforms/predictit-fetch.js b/src/platforms/predictit-fetch.js index e6fb33e..09a47c9 100644 --- a/src/platforms/predictit-fetch.js +++ b/src/platforms/predictit-fetch.js @@ -1,114 +1,116 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Support functions */ async function fetchmarkets() { let response = await axios({ - method: 'get', - url: 'https://www.predictit.org/api/marketdata/all/' - - }) - let openMarkets = response.data.markets.filter(market => market.status == "Open") - return openMarkets + method: "get", + url: "https://www.predictit.org/api/marketdata/all/", + }); + let openMarkets = response.data.markets.filter( + (market) => market.status == "Open" + ); + return openMarkets; } async function fetchmarketrules(market_id) { let response = await axios({ - method: 'get', - url: 'https://www.predictit.org/api/Market/' + market_id - }) - return response.data.rule + method: "get", + url: "https://www.predictit.org/api/Market/" + market_id, + }); + return response.data.rule; } -async function fetchmarketvolumes(){ +async function fetchmarketvolumes() { let response = await axios({ - method: 'get', - url: "https://predictit-f497e.firebaseio.com/marketStats.json" - }) - return response.data + method: "get", + url: "https://predictit-f497e.firebaseio.com/marketStats.json", + }); + return response.data; } function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise((resolve) => setTimeout(resolve, ms)); } - /* Body */ export async function predictit() { - let markets = await fetchmarkets() - let marketVolumes = await fetchmarketvolumes() + let markets = await fetchmarkets(); + let marketVolumes = await fetchmarketvolumes(); - markets = markets.map(market => ({ + markets = markets.map((market) => ({ ...market, - TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"] - })) + TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"], + })); // console.log(markets) - let results = [] + let results = []; for (let market of markets) { // console.log(market.name) + let id = `predictit-${market.id}`; let isbinary = market.contracts.length == 1; - await sleep(3000 * (1 + Math.random())) - let descriptionraw = await fetchmarketrules(market.id) - let descriptionprocessed1 = toMarkdown(descriptionraw) - let description = descriptionprocessed1 - let shares_volume = market["TotalSharesTraded"] + await sleep(3000 * (1 + Math.random())); + let descriptionraw = await fetchmarketrules(market.id); + let descriptionprocessed1 = toMarkdown(descriptionraw); + let description = descriptionprocessed1; + let shares_volume = market["TotalSharesTraded"]; // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none" - let options = market.contracts.map(contract => ({ - "name": contract.name, - "probability": contract.lastTradePrice, - "type": "PROBABILITY" - })) + let options = market.contracts.map((contract) => ({ + name: contract.name, + probability: contract.lastTradePrice, + type: "PROBABILITY", + })); let totalValue = options - .map(element => Number(element.probability)) - .reduce((a, b) => (a + b), 0) + .map((element) => Number(element.probability)) + .reduce((a, b) => a + b, 0); if (options.length != 1 && totalValue > 1) { - options = options.map(element => ({ + options = options.map((element) => ({ ...element, - probability: Number(element.probability) / totalValue - })) + probability: Number(element.probability) / totalValue, + })); } else if (options.length == 1) { - let option = options[0] - let probability = option["probability"] + let option = options[0]; + let probability = option["probability"]; options = [ { - "name": "Yes", - "probability": probability, - "type": "PROBABILITY" + name: "Yes", + probability: probability, + type: "PROBABILITY", }, { - "name": "No", - "probability": 1 - probability, - "type": "PROBABILITY" - } - ] + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; } - let obj = ({ - "title": market["name"], - "url": market.url, - "platform": "PredictIt", - "description": description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "stars": calculateStars("PredictIt", ({})), - "shares_volume": shares_volume - } - }) + let obj = { + id: id, + title: market["name"], + url: market.url, + platform: "PredictIt", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("PredictIt", {}), + shares_volume: shares_volume, + }, + }; // console.log(obj) - results.push(obj) + results.push(obj); } //console.log(results) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/predictit-questions.json', string); - await upsert(results, "predictit-questions") + await upsert(results, "predictit-questions"); - console.log("Done") + console.log("Done"); } diff --git a/src/platforms/rootclaim-fetch.js b/src/platforms/rootclaim-fetch.js index 734d98e..0643291 100644 --- a/src/platforms/rootclaim-fetch.js +++ b/src/platforms/rootclaim-fetch.js @@ -1,12 +1,13 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let jsonEndpoint = "https://www.rootclaim.com/main_page_stories?number=100&offset=0"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' +let jsonEndpoint = + "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' /* Support functions async function fetchAllContractInfo(){ // for info which the polymarket graphql API @@ -15,56 +16,60 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql return response } */ -async function fetchAllRootclaims() { // for info which the polymarket graphql API - let response = await axios.get(jsonEndpoint) - .then(response => response.data) - if(response.length != (response[0]+1)){ - console.log(response.length) - console.log(response[0]) - //throw Error("Rootclaim's backend has changed.") +async function fetchAllRootclaims() { + // for info which the polymarket graphql API + let response = await axios + .get(jsonEndpoint) + .then((response) => response.data); + if (response.length != response[0] + 1) { + console.log(response.length); + console.log(response[0]); + //throw Error("Rootclaim's backend has changed.") } - response.shift() - return response + response.shift(); + return response; } async function fetchAndProcessData() { - let claims = await fetchAllRootclaims() - let results = [] + let claims = await fetchAllRootclaims(); + let results = []; for (let claim of claims) { - let options = [] + let id = `rootclaim-${claim.slug.toLowerCase()}`; + let options = []; for (let scenario of claim.scenarios) { - //console.log(scenario) - options.push({ - "name": toMarkdown(scenario.text).replace("\n","").replace("'","'"), - "probability": scenario.net_prob / 100, - "type": "PROBABILITY" - }) + //console.log(scenario) + options.push({ + name: toMarkdown(scenario.text).replace("\n", "").replace("'", "'"), + probability: scenario.net_prob / 100, + type: "PROBABILITY", + }); } - let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis" - let obj = ({ - "title": toMarkdown(claim.question).replace("\n",""), - "url": `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, - "platform": "Rootclaim", - "description": toMarkdown(claim.background).replace("'","'"), - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "numforecasts": 1, - "stars": calculateStars("Rootclaim", ({})) - } - }) - results.push(obj) - } - return results + let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"; + let obj = { + id: id, + title: toMarkdown(claim.question).replace("\n", ""), + url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, + platform: "Rootclaim", + description: toMarkdown(claim.background).replace("'", "'"), + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + numforecasts: 1, + stars: calculateStars("Rootclaim", {}), + }, + }; + results.push(obj); + } + return results; } /* Body */ export async function rootclaim() { - let results = await fetchAndProcessData() + let results = await fetchAndProcessData(); //console.log(JSON.stringify(results, null, 4)) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('rootclaim-questions.json', string); - await upsert(results, "rootclaim-questions") - console.log("Done") + await upsert(results, "rootclaim-questions"); + console.log("Done"); } //rootclaim() diff --git a/src/platforms/smarkets-fetch.js b/src/platforms/smarkets-fetch.js index b5ed33c..f17c11e 100644 --- a/src/platforms/smarkets-fetch.js +++ b/src/platforms/smarkets-fetch.js @@ -1,127 +1,127 @@ /* Imports */ -import fs from 'fs' -import axios from "axios" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" +import fs from "fs"; +import axios from "axios"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let htmlEndPointEntrance = 'https://api.smarkets.com/v3/events/' +let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; /* Support functions */ async function fetchEvents(url) { let response = await axios({ url: htmlEndPointEntrance + url, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); // console.log(response) - return response + return response; } async function fetchMarkets(eventid) { let response = await axios({ url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, - method: 'GET', - headers: ({ - 'Content-Type': 'text/json', - }), + method: "GET", + headers: { + "Content-Type": "text/json", + }, }) - .then(res => res.data) - .then(res => res.markets) - return response + .then((res) => res.data) + .then((res) => res.markets); + return response; } async function fetchContracts(marketid) { let response = await axios({ url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); // console.log(response) - return response - + return response; } async function fetchPrices(marketid) { let response = await axios({ url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, - method: 'GET', - headers: ({ - 'Content-Type': 'text/html', - }), - }) - .then(res => res.data) + method: "GET", + headers: { + "Content-Type": "text/html", + }, + }).then((res) => res.data); // console.log(response) - return response - + return response; } /* Body */ export async function smarkets() { - let htmlPath = '?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50' + let htmlPath = + "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; - let events = [] + let events = []; while (htmlPath) { - let data = await fetchEvents(htmlPath) - events.push(...data.events) - htmlPath = data.pagination.next_page + let data = await fetchEvents(htmlPath); + events.push(...data.events); + htmlPath = data.pagination.next_page; } // console.log(events) - let markets = [] + let markets = []; for (let event of events) { // console.log(Date.now()) // console.log(event.name) - let eventMarkets = await fetchMarkets(event.id) - eventMarkets = eventMarkets.map(market => ({ ...market, slug: event.full_slug })) + let eventMarkets = await fetchMarkets(event.id); + eventMarkets = eventMarkets.map((market) => ({ + ...market, + slug: event.full_slug, + })); // console.log("Markets fetched") // console.log(event.id) // console.log(market) - markets.push(...eventMarkets) + markets.push(...eventMarkets); //let lastPrices = await fetchPrices(market.id) } // console.log(markets) - let results = [] + let results = []; for (let market of markets) { // console.log("================") // console.log("Market: ", market) - let name = market.name + let id = `smarkets-${market.id}`; + let name = market.name; - let contracts = await fetchContracts(market.id) + let contracts = await fetchContracts(market.id); // console.log("Contracts: ", contracts) - let prices = await fetchPrices(market.id) + let prices = await fetchPrices(market.id); // console.log("Prices: ", prices["last_executed_prices"][market.id]) - let options = {} + let options = {}; for (let contract of contracts["contracts"]) { - options[contract.id] = { name: contract.name } + options[contract.id] = { name: contract.name }; } for (let price of prices["last_executed_prices"][market.id]) { options[price.contract_id] = { ...options[price.contract_id], probability: Number(price.last_executed_price), - type: "PROBABILITY" - } + type: "PROBABILITY", + }; } - options = Object.values(options) + options = Object.values(options); let totalValue = options - .map(element => Number(element.probability)) - .reduce((a, b) => (a + b), 0) + .map((element) => Number(element.probability)) + .reduce((a, b) => a + b, 0); - options = options.map(element => ({ + options = options.map((element) => ({ ...element, - probability: Number(element.probability) / totalValue - })) + probability: Number(element.probability) / totalValue, + })); // console.log(options) @@ -135,26 +135,24 @@ export async function smarkets() { } */ let result = { - "title": name, - "url": "https://smarkets.com/event/" + market.event_id + market.slug, - "platform": "Smarkets", - "description": market.description, - "options": options, - "timestamp": new Date().toISOString(), - "qualityindicators": { - "stars": calculateStars("Smarkets", ({})) - } - - } + id: id, + title: name, + url: "https://smarkets.com/event/" + market.event_id + market.slug, + platform: "Smarkets", + description: market.description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Smarkets", {}), + }, + }; // console.log(result) - results.push(result) + results.push(result); } // console.log(results) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/smarkets-questions.json', string); - await upsert(results, "smarkets-questions") - - + await upsert(results, "smarkets-questions"); } //smarkets() diff --git a/src/platforms/wildeford-fetch.js b/src/platforms/wildeford-fetch.js index aa693dd..0290a00 100644 --- a/src/platforms/wildeford-fetch.js +++ b/src/platforms/wildeford-fetch.js @@ -1,51 +1,61 @@ /* Imports */ -import fs from 'fs' +import fs from "fs"; // import axios from "axios" -import { GoogleSpreadsheet } from "google-spreadsheet" -import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" -import toMarkdown from "../utils/toMarkdown.js" -import { calculateStars } from "../utils/stars.js" -import {upsert} from "../utils/mongo-wrapper.js" - +import { GoogleSpreadsheet } from "google-spreadsheet"; +import { getCookie, applyIfCookieExists } from "../utils/getCookies.js"; +import toMarkdown from "../utils/toMarkdown.js"; +import { calculateStars } from "../utils/stars.js"; +import { upsert } from "../utils/mongo-wrapper.js"; +import { hash } from "../utils/hash.js"; /* Definitions */ -const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0" // spreadsheet key is the long id in the sheets URL -const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0` +const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL +const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`; // https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0 /* Support functions */ -const formatRow = row => { - let colNames = ["Prediction Date", "Prediction", "Odds", "Actual", "Resolution Date", "Prediction Right?", "Brier Score", "Notes"] - let result = ({}) - row.forEach((col,i) => { - result[colNames[i]] = col - }) - return result -} +const formatRow = (row) => { + let colNames = [ + "Prediction Date", + "Prediction", + "Odds", + "Actual", + "Resolution Date", + "Prediction Right?", + "Brier Score", + "Notes", + ]; + let result = {}; + row.forEach((col, i) => { + result[colNames[i]] = col; + }); + return result; +}; -async function fetchGoogleDoc(google_api_key){ +async function fetchGoogleDoc(google_api_key) { // https://gist.github.com/micalevisk/9bc831bd4b3e5a3f62b9810330129c59 - let results = [] - const doc = new GoogleSpreadsheet(SHEET_ID) - doc.useApiKey(google_api_key) + let results = []; + const doc = new GoogleSpreadsheet(SHEET_ID); + doc.useApiKey(google_api_key); - await doc.loadInfo() // loads document properties and worksheets - console.log('>>', doc.title) + await doc.loadInfo(); // loads document properties and worksheets + console.log(">>", doc.title); - const sheet = doc.sheetsByIndex[0] - const rows = await sheet.getRows({ offset:0, }) + const sheet = doc.sheetsByIndex[0]; + const rows = await sheet.getRows({ offset: 0 }); - console.log('# ' + - rows[0]._sheet.headerValues.join(',') - ) + console.log("# " + rows[0]._sheet.headerValues.join(",")); let isEnd = false; for (let i in rows) { - let data = rows[i]._rawData - if(data.length == 0) isEnd = true; - if(!isEnd){ - let result = ({...formatRow(data), "url": endpoint + `&range=A${(Number(i) + 2)}`}) + let data = rows[i]._rawData; + if (data.length == 0) isEnd = true; + if (!isEnd) { + let result = { + ...formatRow(data), + url: endpoint + `&range=A${Number(i) + 2}`, + }; // +2: +1 for the header row, +1 for starting at 1 and not at 0. // console.log(result) - results.push(result) + results.push(result); // console.log(rows[i]) // console.log(rows[i]._rawData) @@ -55,66 +65,71 @@ async function fetchGoogleDoc(google_api_key){ // console.log(row._rawData.join(',')) } // console.log(results) - return(results) + return results; } -async function processPredictions(predictions){ - let currentPredictions = predictions.filter(prediction => prediction["Actual"] == "Unknown" ) - let results = currentPredictions.map(prediction => { - let probability = Number(prediction["Odds"].replace("%", ""))/100 +async function processPredictions(predictions) { + let currentPredictions = predictions.filter( + (prediction) => prediction["Actual"] == "Unknown" + ); + let results = currentPredictions.map((prediction) => { + let title = prediction["Prediction"].replace(" [update]", ""); + let id = `wildeford-${hash(title)}`; + let probability = Number(prediction["Odds"].replace("%", "")) / 100; let options = [ { - "name": "Yes", - "probability": probability, - "type": "PROBABILITY" + name: "Yes", + probability: probability, + type: "PROBABILITY", }, { - "name": "No", - "probability": 1 - probability, - "type": "PROBABILITY" - } - ] - let result = ({ - "title": prediction["Prediction"], - "url": prediction["url"], - "platform": "Peter Wildeford", - "description": prediction["Notes"] || "", - "options": options, - "timestamp": new Date(Date.parse(prediction["Prediction Date"] + "Z")).toISOString(), - "qualityindicators": { - "stars": calculateStars("Peter Wildeford"), - } - }) - return result - }) - results = results.map(result => ({...result, title: result.title.replace(" [update]", "")})).reverse() + name: "No", + probability: 1 - probability, + type: "PROBABILITY", + }, + ]; + let result = { + id: id, + title: title, + url: prediction["url"], + platform: "Peter Wildeford", + description: prediction["Notes"] || "", + options: options, + timestamp: new Date( + Date.parse(prediction["Prediction Date"] + "Z") + ).toISOString(), + qualityindicators: { + stars: calculateStars("Peter Wildeford"), + }, + }; + return result; + }); - let uniqueTitles = [] - let uniqueResults = [] - results.forEach(result => { - if(!uniqueTitles.includes(result.title)) uniqueResults.push(result) - uniqueTitles.push(result.title) - }) - return(uniqueResults) + results = results.reverse(); + let uniqueTitles = []; + let uniqueResults = []; + results.forEach((result) => { + if (!uniqueTitles.includes(result.title)) uniqueResults.push(result); + uniqueTitles.push(result.title); + }); + return uniqueResults; // console.log(results) // console.log(results.map(result => result.options)) + // processPredictions() } -// processPredictions() - /* Body */ export async function wildeford_inner(google_api_key) { - let predictions = await fetchGoogleDoc(google_api_key) - let results = await processPredictions(predictions) // somehow needed + let predictions = await fetchGoogleDoc(google_api_key); + let results = await processPredictions(predictions); // somehow needed // console.log(results.sort((a,b) => (a.title > b.title))) // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('polyprediction-questions.json', string); - await upsert(results, "wildeford-questions") - console.log("Done") + await upsert(results, "wildeford-questions"); + console.log("Done"); } //example() -export async function wildeford(){ - const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getCookie("google-api") // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey - await applyIfCookieExists(GOOGLE_API_KEY, wildeford_inner) +export async function wildeford() { + const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getCookie("google-api"); // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey + await applyIfCookieExists(GOOGLE_API_KEY, wildeford_inner); } -