quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Markets now have unique ids in preparation for dashbards.

Browse Source
This commit is contained in:
NunoSempere 2022-02-11 09:21:36 -05:00
parent d760532e95
commit 3d43aa3b6e
19 changed files with 1368 additions and 1205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
src/platforms/betfair-fetch.js
View File

@ -1,13 +1,13 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import https from 'https'; import https from "https";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
/* Utilities */ /* Utilities */
let arraysEqual = (a, b) => { let arraysEqual = (a, b) => {
@ -24,74 +24,81 @@ let arraysEqual = (a, b) => {
if (a[i] !== b[i]) return false; if (a[i] !== b[i]) return false;
} }
return true; return true;
} };
let mergeRunners = (runnerCatalog, runnerBook) => { let mergeRunners = (runnerCatalog, runnerBook) => {
let keys = Object.keys(runnerCatalog) let keys = Object.keys(runnerCatalog);
let result = [] let result = [];
for(let key of keys){ for (let key of keys) {
result.push({...runnerCatalog[key], ...runnerBook[key]}) result.push({ ...runnerCatalog[key], ...runnerBook[key] });
} }
return result return result;
} };
/* Support functions */ /* Support functions */
async function fetchPredictions() { async function fetchPredictions() {
const agent = new https.Agent({ const agent = new https.Agent({
rejectUnauthorized: false rejectUnauthorized: false,
}); });
let response = await axios({ let response = await axios({
url: endpoint, url: endpoint,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
}), },
httpsAgent: agent httpsAgent: agent,
}).then(response => response.data) }).then((response) => response.data);
return response return response;
} }
async function whipIntoShape(data){ async function whipIntoShape(data) {
let catalogues = data.market_catalogues;
let books = data.market_books;
let catalogues = data.market_catalogues let keys1 = Object.keys(catalogues).sort();
let books = data.market_books let keys2 = Object.keys(books).sort();
let keys1 = Object.keys(catalogues).sort()
let keys2 = Object.keys(books).sort()
// console.log(keys1) // console.log(keys1)
// console.log(keys2) // console.log(keys2)
let results = [] let results = [];
if(!arraysEqual(keys1, keys2)){ if (!arraysEqual(keys1, keys2)) {
throw new Error("Betfair: Error in endpoint; Betfair catalogues and books do not match") throw new Error(
}else{ "Betfair: Error in endpoint; Betfair catalogues and books do not match"
for(let key of keys1){ );
results.push({...catalogues[key], ...books[key], options: mergeRunners(catalogues[key].runners, books[key].runners)}) } else {
for (let key of keys1) {
results.push({
...catalogues[key],
...books[key],
options: mergeRunners(catalogues[key].runners, books[key].runners),
});
} }
} }
return results return results;
} }
async function processPredictions(data) { async function processPredictions(data) {
let predictions = await whipIntoShape(data) let predictions = await whipIntoShape(data);
// console.log(JSON.stringify(predictions, null, 4)) // console.log(JSON.stringify(predictions, null, 4))
let results = predictions.map(prediction => { let results = predictions.map((prediction) => {
/* if(Math.floor(Math.random() * 10) % 20 ==0){ /* if(Math.floor(Math.random() * 10) % 20 ==0){
console.log(JSON.stringify(prediction, null, 4)) console.log(JSON.stringify(prediction, null, 4))
} */ } */
let id = `betfair-${prediction.marketId}`;
let normalizationFactor = (prediction.options let normalizationFactor = prediction.options
.filter(option => option.status == "ACTIVE" && option.totalMatched > 0) .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
.map(option => option.lastPriceTraded)) .map((option) => option.lastPriceTraded)
.map(x => 1/x) .map((x) => 1 / x)
.reduce((a, b) => a + b, 0) .reduce((a, b) => a + b, 0);
let options = prediction.options let options = prediction.options
.filter(option => option.status == "ACTIVE" && option.totalMatched > 0) .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
.map(option => ({ .map((option) => ({
"name": option.runnerName, name: option.runnerName,
"probability": option.lastPriceTraded !=0 ? (1/option.lastPriceTraded)/normalizationFactor : 0, // https://www.aceodds.com/bet-calculator/odds-converter.html probability:
"type": "PROBABILITY" option.lastPriceTraded != 0
})) ? 1 / option.lastPriceTraded / normalizationFactor
: 0, // https://www.aceodds.com/bet-calculator/odds-converter.html
type: "PROBABILITY",
}));
// console.log(prediction.options) // console.log(prediction.options)
@ -102,41 +109,42 @@ async function processPredictions(data) {
.replace(/<b>/g, " ") .replace(/<b>/g, " ")
.replace(/<\/b>/g, " ") .replace(/<\/b>/g, " ")
.replace(/\n/g, " ") .replace(/\n/g, " ")
.trim() .trim();
if(rules == undefined){ if (rules == undefined) {
// console.log(prediction.description) // console.log(prediction.description)
} }
let title = rules.split("? ")[0] + "?" let title = rules.split("? ")[0] + "?";
let description = rules.split("? ")[1].trim() let description = rules.split("? ")[1].trim();
if(title.includes("of the named")){ if (title.includes("of the named")) {
title = prediction.marketName + ": "+ title title = prediction.marketName + ": " + title;
} }
let result = ({ let result = {
"title": title, id: id,
"url": `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`, title: title,
"platform": "Betfair", url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
"description": description, platform: "Betfair",
"options": options, description: description,
"timestamp": new Date().toISOString(), options: options,
"qualityindicators": { timestamp: new Date().toISOString(),
"stars": calculateStars("Betfair", ({volume: prediction.totalMatched})), qualityindicators: {
"volume": prediction.totalMatched, stars: calculateStars("Betfair", { volume: prediction.totalMatched }),
} volume: prediction.totalMatched,
}) },
return result };
}) return result;
return results //resultsProcessed });
return results; //resultsProcessed
} }
/* Body */ /* Body */
export async function betfair() { export async function betfair() {
let data = await fetchPredictions() let data = await fetchPredictions();
let results = await processPredictions(data) // somehow needed let results = await processPredictions(data); // somehow needed
// console.log(results.map(result => ({title: result.title, description: result.description}))) // console.log(results.map(result => ({title: result.title, description: result.description})))
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string); // fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "betfair-questions") await upsert(results, "betfair-questions");
console.log("Done") console.log("Done");
} }
// betfair() // betfair()

146
src/platforms/deprecated/ladbrokes-fetch.js Normal file
View File

@ -0,0 +1,146 @@
/* Imports */
import axios from "axios";
import fs from "fs";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`;
let enpointDrillDown = (id) =>
`https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`;
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
/* Support functions */
async function fetchUrl(url) {
let response = await axios(url, {
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0",
},
method: "GET",
mode: "cors",
}).then((response) => response.data);
return response;
}
let processResults = async (json) => {
let results = [];
let children = json.SSResponse.children;
children.pop();
let ids = children.map((child) => child.event.id);
let markets = [];
for (let id of ids) {
let marketsContainer = await fetchUrl(enpointDrillDown(id));
let marketsObj = marketsContainer.SSResponse.children[0].event;
let newMarkets = marketsObj.children;
newMarkets = newMarkets.map((market) => ({
...market.market,
parent: marketsObj.name,
}));
markets.push(...newMarkets);
}
let normalMarkets = markets.filter(
(market) => !market.name.includes("Specials")
);
//console.log(normalMarkets)
for (let normalMarket of normalMarkets) {
let title = normalMarket.parent + ": " + normalMarket.name;
title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
let options = normalMarket.children.map((child) => {
let name = child.outcome.name;
let priceData = child.outcome.children[0].price;
let priceDecimal = Number(priceData.priceDec);
let probability = 1 / priceDecimal;
let option = {
name: name,
probability: probability,
type: "PROBABILITY",
};
return option;
});
// normalize probabilities
let totalValue = options
.map((element) => Number(element.probability))
.reduce((a, b) => a + b, 0);
options = options.map((element) => ({
...element,
probability: Number(element.probability) / totalValue,
}));
// Filter very unlikely probabilities: Not here, but on the front end
// options = options.filter(element => element.probability > 0.02)
let obj = {
title: title,
url: "https://sports.ladbrokes.com/sport/politics/outrights",
platform: "Ladbrokes",
description: "",
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Ladbrokes", {}),
},
};
results.push(obj);
}
let specialMarkets = markets.filter((market) =>
market.name.includes("Specials")
);
for (let specialMarket of specialMarkets) {
//let title = specialMarket.parent + ": " + specialMarket.name
//console.log(title)
specialMarket.children.forEach((child) => {
let name = specialMarket.parent.includes("Specials")
? child.outcome.name
: specialMarket.parent + ": " + child.outcome.name;
name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
let priceData = child.outcome.children[0].price;
let priceDecimal = Number(priceData.priceDec);
let probability = 1 / priceDecimal;
let obj = {
title: name,
url: "https://sports.ladbrokes.com/sport/politics/outrights",
platform: "LadBrokes",
options: [
{
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
],
qualityindicators: {
stars: calculateStars("Ladbrokes", {}),
},
};
results.push(obj);
});
}
return results;
};
/* Body */
export async function ladbrokes() {
let response = await fetchUrl(endpointPolitics);
let results = await processResults(response);
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/ladbrokes-questions.json', string);
await upsert(results, "ladbrokes-questions");
console.log("Done");
}
//ladbrokes()

0
src/platforms/williamhill-fetch.js → src/platforms/deprecated/williamhill-fetch.js
View File

86
src/platforms/example-fetch.js
View File

@ -1,70 +1,70 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let endpoint = 'https://example.com/' let endpoint = "https://example.com/";
/* Support functions */ /* Support functions */
async function fetchData() { async function fetchData() {
let response = await axios({ let response = await axios({
url: endpoint, url: endpoint,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
}), },
}) }).then((response) => response.data);
.then(response => response.data)
// console.log(response) // console.log(response)
return response return response;
} }
async function processPredictions(predictions) { async function processPredictions(predictions) {
let results = await predictions.map(prediction => { let results = await predictions.map((prediction) => {
let probability = prediction.probability let id = `platform-${prediction.id}`;
let probability = prediction.probability;
let options = [ let options = [
{ {
"name": "Yes", name: "Yes",
"probability": probability, probability: probability,
"type": "PROBABILITY" type: "PROBABILITY",
}, },
{ {
"name": "No", name: "No",
"probability": 1 - probability, probability: 1 - probability,
"type": "PROBABILITY" type: "PROBABILITY",
} },
] ];
let result = ({ let result = {
"title": prediction.title, title: prediction.title,
"url": `https://example.com`, url: `https://example.com`,
"platform": "Example", platform: "Example",
"description": prediction.description, description: prediction.description,
"options": options, options: options,
"timestamp": new Date().toISOString(), timestamp: new Date().toISOString(),
"qualityindicators": { qualityindicators: {
"stars": calculateStars("Example", ({some: somex, factors: factors})), stars: calculateStars("Example", { some: somex, factors: factors }),
"other": prediction.otherx, other: prediction.otherx,
"indicators": prediction.indicatorx indicators: prediction.indicatorx,
} },
}) };
return result return result;
}) });
return results //resultsProcessed return results; //resultsProcessed
} }
/* Body */ /* Body */
export async function example() { export async function example() {
let data = await fetchData() let data = await fetchData();
let results = await processPredictions(data) // somehow needed let results = await processPredictions(data); // somehow needed
// console.log(results) // console.log(results)
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string); // fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "example-questions") await upsert(results, "example-questions");
console.log("Done") console.log("Done");
} }
//example() //example()

169
src/platforms/fantasyscotus-fetch.js
View File

@ -1,113 +1,124 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let unixtime = new Date() .getTime() let unixtime = new Date().getTime();
let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=0&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=12&_=${unixtime}` let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=0&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=12&_=${unixtime}`;
async function fetchData() { async function fetchData() {
let response = await axios({ let response = await axios({
"url": endpoint, url: endpoint,
"credentials": "omit", credentials: "omit",
"headers": { headers: {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", "User-Agent":
"Accept": "application/json, text/javascript, */*; q=0.01", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
"Accept-Language": "en-US,en;q=0.5", Accept: "application/json, text/javascript, */*; q=0.01",
"Content-Type": "application/x-www-form-urlencoded", "Accept-Language": "en-US,en;q=0.5",
"X-Requested-With": "XMLHttpRequest" "Content-Type": "application/x-www-form-urlencoded",
}, "X-Requested-With": "XMLHttpRequest",
"referrer": "https://fantasyscotus.net/case/list/", },
"method": "GET", referrer: "https://fantasyscotus.net/case/list/",
"mode": "cors" method: "GET",
}) mode: "cors",
.then(res => res.data) }).then((res) => res.data);
//console.log(response) //console.log(response)
return response return response;
} }
async function getPredictionsData(caseUrl) { async function getPredictionsData(caseUrl) {
let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}` let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`;
//console.log(newCaseUrl) //console.log(newCaseUrl)
let predictions = await axios({ let predictions = await axios({
"url": newCaseUrl, url: newCaseUrl,
"credentials": "include", credentials: "include",
"headers": { headers: {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", "User-Agent":
"Accept": "application/json, text/javascript, */*; q=0.01", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
"Accept-Language": "en-US,en;q=0.5", Accept: "application/json, text/javascript, */*; q=0.01",
"Content-Type": "application/x-www-form-urlencoded", "Accept-Language": "en-US,en;q=0.5",
"X-Requested-With": "XMLHttpRequest" "Content-Type": "application/x-www-form-urlencoded",
}, "X-Requested-With": "XMLHttpRequest",
"referrer": newCaseUrl, },
"method": "GET", referrer: newCaseUrl,
"mode": "cors" method: "GET",
}) mode: "cors",
.then(res => res.data) }).then((res) => res.data);
let predictionsAffirm = predictions.filter(prediction => prediction.percent_affirm > 50) let predictionsAffirm = predictions.filter(
(prediction) => prediction.percent_affirm > 50
);
//console.log(predictions) //console.log(predictions)
//console.log(predictionsAffirm.length/predictions.length) //console.log(predictionsAffirm.length/predictions.length)
return ({ return {
numAffirm: predictionsAffirm.length, numAffirm: predictionsAffirm.length,
proportionAffirm: predictionsAffirm.length/predictions.length, proportionAffirm: predictionsAffirm.length / predictions.length,
numForecasts: predictions.length numForecasts: predictions.length,
}) };
} }
async function processData(data) { async function processData(data) {
let events = data.object_list let events = data.object_list;
let historicalPercentageCorrect = data.stats.pcnt_correct let historicalPercentageCorrect = data.stats.pcnt_correct;
let historicalProbabilityCorrect = Number(historicalPercentageCorrect.replace("%",""))/100 let historicalProbabilityCorrect =
let results = [] Number(historicalPercentageCorrect.replace("%", "")) / 100;
for(let event of events){ let results = [];
if(event.accuracy == ''){ // if the thing hasn't already resolved for (let event of events) {
let predictionData = await getPredictionsData(event.docket_url) if (event.accuracy == "") {
let pAffirm = predictionData.proportionAffirm let id = `fantasyscotus-${event.id}`;
// if the thing hasn't already resolved
let predictionData = await getPredictionsData(event.docket_url);
let pAffirm = predictionData.proportionAffirm;
//let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect //let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
let eventObject = ({ let eventObject = {
"title": `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`, id: id,
"url": `https://fantasyscotus.net/user-predictions${event.docket_url}`, title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
"platform": "FantasySCOTUS", url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,
"description": `${(pAffirm*100).toFixed(2)}% (${predictionData.numAffirm} out of ${predictionData.numForecasts}) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of ${event.prediction}. Historically, FantasySCOTUS has chosen the correct side ${historicalPercentageCorrect} of the time.`, platform: "FantasySCOTUS",
"options": [ description: `${(pAffirm * 100).toFixed(2)}% (${
{ predictionData.numAffirm
"name": "Yes", } out of ${
"probability": pAffirm, predictionData.numForecasts
"type": "PROBABILITY" }) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of ${
}, event.prediction
{ }. Historically, FantasySCOTUS has chosen the correct side ${historicalPercentageCorrect} of the time.`,
"name": "No", options: [
"probability": 1-pAffirm, {
"type": "PROBABILITY" name: "Yes",
} probability: pAffirm,
], type: "PROBABILITY",
"timestamp": new Date().toISOString(), },
"qualityindicators": { {
"numforecasts": Number(predictionData.numForecasts), name: "No",
"stars": calculateStars("FantasySCOTUS", ({})) probability: 1 - pAffirm,
} type: "PROBABILITY",
}) },
],
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(predictionData.numForecasts),
stars: calculateStars("FantasySCOTUS", {}),
},
};
// console.log(eventObject) // console.log(eventObject)
results.push(eventObject) results.push(eventObject);
} }
} }
return results return results;
} }
/* Body */ /* Body */
export async function fantasyscotus() { export async function fantasyscotus() {
let rawData = await fetchData() let rawData = await fetchData();
let results = await processData(rawData) let results = await processData(rawData);
//console.log(results) //console.log(results)
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/fantasyscotus-questions.json', string); // fs.writeFileSync('./data/fantasyscotus-questions.json', string);
await upsert(results, "fantasyscotus-questions") await upsert(results, "fantasyscotus-questions");
console.log("Done") console.log("Done");
} }
//fantasyscotus() //fantasyscotus()

104
src/platforms/foretold-fetch.js
View File

@ -1,20 +1,26 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let graphQLendpoint = "https://api.foretold.io/graphql" let graphQLendpoint = "https://api.foretold.io/graphql";
let highQualityCommunities = ["0104d8e8-07e4-464b-8b32-74ef22b49f21", "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", "cf663021-f87f-4632-ad82-962d889a2d39", "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", "b2412a1d-0aa4-4e37-a12a-0aca9e440a96"] let highQualityCommunities = [
"0104d8e8-07e4-464b-8b32-74ef22b49f21",
"c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2",
"cf663021-f87f-4632-ad82-962d889a2d39",
"47ff5c49-9c20-4f3d-bd57-1897c35cd42d",
"b2412a1d-0aa4-4e37-a12a-0aca9e440a96",
];
/* Support functions */ /* Support functions */
async function fetchAllCommunityQuestions(communityId) { async function fetchAllCommunityQuestions(communityId) {
let response = await axios({ let response = await axios({
url: graphQLendpoint, url: graphQLendpoint,
method: 'POST', method: "POST",
headers: ({ 'Content-Type': 'application/json' }), headers: { "Content-Type": "application/json" },
data: JSON.stringify(({ data: JSON.stringify({
query: ` query: `
query { query {
measurables( measurables(
@ -38,62 +44,64 @@ async function fetchAllCommunityQuestions(communityId) {
} }
} }
} }
` `,
})), }),
}) })
.then(res => res.data) .then((res) => res.data)
.then(res => res.data.measurables.edges) .then((res) => res.data.measurables.edges);
//console.log(response) //console.log(response)
return response return response;
} }
/* Body */ /* Body */
export async function foretold(){ export async function foretold() {
let results = [] let results = [];
for(let community of highQualityCommunities){ for (let community of highQualityCommunities) {
let questions = await fetchAllCommunityQuestions(community) let questions = await fetchAllCommunityQuestions(community);
questions = questions.map(question => question.node) questions = questions.map((question) => question.node);
questions = questions.filter(question => question.previousAggregate) // Questions without any predictions questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
questions.forEach(question => { questions.forEach((question) => {
let options = [] let id = `foretold-${question.id}`;
if(question.valueType == "PERCENTAGE"){ let options = [];
let probability = question.previousAggregate.value.percentage if (question.valueType == "PERCENTAGE") {
let probability = question.previousAggregate.value.percentage;
options = [ options = [
{ {
"name": "Yes", name: "Yes",
"probability": probability/100, probability: probability / 100,
"type": "PROBABILITY" type: "PROBABILITY",
}, },
{ {
"name": "No", name: "No",
"probability": 1-probability/100, probability: 1 - probability / 100,
"type": "PROBABILITY" type: "PROBABILITY",
} },
] ];
} }
let result = { let result = {
"title": question.name, id: id,
"url": `https://www.foretold.io/c/${community}/m/${question.id}`, title: question.name,
"platform": "Foretold", url: `https://www.foretold.io/c/${community}/m/${question.id}`,
"description": "", platform: "Foretold",
"options": options, description: "",
"timestamp": new Date().toISOString(), options: options,
"qualityindicators": { timestamp: new Date().toISOString(),
"numforecasts": Math.floor(Number(question.measurementCount) / 2), qualityindicators: {
"stars": calculateStars("Foretold", ({ })) numforecasts: Math.floor(Number(question.measurementCount) / 2),
} stars: calculateStars("Foretold", {}),
/*liquidity: liquidity.toFixed(2), },
/*liquidity: liquidity.toFixed(2),
tradevolume: tradevolume.toFixed(2), tradevolume: tradevolume.toFixed(2),
address: obj.address*/ address: obj.address*/
} };
// console.log(result) // console.log(result)
results.push(result) results.push(result);
}) });
} }
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/foretold-questions.json', string); // fs.writeFileSync('./data/foretold-questions.json', string);
await upsert(results, "foretold-questions") await upsert(results, "foretold-questions");
console.log("Done") console.log("Done");
} }
// foretold() // foretold()

74
src/platforms/givewellopenphil-fetch.js
View File

@ -1,61 +1,69 @@
/* Imports */ /* Imports */
import fs from "fs" import fs from "fs";
import axios from "axios" import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let locationData = "./data/" let locationData = "./data/";
/* Support functions */ /* Support functions */
async function fetchPage(url) { async function fetchPage(url) {
let response = await axios({ let response = await axios({
url: url, url: url,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
}), },
}) }).then((res) => res.data);
.then(res => res.data)
//console.log(response) //console.log(response)
return response return response;
} }
/* Body */ /* Body */
async function main() { async function main() {
let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt") let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt");
let data = rawdata.toString().split("\n").filter(url => url != ""); let data = rawdata
.toString()
.split("\n")
.filter((url) => url != "");
// console.log(data) // console.log(data)
let results = [] let results = [];
for (let url of data) { for (let url of data) {
// console.log(url) // console.log(url)
let page = await fetchPage(url) let page = await fetchPage(url);
// Title // Title
let titleraw = page.split('<meta name="twitter:title" content="')[1] let titleraw = page.split('<meta name="twitter:title" content="')[1];
let title = titleraw.split('" />')[0] let title = titleraw.split('" />')[0];
// Description // Description
let internalforecasts = page.split("<h2").filter(section => section.includes("Internal forecast") || section.includes("internal forecast")) let internalforecasts = page
let description = "<h2 " + internalforecasts[1] .split("<h2")
.filter(
(section) =>
section.includes("Internal forecast") ||
section.includes("internal forecast")
);
let description = "<h2 " + internalforecasts[1];
let result = { let result = {
"title": title, title: title,
"url": url, url: url,
"platform": "GiveWell", platform: "GiveWell",
"description": description, description: description,
"timestamp": new Date().toISOString(), timestamp: new Date().toISOString(),
"qualityindicators": { qualityindicators: {
"stars": calculateStars("GiveWell/OpenPhilanthropy", ({})), stars: calculateStars("GiveWell/OpenPhilanthropy", {}),
} },
} // Note: This requires some processing afterwards }; // Note: This requires some processing afterwards
// console.log(result) // console.log(result)
results.push(result) results.push(result);
} }
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/givewell-questions-unprocessed.json', string); // fs.writeFileSync('./data/givewell-questions-unprocessed.json', string);
await upsert(results, "givewell-questions-unprocessed") await upsert(results, "givewell-questions-unprocessed");
} }
main() main();

96
src/platforms/goodjudgment-fetch.js
View File

@ -6,6 +6,7 @@ import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js"; import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js"; import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js"; import { upsert } from "../utils/mongo-wrapper.js";
import { hash } from "../utils/hash.js";
/* Definitions */ /* Definitions */
let endpoint = "https://goodjudgment.io/superforecasts/"; let endpoint = "https://goodjudgment.io/superforecasts/";
@ -63,51 +64,56 @@ export async function goodjudgment() {
for (let table of jsonTable) { for (let table of jsonTable) {
// console.log(table) // console.log(table)
let title = table[0]["0"].split("\t\t\t").splice(3)[0]; let title = table[0]["0"].split("\t\t\t").splice(3)[0];
let description = table if (title != undefined) {
.filter((row) => row["0"].includes("BACKGROUND:")) title = title.replaceAll("</a>", "");
.map((row) => row["0"]) let id = `goodjudgment-${hash(title)}`;
.map((text) => let description = table
text .filter((row) => row["0"].includes("BACKGROUND:"))
.split("BACKGROUND:")[1] .map((row) => row["0"])
.split("Examples of Superforecaster")[0] .map((text) =>
.split("AT A GLANCE")[0] text
.replaceAll("\n\n", "\n") .split("BACKGROUND:")[1]
.split("\n") .split("Examples of Superforecaster")[0]
.slice(3) .split("AT A GLANCE")[0]
.join(" ") .replaceAll("\n\n", "\n")
.replaceAll(" ", "") .split("\n")
.replaceAll("<br> ", "") .slice(3)
)[0]; .join(" ")
let options = table .replaceAll(" ", "")
.filter((row) => "4" in row) .replaceAll("<br> ", "")
.map((row) => ({ )[0];
name: row["2"].split('<span class="qTitle">')[1].replace("</span>", ""), let options = table
probability: Number(row["3"].split("%")[0]) / 100, .filter((row) => "4" in row)
type: "PROBABILITY", .map((row) => ({
})); name: row["2"]
let analysis = table.filter((row) => .split('<span class="qTitle">')[1]
row[0] ? row[0].toLowerCase().includes("commentary") : false .replace("</span>", ""),
); probability: Number(row["3"].split("%")[0]) / 100,
// "Examples of Superforecaster Commentary" / Analysis type: "PROBABILITY",
// The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list. }));
analysis = analysis ? analysis[0] : ""; let analysis = table.filter((row) =>
analysis = analysis ? analysis[0] : ""; row[0] ? row[0].toLowerCase().includes("commentary") : false
// console.log(analysis) );
let standardObj = { // "Examples of Superforecaster Commentary" / Analysis
title: title, // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list.
url: endpoint, analysis = analysis ? analysis[0] : "";
platform: "Good Judgment", analysis = analysis ? analysis[0] : ""; // not a duplicate
description: description, // console.log(analysis)
options: options, let standardObj = {
timestamp: new Date().toISOString(), id: id,
qualityindicators: { title: title,
stars: calculateStars("Good Judgment", {}), url: endpoint,
}, platform: "Good Judgment",
extra: { description: description,
superforecastercommentary: analysis || "", options: options,
}, timestamp: new Date().toISOString(),
}; qualityindicators: {
if (standardObj.title != undefined) { stars: calculateStars("Good Judgment", {}),
},
extra: {
superforecastercommentary: analysis || "",
},
};
results.push(standardObj); results.push(standardObj);
} }
} }

293
src/platforms/goodjudmentopen-fetch.js
View File

@ -1,211 +1,242 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import { getCookie, applyIfCookieExists } from "../utils/getCookies.js";
import { Tabletojson } from "tabletojson" import { Tabletojson } from "tabletojson";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { upsert } from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let htmlEndPoint = 'https://www.gjopen.com/questions?page=' let htmlEndPoint = "https://www.gjopen.com/questions?page=";
let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"] let annoyingPromptUrls = [
const DEBUG_MODE = "off" // "on" "https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
const id = x => x "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters",
"https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
"https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
];
const DEBUG_MODE = "off"; // "on"
const id = (x) => x;
/* Support functions */ /* Support functions */
async function fetchPage(page, cookie) { async function fetchPage(page, cookie) {
let response = await axios({ let response = await axios({
url: htmlEndPoint + page, url: htmlEndPoint + page,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
'Cookie': cookie Cookie: cookie,
}), },
}) }).then((res) => res.data);
.then(res => res.data)
//console.log(response) //console.log(response)
return response return response;
} }
async function fetchStats(questionUrl, cookie) { async function fetchStats(questionUrl, cookie) {
let response = await axios({ let response = await axios({
url: questionUrl + "/stats", url: questionUrl + "/stats",
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
'Cookie': cookie, Cookie: cookie,
'Referer': questionUrl, Referer: questionUrl,
}), },
}) }).then((res) => res.data);
.then(res => res.data)
//console.log(response) //console.log(response)
// Is binary? // Is binary?
let isbinary = response.includes("binary?&quot;:true") let isbinary = response.includes("binary?&quot;:true");
let options = [] let options = [];
if (isbinary) { if (isbinary) {
// Crowd percentage // Crowd percentage
let htmlElements = response.split("\n") let htmlElements = response.split("\n");
let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0] let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
// console.log(h3Element) // console.log(h3Element)
let crowdpercentage = h3Element.split(">")[1].split("<")[0] let crowdpercentage = h3Element.split(">")[1].split("<")[0];
let probability = Number(crowdpercentage.replace("%", "")) / 100 let probability = Number(crowdpercentage.replace("%", "")) / 100;
options.push(({ options.push(
name: "Yes", {
probability: probability, name: "Yes",
type: "PROBABILITY" probability: probability,
}), ({ type: "PROBABILITY",
name: "No", },
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans {
type: "PROBABILITY" name: "No",
})) probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY",
}
);
} else { } else {
let optionsHtmlElement = "<table" + response.split("tbody")[1] + "table>" let optionsHtmlElement = "<table" + response.split("tbody")[1] + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement) let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0] let firstTable = tablesAsJson[0];
options = firstTable.map(element => ({ options = firstTable.map((element) => ({
name: element['0'], name: element["0"],
probability: Number(element['1'].replace("%", "")) / 100, probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY" type: "PROBABILITY",
})) }));
//console.log(optionsHtmlElement) //console.log(optionsHtmlElement)
//console.log(options) //console.log(options)
} }
// Description // Description
let descriptionraw = response.split(`<div id="question-background" class="collapse smb">`)[1] let descriptionraw = response.split(
let descriptionprocessed1 = descriptionraw.split(`</div>`)[0] `<div id="question-background" class="collapse smb">`
let descriptionprocessed2 = toMarkdown(descriptionprocessed1) )[1];
let descriptionprocessed3 = descriptionprocessed2.split("\n") let descriptionprocessed1 = descriptionraw.split(`</div>`)[0];
.filter(string => !string.includes("Confused? Check our")) let descriptionprocessed2 = toMarkdown(descriptionprocessed1);
.join("\n") let descriptionprocessed3 = descriptionprocessed2
let description = descriptionprocessed3 .split("\n")
.filter((string) => !string.includes("Confused? Check our"))
.join("\n");
let description = descriptionprocessed3;
// Number of forecasts // Number of forecasts
let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0] let numforecasts = response
.split("prediction_sets_count&quot;:")[1]
.split(",")[0];
//console.log(numforecasts) //console.log(numforecasts)
// Number of predictors // Number of predictors
let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0] let numforecasters = response
.split("predictors_count&quot;:")[1]
.split(",")[0];
//console.log(numpredictors) //console.log(numpredictors)
// Calculate the stars // Calculate the stars
let minProbability = Math.min(...options.map(option => option.probability)) let minProbability = Math.min(...options.map((option) => option.probability));
let maxProbability = Math.max(...options.map(option => option.probability)) let maxProbability = Math.max(...options.map((option) => option.probability));
let result = { let result = {
"description": description, description: description,
"options": options, options: options,
"timestamp": new Date().toISOString(), timestamp: new Date().toISOString(),
"qualityindicators": { qualityindicators: {
"numforecasts": Number(numforecasts), numforecasts: Number(numforecasts),
"numforecasters": Number(numforecasters), numforecasters: Number(numforecasters),
"stars": calculateStars("Good Judgment Open", ({ numforecasts, minProbability, maxProbability })) stars: calculateStars("Good Judgment Open", {
} numforecasts,
} minProbability,
return result maxProbability,
}),
},
};
return result;
} }
function isSignedIn(html){ function isSignedIn(html) {
let isSignedInBool = !(
let isSignedInBool = !( html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") ) html.includes("You need to sign in or sign up before continuing") ||
html.includes("Sign up")
);
// console.log(html) // console.log(html)
if(!isSignedInBool){ if (!isSignedInBool) {
console.log("Error: Not signed in.") console.log("Error: Not signed in.");
} }
console.log(`is signed in? ${isSignedInBool}`) console.log(`is signed in? ${isSignedInBool}`);
return isSignedInBool return isSignedInBool;
} }
function isEnd(html){ function isEnd(html) {
let isEndBool = html.includes("No questions match your filter") let isEndBool = html.includes("No questions match your filter");
if(isEndBool){ if (isEndBool) {
//console.log(html) //console.log(html)
} }
console.log(`IsEnd? ${isEndBool}`) console.log(`IsEnd? ${isEndBool}`);
return isEndBool return isEndBool;
} }
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
/* Body */ /* Body */
async function goodjudgmentopen_inner(cookie) { async function goodjudgmentopen_inner(cookie) {
let i = 1 let i = 1;
let response = await fetchPage(i, cookie) let response = await fetchPage(i, cookie);
let results = [] let results = [];
let init = Date.now() let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.") // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while(!isEnd(response) && isSignedIn(response)){ while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n") let htmlLines = response.split("\n");
DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id() DEBUG_MODE == "on" ? htmlLines.forEach((line) => console.log(line)) : id();
let h5elements = htmlLines.filter(str => str.includes("<h5> <a href=")) let h5elements = htmlLines.filter((str) => str.includes("<h5> <a href="));
DEBUG_MODE == "on" ? console.log(h5elements) : id() DEBUG_MODE == "on" ? console.log(h5elements) : id();
let j = 0 let j = 0;
for (let h5element of h5elements) { for (let h5element of h5elements) {
let h5elementSplit = h5element.split('"><span>') let h5elementSplit = h5element.split('"><span>');
let url = h5elementSplit[0].split('<a href="')[1] let url = h5elementSplit[0].split('<a href="')[1];
if(!annoyingPromptUrls.includes(url)){ if (!annoyingPromptUrls.includes(url)) {
let title = h5elementSplit[1].replace('</span></a></h5>', "") let title = h5elementSplit[1].replace("</span></a></h5>", "");
await sleep(1000 + Math.random() * 1000) // don't be as noticeable await sleep(1000 + Math.random() * 1000); // don't be as noticeable
try { try {
let moreinfo = await fetchStats(url, cookie) let moreinfo = await fetchStats(url, cookie);
if (moreinfo.isbinary) { if (moreinfo.isbinary) {
if (!moreinfo.crowdpercentage) { // then request again. if (!moreinfo.crowdpercentage) {
moreinfo = await fetchStats(url, cookie) // then request again.
moreinfo = await fetchStats(url, cookie);
} }
} }
let question = ({ let questionNumRegex = new RegExp("questions/([0-9]+)");
"title": title, let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
"url": url, let id = `goodjudmentopen-${questionNum}`;
"platform": "Good Judgment Open", let question = {
...moreinfo id: id,
}) title: title,
if(j % 30 == 0 || DEBUG_MODE == "on"){ url: url,
console.log(`Page #${i}`) platform: "Good Judgment Open",
console.log(question) ...moreinfo,
};
if (j % 30 == 0 || DEBUG_MODE == "on") {
console.log(`Page #${i}`);
console.log(question);
} }
// console.log(question) // console.log(question)
results.push(question) results.push(question);
} catch (error) { } catch (error) {
console.log(error) console.log(error);
console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`) console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
} }
} }
j = j+1 j = j + 1;
} }
i = i + 1 i = i + 1;
// console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers") // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
await sleep(5000 + Math.random() * 1000) // don't be a dick to gjopen server await sleep(5000 + Math.random() * 1000); // don't be a dick to gjopen server
try { try {
response = await fetchPage(i, cookie) response = await fetchPage(i, cookie);
} catch (error) { } catch (error) {
console.log(error) console.log(error);
console.log(`We encountered some error when fetching page #${i}, so it won't appear on the final json`) console.log(
`We encountered some error when fetching page #${i}, so it won't appear on the final json`
);
} }
} }
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/goodjudmentopen-questions.json', string); // fs.writeFileSync('./data/goodjudmentopen-questions.json', string);
console.log(results) console.log(results);
if(results.length > 0){ if (results.length > 0) {
await upsert(results, "goodjudmentopen-questions") await upsert(results, "goodjudmentopen-questions");
}else{ } else {
console.log("Not updating results, as process was not signed in") console.log("Not updating results, as process was not signed in");
} }
let end = Date.now() let end = Date.now();
let difference = end - init let difference = end - init;
console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`) console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
} }
export async function goodjudgmentopen(){ export async function goodjudgmentopen() {
let cookie = process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudmentopen") let cookie =
await applyIfCookieExists(cookie, goodjudgmentopen_inner) process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudmentopen");
await applyIfCookieExists(cookie, goodjudgmentopen_inner);
} }

315
src/platforms/infer-fetch.js
View File

@ -1,187 +1,201 @@
/* Imports */ /* Imports */
import axios from "axios" import axios from "axios";
import { getCookie, applyIfCookieExists } from "../utils/getCookies.js" import { getCookie, applyIfCookieExists } from "../utils/getCookies.js";
import { Tabletojson } from "tabletojson" import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let htmlEndPoint = 'https://www.infer-pub.com/questions' let htmlEndPoint = "https://www.infer-pub.com/questions";
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } String.prototype.replaceAll = function replaceAll(search, replace) {
const DEBUG_MODE = "on"// "off" return this.split(search).join(replace);
const SLEEP_TIME_RANDOM = 7000 // miliseconds };
const SLEEP_TIME_EXTRA = 2000 const DEBUG_MODE = "on"; // "off"
const SLEEP_TIME_RANDOM = 7000; // miliseconds
const SLEEP_TIME_EXTRA = 2000;
/* Support functions */ /* Support functions */
async function fetchPage(page, cookie) { async function fetchPage(page, cookie) {
console.log(`Page #${page}`) console.log(`Page #${page}`);
if (page == 1) { if (page == 1) {
cookie = cookie.split(";")[0] // Interesting that it otherwise doesn't work :( cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :(
} }
let urlEndpoint = `${htmlEndPoint}/?page=${page}` let urlEndpoint = `${htmlEndPoint}/?page=${page}`;
console.log(urlEndpoint) console.log(urlEndpoint);
let response = await axios({ let response = await axios({
url: urlEndpoint, url: urlEndpoint,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
'Cookie': cookie Cookie: cookie,
}), },
}) }).then((res) => res.data);
.then(res => res.data)
// console.log(response) // console.log(response)
return response return response;
} }
async function fetchStats(questionUrl, cookie) { async function fetchStats(questionUrl, cookie) {
let response = await axios({ let response = await axios({
url: questionUrl + "/stats", url: questionUrl + "/stats",
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
'Cookie': cookie, Cookie: cookie,
'Referer': questionUrl, Referer: questionUrl,
}), },
}) }).then((res) => res.data);
.then(res => res.data)
if (response.includes("Sign up or sign in to forecast")) { if (response.includes("Sign up or sign in to forecast")) {
throw Error("Not logged in") throw Error("Not logged in");
} }
// Is binary? // Is binary?
let isbinary = response.includes("binary?&quot;:true") let isbinary = response.includes("binary?&quot;:true");
// console.log(`is binary? ${isbinary}`) // console.log(`is binary? ${isbinary}`)
let options = [] let options = [];
if (isbinary) { if (isbinary) {
// Crowd percentage // Crowd percentage
let htmlElements = response.split("\n") let htmlElements = response.split("\n");
// DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id() // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0] let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
// DEBUG_MODE == "on" ? console.log(h5elements) : id() // DEBUG_MODE == "on" ? console.log(h5elements) : id()
let crowdpercentage = h3Element.split(">")[1].split("<")[0] let crowdpercentage = h3Element.split(">")[1].split("<")[0];
let probability = Number(crowdpercentage.replace("%", "")) / 100 let probability = Number(crowdpercentage.replace("%", "")) / 100;
options.push(({ options.push(
name: "Yes", {
probability: probability, name: "Yes",
type: "PROBABILITY" probability: probability,
}), ({ type: "PROBABILITY",
name: "No", },
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans {
type: "PROBABILITY" name: "No",
})) probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY",
}
);
} else { } else {
try { try {
let optionsBody = response.split("tbody")[1] // Previously [1], then previously [3] but they added a new table. let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table.
// console.log(optionsBody) // console.log(optionsBody)
let optionsHtmlElement = "<table" + optionsBody + "table>" let optionsHtmlElement = "<table" + optionsBody + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement) let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0] let firstTable = tablesAsJson[0];
options = firstTable.map(element => ({ options = firstTable.map((element) => ({
name: element['0'], name: element["0"],
probability: Number(element['1'].replace("%", "")) / 100, probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY" type: "PROBABILITY",
})) }));
} catch (error) { } catch (error) {
let optionsBody = response.split("tbody")[3] // Catch if the error is related to table position let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position
let optionsHtmlElement = "<table" + optionsBody + "table>" let optionsHtmlElement = "<table" + optionsBody + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement) let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0] let firstTable = tablesAsJson[0];
if (firstTable) { if (firstTable) {
options = firstTable.map(element => ({ options = firstTable.map((element) => ({
name: element['0'], name: element["0"],
probability: Number(element['1'].replace("%", "")) / 100, probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY" type: "PROBABILITY",
})) }));
} else { } else {
// New type of question, tricky to parse the options // New type of question, tricky to parse the options
// Just leave options = [] for now. // Just leave options = [] for now.
// https://www.cset-foretell.com/blog/rolling-question-formats // https://www.cset-foretell.com/blog/rolling-question-formats
} }
} }
} }
// Description // Description
let descriptionraw = response.split(`<meta name="description" content="`)[1] let descriptionraw = response.split(`<meta name="description" content="`)[1];
let descriptionprocessed1 = descriptionraw.split(`">`)[0] let descriptionprocessed1 = descriptionraw.split(`">`)[0];
let descriptionprocessed2 = descriptionprocessed1.replace(">", "") let descriptionprocessed2 = descriptionprocessed1.replace(">", "");
let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``) let descriptionprocessed3 = descriptionprocessed2.replace(
"To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.",
``
);
// console.log(descriptionprocessed3) // console.log(descriptionprocessed3)
let descriptionprocessed4 = descriptionprocessed3.replaceAll("\r\n\r\n", "\n") let descriptionprocessed4 = descriptionprocessed3.replaceAll(
let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n") "\r\n\r\n",
let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`) "\n"
let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'") );
let descriptionprocessed8 = toMarkdown(descriptionprocessed7) let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n");
let description = descriptionprocessed8 let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`);
let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'");
let descriptionprocessed8 = toMarkdown(descriptionprocessed7);
let description = descriptionprocessed8;
// Number of forecasts // Number of forecasts
//console.log(response) //console.log(response)
//console.log(response.split("prediction_sets_count&quot;:")[1]) //console.log(response.split("prediction_sets_count&quot;:")[1])
let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0] let numforecasts = response
.split("prediction_sets_count&quot;:")[1]
.split(",")[0];
// console.log(numforecasts) // console.log(numforecasts)
// Number of predictors // Number of predictors
let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0] let numforecasters = response
.split("predictors_count&quot;:")[1]
.split(",")[0];
// console.log(numpredictors) // console.log(numpredictors)
let result = { let result = {
"description": description, description: description,
"options": options, options: options,
"timestamp": new Date().toISOString(), timestamp: new Date().toISOString(),
"qualityindicators": { qualityindicators: {
"numforecasts": Number(numforecasts), numforecasts: Number(numforecasts),
"numforecasters": Number(numforecasters), numforecasters: Number(numforecasters),
"stars": calculateStars("Infer", { numforecasts }) stars: calculateStars("Infer", { numforecasts }),
} },
} };
return result return result;
} }
function isSignedIn(html) { function isSignedIn(html) {
let isSignedInBool = !(
let isSignedInBool = !(html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up")) html.includes("You need to sign in or sign up before continuing") ||
html.includes("Sign up")
);
if (!isSignedInBool) { if (!isSignedInBool) {
console.log("Error: Not signed in.") console.log("Error: Not signed in.");
} }
console.log(`Signed in? ${isSignedInBool}`) console.log(`Signed in? ${isSignedInBool}`);
return isSignedInBool return isSignedInBool;
} }
function isEnd(html) { function isEnd(html) {
let isEndBool = html.includes("No questions match your filter") let isEndBool = html.includes("No questions match your filter");
if (isEndBool) { if (isEndBool) {
//console.log(html) //console.log(html)
} }
console.log(`IsEnd? ${isEndBool}`) console.log(`IsEnd? ${isEndBool}`);
return isEndBool return isEndBool;
} }
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
/* Body */ /* Body */
async function infer_inner(cookie) { async function infer_inner(cookie) {
let i = 1 let i = 1;
let response = await fetchPage(i, cookie) let response = await fetchPage(i, cookie);
let results = [] let results = [];
let init = Date.now() let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.") // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) { while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n");
let htmlLines = response.split("\n")
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href=")) // let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter(str => str.includes("https://www.infer-pub.com/questions/")) let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.infer-pub.com/questions/")
);
// console.log(questionHrefs) // console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log(response) //console.log(response)
console.log("questionHrefs: ") console.log("questionHrefs: ");
console.log(questionHrefs) console.log(questionHrefs);
} }
//console.log("") //console.log("")
@ -191,64 +205,83 @@ async function infer_inner(cookie) {
for (let questionHref of questionHrefs) { for (let questionHref of questionHrefs) {
//console.log(h4element) //console.log(h4element)
let elementSplit = questionHref.split('"><span>') let elementSplit = questionHref.split('"><span>');
let url = elementSplit[0].split('<a href="')[1] let url = elementSplit[0].split('<a href="')[1];
let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "") let title = elementSplit[1]
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable .replace("</h4>", "")
.replace("</h5>", "")
.replace("</span></a>", "");
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try { try {
let moreinfo = await fetchStats(url, cookie) let moreinfo = await fetchStats(url, cookie);
let question = ({ let questionNumRegex = new RegExp("questions/([0-9]+)");
"title": title, let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
"url": url, let id = `infer-${questionNum}`;
"platform": "Infer", let question = {
...moreinfo id: id,
}) title: title,
if (i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) { url: url,
console.log(`Page #${i}` && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) platform: "Infer",
console.log(question) ...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(
`Page #${i}` &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
);
console.log(question);
} }
results.push(question) results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url) console.log(url);
console.log(question) console.log(question);
} }
} catch (error) { } catch (error) {
console.log(error) console.log(error);
console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`) console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
} }
} }
i++ i++;
//i=Number(i)+1 //i=Number(i)+1
console.log("Sleeping for ~5secs so as to not be as noticeable to the infer servers") console.log(
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable "Sleeping for ~5secs so as to not be as noticeable to the infer servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try { try {
response = await fetchPage(i, cookie) response = await fetchPage(i, cookie);
} catch (error) { } catch (error) {
console.log(error) console.log(error);
console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`) console.log(
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
);
} }
} }
// let string = JSON.stringify(results,null, 2) // let string = JSON.stringify(results,null, 2)
// fs.writeFileSync('./data/infer-questions.json', string); // fs.writeFileSync('./data/infer-questions.json', string);
// console.log(results) // console.log(results)
if (results.length > 0) { if (results.length > 0) {
await upsert(results, "infer-questions") await upsert(results, "infer-questions");
} else { } else {
console.log("Not updating results, as process was not signed in") console.log("Not updating results, as process was not signed in");
} }
let end = Date.now() let end = Date.now();
let difference = end - init let difference = end - init;
console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`) console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
} }
export async function infer() { export async function infer() {
let cookie = process.env.INFER_COOKIE || getCookie("infer") let cookie = process.env.INFER_COOKIE || getCookie("infer");
await applyIfCookieExists(cookie, infer_inner) await applyIfCookieExists(cookie, infer_inner);
} }

102
src/platforms/kalshi-fetch.js
View File

@ -1,11 +1,11 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
/* Support functions /* Support functions
async function fetchAllContractInfo(){ // for info which the polymarket graphql API async function fetchAllContractInfo(){ // for info which the polymarket graphql API
@ -14,66 +14,76 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql
return response return response
} }
*/ */
async function fetchAllMarkets() { // for info which the polymarket graphql API async function fetchAllMarkets() {
let response = await axios.get(jsonEndpoint).then(response => response.data.markets) // for info which the polymarket graphql API
let response = await axios
.get(jsonEndpoint)
.then((response) => response.data.markets);
// console.log(response) // console.log(response)
return response return response;
} }
async function processMarkets(markets) { async function processMarkets(markets) {
let dateNow = new Date().toISOString() let dateNow = new Date().toISOString();
// console.log(markets) // console.log(markets)
markets = markets.filter(market => market.close_date > dateNow) markets = markets.filter((market) => market.close_date > dateNow);
let results = await markets.map(market => { let results = await markets.map((market) => {
let probability = market.last_price/100 let probability = market.last_price / 100;
let options = [ let options = [
{ {
"name": "Yes", name: "Yes",
"probability": probability, probability: probability,
"type": "PROBABILITY" type: "PROBABILITY",
}, },
{ {
"name": "No", name: "No",
"probability": 1 - probability, probability: 1 - probability,
"type": "PROBABILITY" type: "PROBABILITY",
} },
] ];
let result = ({ let id = `kalshi-${market.id}`;
"title": market.title.replaceAll("*", ""), let result = {
"url": `https://kalshi.com/markets/${market.ticker_name}`, id: id,
"platform": "Kalshi", title: market.title.replaceAll("*", ""),
"description": `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`, url: `https://kalshi.com/markets/${market.ticker_name}`,
"options": options, platform: "Kalshi",
"timestamp": new Date().toISOString(), description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
"qualityindicators": { options: options,
"stars": calculateStars("Kalshi", ({shares_volume: market.volume, interest: market.open_interest})), timestamp: new Date().toISOString(),
"yes_bid": market.yes_bid, qualityindicators: {
"yes_ask": market.yes_ask, stars: calculateStars("Kalshi", {
"spread": Math.abs(market.yes_bid-market.yes_ask), shares_volume: market.volume,
"shares_volume": market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if. interest: market.open_interest,
// "open_interest": market.open_interest, also in shares }),
} yes_bid: market.yes_bid,
}) yes_ask: market.yes_ask,
return result spread: Math.abs(market.yes_bid - market.yes_ask),
}) shares_volume: market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if.
// "open_interest": market.open_interest, also in shares
},
};
return result;
});
//console.log(results.length) //console.log(results.length)
// console.log(results.map(result => result.title)) // console.log(results.map(result => result.title))
// console.log(results.map(result => result.title).length) // console.log(results.map(result => result.title).length)
console.log([...new Set(results.map(result => result.title))]) console.log([...new Set(results.map((result) => result.title))]);
console.log("Number of unique questions: ", [...new Set(results.map(result => result.title))].length) console.log(
"Number of unique questions: ",
[...new Set(results.map((result) => result.title))].length
);
// console.log([...new Set(results.map(result => result.title))].length) // console.log([...new Set(results.map(result => result.title))].length)
return results //resultsProcessed return results; //resultsProcessed
} }
/* Body */ /* Body */
export async function kalshi() { export async function kalshi() {
let markets = await fetchAllMarkets() let markets = await fetchAllMarkets();
let results = await processMarkets(markets) // somehow needed let results = await processMarkets(markets); // somehow needed
// console.log(results) // console.log(results)
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polymarket-questions.json', string); // fs.writeFileSync('polymarket-questions.json', string);
await upsert(results, "kalshi-questions") await upsert(results, "kalshi-questions");
console.log("Done") console.log("Done");
} }
// kalshi() // kalshi()

136
src/platforms/ladbrokes-fetch.js
View File

@ -1,136 +0,0 @@
/* Imports */
import axios from "axios"
import fs from "fs"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
/* Definitions */
let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`
let enpointDrillDown = (id) => `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
/* Support functions */
async function fetchUrl(url){
let response = await axios(url, {
"credentials": "include",
"headers": {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0"
},
"method": "GET",
"mode": "cors"
}).then(response => response.data)
return response
}
let processResults = async (json) => {
let results = []
let children = json.SSResponse.children
children.pop()
let ids = children.map(child => child.event.id)
let markets = []
for(let id of ids){
let marketsContainer = await fetchUrl(enpointDrillDown(id))
let marketsObj = marketsContainer.SSResponse.children[0].event
let newMarkets = marketsObj.children
newMarkets = newMarkets.map(market => ({...market.market, parent: marketsObj.name}))
markets.push(...newMarkets)
}
let normalMarkets = markets.filter(market => !market.name.includes("Specials"))
//console.log(normalMarkets)
for(let normalMarket of normalMarkets){
let title = normalMarket.parent + ": " + normalMarket.name
title = title.replace("Boris Johnson Specials", "Boris Johnson") // small kludge
let options = normalMarket.children.map(child => {
let name = child.outcome.name
let priceData = child.outcome.children[0].price
let priceDecimal = Number(priceData.priceDec)
let probability = 1/priceDecimal
let option = ({
"name":name,
"probability": probability,
"type": "PROBABILITY"
})
return option
})
// normalize probabilities
let totalValue = options
.map(element => Number(element.probability))
.reduce((a, b) => (a + b), 0)
options = options.map(element => ({
...element,
probability: Number(element.probability) / totalValue
}))
// Filter very unlikely probabilities: Not here, but on the front end
// options = options.filter(element => element.probability > 0.02)
let obj = ({
"title": title,
"url": "https://sports.ladbrokes.com/sport/politics/outrights",
"platform": "Ladbrokes",
"description": "",
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Ladbrokes", ({}))
}
})
results.push(obj)
}
let specialMarkets = markets.filter(market => market.name.includes("Specials"))
for(let specialMarket of specialMarkets){
//let title = specialMarket.parent + ": " + specialMarket.name
//console.log(title)
specialMarket.children.forEach(child => {
let name = specialMarket.parent.includes("Specials")? child.outcome.name : specialMarket.parent + ": " + child.outcome.name
name = name.replace("Boris Johnson Specials", "Boris Johnson") // small kludge
let priceData = child.outcome.children[0].price
let priceDecimal = Number(priceData.priceDec)
let probability = 1/priceDecimal
let obj = ({
"title": name,
"url": "https://sports.ladbrokes.com/sport/politics/outrights",
"platform": "LadBrokes",
"options": [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
],
"qualityindicators": {
"stars": calculateStars("Ladbrokes", ({}))
}
})
results.push(obj)
})
}
return results
}
/* Body */
export async function ladbrokes() {
let response = await fetchUrl(endpointPolitics)
let results = await processResults(response)
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/ladbrokes-questions.json', string);
await upsert(results, "ladbrokes-questions")
console.log("Done")
}
//ladbrokes()

2
src/platforms/manifoldmarkets-fetch.js
View File

@ -43,6 +43,7 @@ function showStatistics(results) {
async function processPredictions(predictions) { async function processPredictions(predictions) {
let results = await predictions.map((prediction) => { let results = await predictions.map((prediction) => {
let id = `manifold-${prediction.id}`;
let probability = prediction.probability; let probability = prediction.probability;
let options = [ let options = [
{ {
@ -57,6 +58,7 @@ async function processPredictions(predictions) {
}, },
]; ];
let result = { let result = {
id: id,
title: prediction.question, title: prediction.question,
url: prediction.url, url: prediction.url,
platform: "Manifold Markets", platform: "Manifold Markets",

191
src/platforms/metaculus-fetch.js
View File

@ -1,158 +1,161 @@
/* Imports */ /* Imports */
import axios from "axios" import axios from "axios";
import fs from 'fs' import fs from "fs";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let jsonEndPoint = 'https://www.metaculus.com/api2/questions/?page=' let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
let all_questions = []; let all_questions = [];
let now = new Date().toISOString() let now = new Date().toISOString();
let DEBUG_MODE = "off";
/* Support functions */ /* Support functions */
async function fetchMetaculusQuestions(next) { async function fetchMetaculusQuestions(next) {
// Numbers about a given address: how many, how much, at what price, etc. // Numbers about a given address: how many, how much, at what price, etc.
let response = await axios(({ let response = await axios({
url: next, url: next,
method: 'GET', method: "GET",
headers: ({ 'Content-Type': 'application/json' }) headers: { "Content-Type": "application/json" },
})) }).then((res) => res.data);
.then(res => res.data)
// console.log(response) // console.log(response)
return response return response;
} }
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
async function fetchMetaculusQuestionDescription(slug) { async function fetchMetaculusQuestionDescription(slug) {
try { try {
let response = await axios({ let response = await axios({
method: 'get', method: "get",
url: "https://www.metaculus.com" + slug url: "https://www.metaculus.com" + slug,
}).then(response => response.data) }).then((response) => response.data);
return response return response;
} catch (error) { } catch (error) {
console.log(error) console.log(error);
console.log(`We encountered some error when attempting to fetch a metaculus page. Trying again`) console.log(
await sleep(10000) `We encountered some error when attempting to fetch a metaculus page. Trying again`
);
await sleep(10000);
try { try {
let response = await axios({ let response = await axios({
method: 'get', method: "get",
url: "https://www.metaculus.com" + slug url: "https://www.metaculus.com" + slug,
}).then(response => response.data) }).then((response) => response.data);
// console.log(response) // console.log(response)
return response return response;
} catch (error) { } catch (error) {
console.log(`We encountered some error when attempting to fetch a metaculus page.`) console.log(
console.log("Error", error) `We encountered some error when attempting to fetch a metaculus page.`
throw "Giving up" );
console.log("Error", error);
throw "Giving up";
} }
} }
} }
/* Body */ /* Body */
export async function metaculus() { export async function metaculus() {
// let metaculusQuestionsInit = await fetchMetaculusQuestions(1) // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
// console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
// for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test
let next = "https://www.metaculus.com/api2/questions/" let next = "https://www.metaculus.com/api2/questions/";
let i = 1 let i = 1;
while(next){ while (next) {
if (i % 20 == 0) { if (i % 20 == 0) {
console.log("Sleeping for 5secs") console.log("Sleeping for 5secs");
await sleep(5000) await sleep(5000);
} }
console.log(`\nQuery #${i}`) console.log(`\nQuery #${i}`);
let metaculusQuestions = await fetchMetaculusQuestions(next) let metaculusQuestions = await fetchMetaculusQuestions(next);
let results = metaculusQuestions.results; let results = metaculusQuestions.results;
let j=false let j = false;
for (let result of results) { for (let result of results) {
if ( if (result.publish_time < now && now < result.resolve_time) {
(result.publish_time < now) && await sleep(5000);
(now < result.resolve_time) let questionPage = await fetchMetaculusQuestionDescription(
) { result.page_url
await sleep(5000) );
let questionPage = await fetchMetaculusQuestionDescription(result.page_url) if (!questionPage.includes("A public prediction by")) {
if(!questionPage.includes("A public prediction by")){
// console.log(questionPage) // console.log(questionPage)
let descriptionraw = questionPage.split(`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1] let descriptionraw = questionPage.split(
let descriptionprocessed1 = descriptionraw.split("</div>")[0] `<div class="content" ng-bind-html-compile="qctrl.question.description_html">`
let descriptionprocessed2 = toMarkdown(descriptionprocessed1) )[1]; //.split(`<div class="question__content">`)[1]
let description = descriptionprocessed2 let descriptionprocessed1 = descriptionraw.split("</div>")[0];
let descriptionprocessed2 = toMarkdown(descriptionprocessed1);
let description = descriptionprocessed2;
let isbinary = result.possibilities.type == "binary" let isbinary = result.possibilities.type == "binary";
let options = [] let options = [];
if (isbinary) { if (isbinary) {
let probability = Number(result.community_prediction.full.q2) let probability = Number(result.community_prediction.full.q2);
options = [ options = [
{ {
"name": "Yes", name: "Yes",
"probability": probability, probability: probability,
"type": "PROBABILITY" type: "PROBABILITY",
}, },
{ {
"name": "No", name: "No",
"probability": 1 - probability, probability: 1 - probability,
"type": "PROBABILITY" type: "PROBABILITY",
} },
] ];
} }
let interestingInfo = ({ let id = `metaculus-${result.id}`;
"title": result.title, let interestingInfo = {
"url": "https://www.metaculus.com" + result.page_url, id: id,
"platform": "Metaculus", title: result.title,
"description": description, url: "https://www.metaculus.com" + result.page_url,
"options": options, platform: "Metaculus",
"timestamp": new Date().toISOString(), description: description,
"qualityindicators": { options: options,
"numforecasts": Number(result.number_of_predictions), timestamp: new Date().toISOString(),
"stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions })) qualityindicators: {
numforecasts: Number(result.number_of_predictions),
stars: calculateStars("Metaculus", {
numforecasts: result.number_of_predictions,
}),
},
extra: {
resolution_data: {
publish_time: result.publish_time,
resolution: result.resolution,
close_time: result.close_time,
resolve_time: result.resolve_time,
},
}, },
"extra": {
"resolution_data": {
"publish_time": result.publish_time,
"resolution": result.resolution,
"close_time": result.close_time,
"resolve_time": result.resolve_time
}
}
//"status": result.status, //"status": result.status,
//"publish_time": result.publish_time, //"publish_time": result.publish_time,
//"close_time": result.close_time, //"close_time": result.close_time,
//"type": result.possibilities.type, // We want binary ones here. //"type": result.possibilities.type, // We want binary ones here.
//"last_activity_time": result.last_activity_time, //"last_activity_time": result.last_activity_time,
}) };
if (Number(result.number_of_predictions) >= 10) { if (Number(result.number_of_predictions) >= 10) {
console.log(`- ${interestingInfo.title}`) console.log(`- ${interestingInfo.title}`);
all_questions.push(interestingInfo) all_questions.push(interestingInfo);
if(!j && (i % 20 == 0)){ if ((!j && i % 20 == 0) || DEBUG_MODE == "on") {
console.log(interestingInfo) console.log(interestingInfo);
j = true j = true;
} }
} }
}else{ } else {
console.log("- [Skipping public prediction]") console.log("- [Skipping public prediction]");
} }
} }
} }
next = metaculusQuestions.next next = metaculusQuestions.next;
i = i+1 i = i + 1;
} }
// let string = JSON.stringify(all_questions, null, 2) // let string = JSON.stringify(all_questions, null, 2)
// fs.writeFileSync('./metaculus-questions.json', string); // fs.writeFileSync('./metaculus-questions.json', string);
await upsert(all_questions, "metaculus-questions") await upsert(all_questions, "metaculus-questions");
console.log("Done") console.log("Done");
} }
//metaculus() //metaculus()

135
src/platforms/polymarket-fetch.js
View File

@ -1,12 +1,13 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"// "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' let graphQLendpoint =
let units = 10 ** 6 "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
let units = 10 ** 6;
/* Support functions /* Support functions
async function fetchAllContractInfo(){ // for info which the polymarket graphql API async function fetchAllContractInfo(){ // for info which the polymarket graphql API
@ -15,20 +16,24 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql
return response return response
} }
*/ */
async function fetchAllContractInfo() { // for info which the polymarket graphql API async function fetchAllContractInfo() {
let response = await axios.get('https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1') // for info which the polymarket graphql API
.then(query => query.data); let response = await axios
response = response.filter(res => res.closed != true) .get(
return response "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1"
)
.then((query) => query.data);
response = response.filter((res) => res.closed != true);
return response;
} }
async function fetchAllContractData() { async function fetchAllContractData() {
let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7 // last week let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week
let response = await axios({ let response = await axios({
url: graphQLendpoint, url: graphQLendpoint,
method: 'POST', method: "POST",
headers: ({ 'Content-Type': 'application/json' }), headers: { "Content-Type": "application/json" },
data: JSON.stringify(({ data: JSON.stringify({
query: ` query: `
{ {
fixedProductMarketMakers(first: 1000 fixedProductMarketMakers(first: 1000
@ -53,23 +58,23 @@ async function fetchAllContractData() {
} }
} }
} }
` `,
})), }),
}) })
.then(res => res.data) .then((res) => res.data)
.then(res => res.data.fixedProductMarketMakers) .then((res) => res.data.fixedProductMarketMakers);
// console.log(response) // console.log(response)
return response return response;
} }
async function fetch_all() { async function fetch_all() {
let allData = await fetchAllContractData() let allData = await fetchAllContractData();
let allInfo = await fetchAllContractInfo() let allInfo = await fetchAllContractInfo();
let infos = ({}) let infos = {};
for (let info of allInfo) { for (let info of allInfo) {
let address = info.marketMakerAddress let address = info.marketMakerAddress;
let addressLowerCase = address.toLowerCase() let addressLowerCase = address.toLowerCase();
//delete info.history //delete info.history
if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long")
infos[addressLowerCase] = { infos[addressLowerCase] = {
@ -79,67 +84,75 @@ async function fetch_all() {
description: info.description, description: info.description,
outcomes: info.outcomes, outcomes: info.outcomes,
options: [], options: [],
category: info.category category: info.category,
} };
} }
let results = [] let results = [];
for (let data of allData) { for (let data of allData) {
let addressLowerCase = data.id let addressLowerCase = data.id;
// console.log(data) // console.log(data)
if (infos[addressLowerCase] != undefined) { if (infos[addressLowerCase] != undefined) {
// console.log(addressLowerCase) // console.log(addressLowerCase)
let info = infos[addressLowerCase] let id = `polymarket-${addressLowerCase.slice(0, 10)}`;
let numforecasts = Number(data.tradesQuantity) let info = infos[addressLowerCase];
let tradevolume = (Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) / units let numforecasts = Number(data.tradesQuantity);
let liquidity = Number(data.liquidityParameter) / units let tradevolume =
(Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) /
units;
let liquidity = Number(data.liquidityParameter) / units;
// let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2 // let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2
// let percentage = Number(data.outcomeTokenPrices[0]) * 100 // let percentage = Number(data.outcomeTokenPrices[0]) * 100
// let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none" // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none"
let options = [] let options = [];
for (let outcome in data.outcomeTokenPrices) { for (let outcome in data.outcomeTokenPrices) {
options.push({ options.push({
"name": info.outcomes[outcome], name: info.outcomes[outcome],
"probability": data.outcomeTokenPrices[outcome], probability: data.outcomeTokenPrices[outcome],
"type": "PROBABILITY" type: "PROBABILITY",
}) });
} }
let result = ({ let result = {
"title": info.title, id: id,
"url": info.url, title: info.title,
"platform": "PolyMarket", url: info.url,
"description": info.description, platform: "PolyMarket",
"options": options, description: info.description,
"timestamp": new Date().toISOString(), options: options,
"qualityindicators": { timestamp: new Date().toISOString(),
"numforecasts": numforecasts.toFixed(0), qualityindicators: {
"liquidity": liquidity.toFixed(2), numforecasts: numforecasts.toFixed(0),
"tradevolume": tradevolume.toFixed(2), liquidity: liquidity.toFixed(2),
"stars": calculateStars("Polymarket", ({ liquidity, option: options[0], volume: tradevolume})) tradevolume: tradevolume.toFixed(2),
stars: calculateStars("Polymarket", {
liquidity,
option: options[0],
volume: tradevolume,
}),
},
extra: {
address: info.address,
}, },
"extra": {
"address": info.address
}
/* /*
*/ */
}) };
if(info.category != "Sports"){ if (info.category != "Sports") {
// console.log(result) // console.log(result)
results.push(result) results.push(result);
} }
} }
} }
return results //resultsProcessed return results; //resultsProcessed
} }
/* Body */ /* Body */
export async function polymarket() { export async function polymarket() {
let results = await fetch_all() let results = await fetch_all();
// console.log(results) // console.log(results)
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polymarket-questions.json', string); // fs.writeFileSync('polymarket-questions.json', string);
await upsert(results, "polymarket-questions") await upsert(results, "polymarket-questions");
console.log("Done") console.log("Done");
} }
// polymarket() // polymarket()

138
src/platforms/predictit-fetch.js
View File

@ -1,114 +1,116 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Support functions */ /* Support functions */
async function fetchmarkets() { async function fetchmarkets() {
let response = await axios({ let response = await axios({
method: 'get', method: "get",
url: 'https://www.predictit.org/api/marketdata/all/' url: "https://www.predictit.org/api/marketdata/all/",
});
}) let openMarkets = response.data.markets.filter(
let openMarkets = response.data.markets.filter(market => market.status == "Open") (market) => market.status == "Open"
return openMarkets );
return openMarkets;
} }
async function fetchmarketrules(market_id) { async function fetchmarketrules(market_id) {
let response = await axios({ let response = await axios({
method: 'get', method: "get",
url: 'https://www.predictit.org/api/Market/' + market_id url: "https://www.predictit.org/api/Market/" + market_id,
}) });
return response.data.rule return response.data.rule;
} }
async function fetchmarketvolumes(){ async function fetchmarketvolumes() {
let response = await axios({ let response = await axios({
method: 'get', method: "get",
url: "https://predictit-f497e.firebaseio.com/marketStats.json" url: "https://predictit-f497e.firebaseio.com/marketStats.json",
}) });
return response.data return response.data;
} }
function sleep(ms) { function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
/* Body */ /* Body */
export async function predictit() { export async function predictit() {
let markets = await fetchmarkets() let markets = await fetchmarkets();
let marketVolumes = await fetchmarketvolumes() let marketVolumes = await fetchmarketvolumes();
markets = markets.map(market => ({ markets = markets.map((market) => ({
...market, ...market,
TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"] TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"],
})) }));
// console.log(markets) // console.log(markets)
let results = [] let results = [];
for (let market of markets) { for (let market of markets) {
// console.log(market.name) // console.log(market.name)
let id = `predictit-${market.id}`;
let isbinary = market.contracts.length == 1; let isbinary = market.contracts.length == 1;
await sleep(3000 * (1 + Math.random())) await sleep(3000 * (1 + Math.random()));
let descriptionraw = await fetchmarketrules(market.id) let descriptionraw = await fetchmarketrules(market.id);
let descriptionprocessed1 = toMarkdown(descriptionraw) let descriptionprocessed1 = toMarkdown(descriptionraw);
let description = descriptionprocessed1 let description = descriptionprocessed1;
let shares_volume = market["TotalSharesTraded"] let shares_volume = market["TotalSharesTraded"];
// let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none" // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none"
let options = market.contracts.map(contract => ({ let options = market.contracts.map((contract) => ({
"name": contract.name, name: contract.name,
"probability": contract.lastTradePrice, probability: contract.lastTradePrice,
"type": "PROBABILITY" type: "PROBABILITY",
})) }));
let totalValue = options let totalValue = options
.map(element => Number(element.probability)) .map((element) => Number(element.probability))
.reduce((a, b) => (a + b), 0) .reduce((a, b) => a + b, 0);
if (options.length != 1 && totalValue > 1) { if (options.length != 1 && totalValue > 1) {
options = options.map(element => ({ options = options.map((element) => ({
...element, ...element,
probability: Number(element.probability) / totalValue probability: Number(element.probability) / totalValue,
})) }));
} else if (options.length == 1) { } else if (options.length == 1) {
let option = options[0] let option = options[0];
let probability = option["probability"] let probability = option["probability"];
options = [ options = [
{ {
"name": "Yes", name: "Yes",
"probability": probability, probability: probability,
"type": "PROBABILITY" type: "PROBABILITY",
}, },
{ {
"name": "No", name: "No",
"probability": 1 - probability, probability: 1 - probability,
"type": "PROBABILITY" type: "PROBABILITY",
} },
] ];
} }
let obj = ({ let obj = {
"title": market["name"], id: id,
"url": market.url, title: market["name"],
"platform": "PredictIt", url: market.url,
"description": description, platform: "PredictIt",
"options": options, description: description,
"timestamp": new Date().toISOString(), options: options,
"qualityindicators": { timestamp: new Date().toISOString(),
"stars": calculateStars("PredictIt", ({})), qualityindicators: {
"shares_volume": shares_volume stars: calculateStars("PredictIt", {}),
} shares_volume: shares_volume,
}) },
};
// console.log(obj) // console.log(obj)
results.push(obj) results.push(obj);
} }
//console.log(results) //console.log(results)
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/predictit-questions.json', string); // fs.writeFileSync('./data/predictit-questions.json', string);
await upsert(results, "predictit-questions") await upsert(results, "predictit-questions");
console.log("Done") console.log("Done");
} }

91
src/platforms/rootclaim-fetch.js
View File

@ -1,12 +1,13 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let jsonEndpoint = "https://www.rootclaim.com/main_page_stories?number=100&offset=0"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' let jsonEndpoint =
"https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
/* Support functions /* Support functions
async function fetchAllContractInfo(){ // for info which the polymarket graphql API async function fetchAllContractInfo(){ // for info which the polymarket graphql API
@ -15,56 +16,60 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql
return response return response
} }
*/ */
async function fetchAllRootclaims() { // for info which the polymarket graphql API async function fetchAllRootclaims() {
let response = await axios.get(jsonEndpoint) // for info which the polymarket graphql API
.then(response => response.data) let response = await axios
if(response.length != (response[0]+1)){ .get(jsonEndpoint)
console.log(response.length) .then((response) => response.data);
console.log(response[0]) if (response.length != response[0] + 1) {
//throw Error("Rootclaim's backend has changed.") console.log(response.length);
console.log(response[0]);
//throw Error("Rootclaim's backend has changed.")
} }
response.shift() response.shift();
return response return response;
} }
async function fetchAndProcessData() { async function fetchAndProcessData() {
let claims = await fetchAllRootclaims() let claims = await fetchAllRootclaims();
let results = [] let results = [];
for (let claim of claims) { for (let claim of claims) {
let options = [] let id = `rootclaim-${claim.slug.toLowerCase()}`;
let options = [];
for (let scenario of claim.scenarios) { for (let scenario of claim.scenarios) {
//console.log(scenario) //console.log(scenario)
options.push({ options.push({
"name": toMarkdown(scenario.text).replace("\n","").replace("&#39;","'"), name: toMarkdown(scenario.text).replace("\n", "").replace("&#39;", "'"),
"probability": scenario.net_prob / 100, probability: scenario.net_prob / 100,
"type": "PROBABILITY" type: "PROBABILITY",
}) });
} }
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis" let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
let obj = ({ let obj = {
"title": toMarkdown(claim.question).replace("\n",""), id: id,
"url": `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, title: toMarkdown(claim.question).replace("\n", ""),
"platform": "Rootclaim", url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
"description": toMarkdown(claim.background).replace("&#39;","'"), platform: "Rootclaim",
"options": options, description: toMarkdown(claim.background).replace("&#39;", "'"),
"timestamp": new Date().toISOString(), options: options,
"qualityindicators": { timestamp: new Date().toISOString(),
"numforecasts": 1, qualityindicators: {
"stars": calculateStars("Rootclaim", ({})) numforecasts: 1,
} stars: calculateStars("Rootclaim", {}),
}) },
results.push(obj) };
} results.push(obj);
return results }
return results;
} }
/* Body */ /* Body */
export async function rootclaim() { export async function rootclaim() {
let results = await fetchAndProcessData() let results = await fetchAndProcessData();
//console.log(JSON.stringify(results, null, 4)) //console.log(JSON.stringify(results, null, 4))
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('rootclaim-questions.json', string); // fs.writeFileSync('rootclaim-questions.json', string);
await upsert(results, "rootclaim-questions") await upsert(results, "rootclaim-questions");
console.log("Done") console.log("Done");
} }
//rootclaim() //rootclaim()

146
src/platforms/smarkets-fetch.js
View File

@ -1,127 +1,127 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
import axios from "axios" import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let htmlEndPointEntrance = 'https://api.smarkets.com/v3/events/' let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
/* Support functions */ /* Support functions */
async function fetchEvents(url) { async function fetchEvents(url) {
let response = await axios({ let response = await axios({
url: htmlEndPointEntrance + url, url: htmlEndPointEntrance + url,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
}), },
}) }).then((res) => res.data);
.then(res => res.data)
// console.log(response) // console.log(response)
return response return response;
} }
async function fetchMarkets(eventid) { async function fetchMarkets(eventid) {
let response = await axios({ let response = await axios({
url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, url: `https://api.smarkets.com/v3/events/${eventid}/markets/`,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/json', "Content-Type": "text/json",
}), },
}) })
.then(res => res.data) .then((res) => res.data)
.then(res => res.markets) .then((res) => res.markets);
return response return response;
} }
async function fetchContracts(marketid) { async function fetchContracts(marketid) {
let response = await axios({ let response = await axios({
url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
}), },
}) }).then((res) => res.data);
.then(res => res.data)
// console.log(response) // console.log(response)
return response return response;
} }
async function fetchPrices(marketid) { async function fetchPrices(marketid) {
let response = await axios({ let response = await axios({
url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`,
method: 'GET', method: "GET",
headers: ({ headers: {
'Content-Type': 'text/html', "Content-Type": "text/html",
}), },
}) }).then((res) => res.data);
.then(res => res.data)
// console.log(response) // console.log(response)
return response return response;
} }
/* Body */ /* Body */
export async function smarkets() { export async function smarkets() {
let htmlPath = '?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50' let htmlPath =
"?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
let events = [] let events = [];
while (htmlPath) { while (htmlPath) {
let data = await fetchEvents(htmlPath) let data = await fetchEvents(htmlPath);
events.push(...data.events) events.push(...data.events);
htmlPath = data.pagination.next_page htmlPath = data.pagination.next_page;
} }
// console.log(events) // console.log(events)
let markets = [] let markets = [];
for (let event of events) { for (let event of events) {
// console.log(Date.now()) // console.log(Date.now())
// console.log(event.name) // console.log(event.name)
let eventMarkets = await fetchMarkets(event.id) let eventMarkets = await fetchMarkets(event.id);
eventMarkets = eventMarkets.map(market => ({ ...market, slug: event.full_slug })) eventMarkets = eventMarkets.map((market) => ({
...market,
slug: event.full_slug,
}));
// console.log("Markets fetched") // console.log("Markets fetched")
// console.log(event.id) // console.log(event.id)
// console.log(market) // console.log(market)
markets.push(...eventMarkets) markets.push(...eventMarkets);
//let lastPrices = await fetchPrices(market.id) //let lastPrices = await fetchPrices(market.id)
} }
// console.log(markets) // console.log(markets)
let results = [] let results = [];
for (let market of markets) { for (let market of markets) {
// console.log("================") // console.log("================")
// console.log("Market: ", market) // console.log("Market: ", market)
let name = market.name let id = `smarkets-${market.id}`;
let name = market.name;
let contracts = await fetchContracts(market.id) let contracts = await fetchContracts(market.id);
// console.log("Contracts: ", contracts) // console.log("Contracts: ", contracts)
let prices = await fetchPrices(market.id) let prices = await fetchPrices(market.id);
// console.log("Prices: ", prices["last_executed_prices"][market.id]) // console.log("Prices: ", prices["last_executed_prices"][market.id])
let options = {} let options = {};
for (let contract of contracts["contracts"]) { for (let contract of contracts["contracts"]) {
options[contract.id] = { name: contract.name } options[contract.id] = { name: contract.name };
} }
for (let price of prices["last_executed_prices"][market.id]) { for (let price of prices["last_executed_prices"][market.id]) {
options[price.contract_id] = { options[price.contract_id] = {
...options[price.contract_id], ...options[price.contract_id],
probability: Number(price.last_executed_price), probability: Number(price.last_executed_price),
type: "PROBABILITY" type: "PROBABILITY",
} };
} }
options = Object.values(options) options = Object.values(options);
let totalValue = options let totalValue = options
.map(element => Number(element.probability)) .map((element) => Number(element.probability))
.reduce((a, b) => (a + b), 0) .reduce((a, b) => a + b, 0);
options = options.map(element => ({ options = options.map((element) => ({
...element, ...element,
probability: Number(element.probability) / totalValue probability: Number(element.probability) / totalValue,
})) }));
// console.log(options) // console.log(options)
@ -135,26 +135,24 @@ export async function smarkets() {
} }
*/ */
let result = { let result = {
"title": name, id: id,
"url": "https://smarkets.com/event/" + market.event_id + market.slug, title: name,
"platform": "Smarkets", url: "https://smarkets.com/event/" + market.event_id + market.slug,
"description": market.description, platform: "Smarkets",
"options": options, description: market.description,
"timestamp": new Date().toISOString(), options: options,
"qualityindicators": { timestamp: new Date().toISOString(),
"stars": calculateStars("Smarkets", ({})) qualityindicators: {
} stars: calculateStars("Smarkets", {}),
},
} };
// console.log(result) // console.log(result)
results.push(result) results.push(result);
} }
// console.log(results) // console.log(results)
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/smarkets-questions.json', string); // fs.writeFileSync('./data/smarkets-questions.json', string);
await upsert(results, "smarkets-questions") await upsert(results, "smarkets-questions");
} }
//smarkets() //smarkets()

169
src/platforms/wildeford-fetch.js
View File

@ -1,51 +1,61 @@
/* Imports */ /* Imports */
import fs from 'fs' import fs from "fs";
// import axios from "axios" // import axios from "axios"
import { GoogleSpreadsheet } from "google-spreadsheet" import { GoogleSpreadsheet } from "google-spreadsheet";
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import { getCookie, applyIfCookieExists } from "../utils/getCookies.js";
import toMarkdown from "../utils/toMarkdown.js" import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js" import { calculateStars } from "../utils/stars.js";
import {upsert} from "../utils/mongo-wrapper.js" import { upsert } from "../utils/mongo-wrapper.js";
import { hash } from "../utils/hash.js";
/* Definitions */ /* Definitions */
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0" // spreadsheet key is the long id in the sheets URL const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0` const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`;
// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0 // https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0
/* Support functions */ /* Support functions */
const formatRow = row => { const formatRow = (row) => {
let colNames = ["Prediction Date", "Prediction", "Odds", "Actual", "Resolution Date", "Prediction Right?", "Brier Score", "Notes"] let colNames = [
let result = ({}) "Prediction Date",
row.forEach((col,i) => { "Prediction",
result[colNames[i]] = col "Odds",
}) "Actual",
return result "Resolution Date",
} "Prediction Right?",
"Brier Score",
"Notes",
];
let result = {};
row.forEach((col, i) => {
result[colNames[i]] = col;
});
return result;
};
async function fetchGoogleDoc(google_api_key){ async function fetchGoogleDoc(google_api_key) {
// https://gist.github.com/micalevisk/9bc831bd4b3e5a3f62b9810330129c59 // https://gist.github.com/micalevisk/9bc831bd4b3e5a3f62b9810330129c59
let results = [] let results = [];
const doc = new GoogleSpreadsheet(SHEET_ID) const doc = new GoogleSpreadsheet(SHEET_ID);
doc.useApiKey(google_api_key) doc.useApiKey(google_api_key);
await doc.loadInfo() // loads document properties and worksheets await doc.loadInfo(); // loads document properties and worksheets
console.log('>>', doc.title) console.log(">>", doc.title);
const sheet = doc.sheetsByIndex[0] const sheet = doc.sheetsByIndex[0];
const rows = await sheet.getRows({ offset:0, }) const rows = await sheet.getRows({ offset: 0 });
console.log('# ' + console.log("# " + rows[0]._sheet.headerValues.join(","));
rows[0]._sheet.headerValues.join(',')
)
let isEnd = false; let isEnd = false;
for (let i in rows) { for (let i in rows) {
let data = rows[i]._rawData let data = rows[i]._rawData;
if(data.length == 0) isEnd = true; if (data.length == 0) isEnd = true;
if(!isEnd){ if (!isEnd) {
let result = ({...formatRow(data), "url": endpoint + `&range=A${(Number(i) + 2)}`}) let result = {
...formatRow(data),
url: endpoint + `&range=A${Number(i) + 2}`,
};
// +2: +1 for the header row, +1 for starting at 1 and not at 0. // +2: +1 for the header row, +1 for starting at 1 and not at 0.
// console.log(result) // console.log(result)
results.push(result) results.push(result);
// console.log(rows[i]) // console.log(rows[i])
// console.log(rows[i]._rawData) // console.log(rows[i]._rawData)
@ -55,66 +65,71 @@ async function fetchGoogleDoc(google_api_key){
// console.log(row._rawData.join(',')) // console.log(row._rawData.join(','))
} }
// console.log(results) // console.log(results)
return(results) return results;
} }
async function processPredictions(predictions){ async function processPredictions(predictions) {
let currentPredictions = predictions.filter(prediction => prediction["Actual"] == "Unknown" ) let currentPredictions = predictions.filter(
let results = currentPredictions.map(prediction => { (prediction) => prediction["Actual"] == "Unknown"
let probability = Number(prediction["Odds"].replace("%", ""))/100 );
let results = currentPredictions.map((prediction) => {
let title = prediction["Prediction"].replace(" [update]", "");
let id = `wildeford-${hash(title)}`;
let probability = Number(prediction["Odds"].replace("%", "")) / 100;
let options = [ let options = [
{ {
"name": "Yes", name: "Yes",
"probability": probability, probability: probability,
"type": "PROBABILITY" type: "PROBABILITY",
}, },
{ {
"name": "No", name: "No",
"probability": 1 - probability, probability: 1 - probability,
"type": "PROBABILITY" type: "PROBABILITY",
} },
] ];
let result = ({ let result = {
"title": prediction["Prediction"], id: id,
"url": prediction["url"], title: title,
"platform": "Peter Wildeford", url: prediction["url"],
"description": prediction["Notes"] || "", platform: "Peter Wildeford",
"options": options, description: prediction["Notes"] || "",
"timestamp": new Date(Date.parse(prediction["Prediction Date"] + "Z")).toISOString(), options: options,
"qualityindicators": { timestamp: new Date(
"stars": calculateStars("Peter Wildeford"), Date.parse(prediction["Prediction Date"] + "Z")
} ).toISOString(),
}) qualityindicators: {
return result stars: calculateStars("Peter Wildeford"),
}) },
results = results.map(result => ({...result, title: result.title.replace(" [update]", "")})).reverse() };
return result;
});
let uniqueTitles = [] results = results.reverse();
let uniqueResults = [] let uniqueTitles = [];
results.forEach(result => { let uniqueResults = [];
if(!uniqueTitles.includes(result.title)) uniqueResults.push(result) results.forEach((result) => {
uniqueTitles.push(result.title) if (!uniqueTitles.includes(result.title)) uniqueResults.push(result);
}) uniqueTitles.push(result.title);
return(uniqueResults) });
return uniqueResults;
// console.log(results) // console.log(results)
// console.log(results.map(result => result.options)) // console.log(results.map(result => result.options))
// processPredictions()
} }
// processPredictions()
/* Body */ /* Body */
export async function wildeford_inner(google_api_key) { export async function wildeford_inner(google_api_key) {
let predictions = await fetchGoogleDoc(google_api_key) let predictions = await fetchGoogleDoc(google_api_key);
let results = await processPredictions(predictions) // somehow needed let results = await processPredictions(predictions); // somehow needed
// console.log(results.sort((a,b) => (a.title > b.title))) // console.log(results.sort((a,b) => (a.title > b.title)))
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string); // fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "wildeford-questions") await upsert(results, "wildeford-questions");
console.log("Done") console.log("Done");
} }
//example() //example()
export async function wildeford(){ export async function wildeford() {
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getCookie("google-api") // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getCookie("google-api"); // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
await applyIfCookieExists(GOOGLE_API_KEY, wildeford_inner) await applyIfCookieExists(GOOGLE_API_KEY, wildeford_inner);
} }