quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Markets now have unique ids in preparation for dashbards.

Browse Source
This commit is contained in:
NunoSempere 2022-02-11 09:21:36 -05:00
parent d760532e95
commit 3d43aa3b6e
19 changed files with 1368 additions and 1205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
src/platforms/betfair-fetch.js
View File

@ -1,13 +1,13 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import https from 'https';
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import https from "https";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
/* Utilities */
let arraysEqual = (a, b) => {
@ -24,74 +24,81 @@ let arraysEqual = (a, b) => {
if (a[i] !== b[i]) return false;
}
return true;
}
};
let mergeRunners = (runnerCatalog, runnerBook) => {
let keys = Object.keys(runnerCatalog)
let result = []
for(let key of keys){
result.push({...runnerCatalog[key], ...runnerBook[key]})
let keys = Object.keys(runnerCatalog);
let result = [];
for (let key of keys) {
result.push({ ...runnerCatalog[key], ...runnerBook[key] });
}
return result
}
return result;
};
/* Support functions */
async function fetchPredictions() {
const agent = new https.Agent({
rejectUnauthorized: false
rejectUnauthorized: false,
});
let response = await axios({
url: endpoint,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
httpsAgent: agent
}).then(response => response.data)
method: "GET",
headers: {
"Content-Type": "text/html",
},
httpsAgent: agent,
}).then((response) => response.data);
return response
return response;
}
async function whipIntoShape(data){
let catalogues = data.market_catalogues
let books = data.market_books
let keys1 = Object.keys(catalogues).sort()
let keys2 = Object.keys(books).sort()
async function whipIntoShape(data) {
let catalogues = data.market_catalogues;
let books = data.market_books;
let keys1 = Object.keys(catalogues).sort();
let keys2 = Object.keys(books).sort();
// console.log(keys1)
// console.log(keys2)
let results = []
if(!arraysEqual(keys1, keys2)){
throw new Error("Betfair: Error in endpoint; Betfair catalogues and books do not match")
}else{
for(let key of keys1){
results.push({...catalogues[key], ...books[key], options: mergeRunners(catalogues[key].runners, books[key].runners)})
let results = [];
if (!arraysEqual(keys1, keys2)) {
throw new Error(
"Betfair: Error in endpoint; Betfair catalogues and books do not match"
);
} else {
for (let key of keys1) {
results.push({
...catalogues[key],
...books[key],
options: mergeRunners(catalogues[key].runners, books[key].runners),
});
}
}
return results
return results;
}
async function processPredictions(data) {
let predictions = await whipIntoShape(data)
let predictions = await whipIntoShape(data);
// console.log(JSON.stringify(predictions, null, 4))
let results = predictions.map(prediction => {
let results = predictions.map((prediction) => {
/* if(Math.floor(Math.random() * 10) % 20 ==0){
console.log(JSON.stringify(prediction, null, 4))
} */
let normalizationFactor = (prediction.options
.filter(option => option.status == "ACTIVE" && option.totalMatched > 0)
.map(option => option.lastPriceTraded))
.map(x => 1/x)
.reduce((a, b) => a + b, 0)
let id = `betfair-${prediction.marketId}`;
let normalizationFactor = prediction.options
.filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
.map((option) => option.lastPriceTraded)
.map((x) => 1 / x)
.reduce((a, b) => a + b, 0);
let options = prediction.options
.filter(option => option.status == "ACTIVE" && option.totalMatched > 0)
.map(option => ({
"name": option.runnerName,
"probability": option.lastPriceTraded !=0 ? (1/option.lastPriceTraded)/normalizationFactor : 0, // https://www.aceodds.com/bet-calculator/odds-converter.html
"type": "PROBABILITY"
}))
.filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
.map((option) => ({
name: option.runnerName,
probability:
option.lastPriceTraded != 0
? 1 / option.lastPriceTraded / normalizationFactor
: 0, // https://www.aceodds.com/bet-calculator/odds-converter.html
type: "PROBABILITY",
}));
// console.log(prediction.options)
@ -102,41 +109,42 @@ async function processPredictions(data) {
.replace(/<b>/g, " ")
.replace(/<\/b>/g, " ")
.replace(/\n/g, " ")
.trim()
if(rules == undefined){
.trim();
if (rules == undefined) {
// console.log(prediction.description)
}
let title = rules.split("? ")[0] + "?"
let description = rules.split("? ")[1].trim()
if(title.includes("of the named")){
title = prediction.marketName + ": "+ title
let title = rules.split("? ")[0] + "?";
let description = rules.split("? ")[1].trim();
if (title.includes("of the named")) {
title = prediction.marketName + ": " + title;
}
let result = ({
"title": title,
"url": `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
"platform": "Betfair",
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Betfair", ({volume: prediction.totalMatched})),
"volume": prediction.totalMatched,
}
})
return result
})
return results //resultsProcessed
let result = {
id: id,
title: title,
url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
platform: "Betfair",
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Betfair", { volume: prediction.totalMatched }),
volume: prediction.totalMatched,
},
};
return result;
});
return results; //resultsProcessed
}
/* Body */
export async function betfair() {
let data = await fetchPredictions()
let results = await processPredictions(data) // somehow needed
let data = await fetchPredictions();
let results = await processPredictions(data); // somehow needed
// console.log(results.map(result => ({title: result.title, description: result.description})))
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "betfair-questions")
console.log("Done")
await upsert(results, "betfair-questions");
console.log("Done");
}
// betfair()

146
src/platforms/deprecated/ladbrokes-fetch.js Normal file
View File

@ -0,0 +1,146 @@
/* Imports */
import axios from "axios";
import fs from "fs";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`;
let enpointDrillDown = (id) =>
`https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`;
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
/* Support functions */
async function fetchUrl(url) {
let response = await axios(url, {
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0",
},
method: "GET",
mode: "cors",
}).then((response) => response.data);
return response;
}
let processResults = async (json) => {
let results = [];
let children = json.SSResponse.children;
children.pop();
let ids = children.map((child) => child.event.id);
let markets = [];
for (let id of ids) {
let marketsContainer = await fetchUrl(enpointDrillDown(id));
let marketsObj = marketsContainer.SSResponse.children[0].event;
let newMarkets = marketsObj.children;
newMarkets = newMarkets.map((market) => ({
...market.market,
parent: marketsObj.name,
}));
markets.push(...newMarkets);
}
let normalMarkets = markets.filter(
(market) => !market.name.includes("Specials")
);
//console.log(normalMarkets)
for (let normalMarket of normalMarkets) {
let title = normalMarket.parent + ": " + normalMarket.name;
title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
let options = normalMarket.children.map((child) => {
let name = child.outcome.name;
let priceData = child.outcome.children[0].price;
let priceDecimal = Number(priceData.priceDec);
let probability = 1 / priceDecimal;
let option = {
name: name,
probability: probability,
type: "PROBABILITY",
};
return option;
});
// normalize probabilities
let totalValue = options
.map((element) => Number(element.probability))
.reduce((a, b) => a + b, 0);
options = options.map((element) => ({
...element,
probability: Number(element.probability) / totalValue,
}));
// Filter very unlikely probabilities: Not here, but on the front end
// options = options.filter(element => element.probability > 0.02)
let obj = {
title: title,
url: "https://sports.ladbrokes.com/sport/politics/outrights",
platform: "Ladbrokes",
description: "",
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Ladbrokes", {}),
},
};
results.push(obj);
}
let specialMarkets = markets.filter((market) =>
market.name.includes("Specials")
);
for (let specialMarket of specialMarkets) {
//let title = specialMarket.parent + ": " + specialMarket.name
//console.log(title)
specialMarket.children.forEach((child) => {
let name = specialMarket.parent.includes("Specials")
? child.outcome.name
: specialMarket.parent + ": " + child.outcome.name;
name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
let priceData = child.outcome.children[0].price;
let priceDecimal = Number(priceData.priceDec);
let probability = 1 / priceDecimal;
let obj = {
title: name,
url: "https://sports.ladbrokes.com/sport/politics/outrights",
platform: "LadBrokes",
options: [
{
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
],
qualityindicators: {
stars: calculateStars("Ladbrokes", {}),
},
};
results.push(obj);
});
}
return results;
};
/* Body */
export async function ladbrokes() {
let response = await fetchUrl(endpointPolitics);
let results = await processResults(response);
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/ladbrokes-questions.json', string);
await upsert(results, "ladbrokes-questions");
console.log("Done");
}
//ladbrokes()

0
src/platforms/williamhill-fetch.js → src/platforms/deprecated/williamhill-fetch.js
View File

86
src/platforms/example-fetch.js
View File

@ -1,70 +1,70 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let endpoint = 'https://example.com/'
let endpoint = "https://example.com/";
/* Support functions */
async function fetchData() {
let response = await axios({
url: endpoint,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
})
.then(response => response.data)
method: "GET",
headers: {
"Content-Type": "text/html",
},
}).then((response) => response.data);
// console.log(response)
return response
return response;
}
async function processPredictions(predictions) {
let results = await predictions.map(prediction => {
let probability = prediction.probability
let results = await predictions.map((prediction) => {
let id = `platform-${prediction.id}`;
let probability = prediction.probability;
let options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
let result = ({
"title": prediction.title,
"url": `https://example.com`,
"platform": "Example",
"description": prediction.description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Example", ({some: somex, factors: factors})),
"other": prediction.otherx,
"indicators": prediction.indicatorx
}
})
return result
})
return results //resultsProcessed
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
];
let result = {
title: prediction.title,
url: `https://example.com`,
platform: "Example",
description: prediction.description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Example", { some: somex, factors: factors }),
other: prediction.otherx,
indicators: prediction.indicatorx,
},
};
return result;
});
return results; //resultsProcessed
}
/* Body */
export async function example() {
let data = await fetchData()
let results = await processPredictions(data) // somehow needed
let data = await fetchData();
let results = await processPredictions(data); // somehow needed
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "example-questions")
console.log("Done")
await upsert(results, "example-questions");
console.log("Done");
}
//example()

151
src/platforms/fantasyscotus-fetch.js
View File

@ -1,113 +1,124 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let unixtime = new Date() .getTime()
let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=0&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=12&_=${unixtime}`
let unixtime = new Date().getTime();
let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=0&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=12&_=${unixtime}`;
async function fetchData() {
let response = await axios({
"url": endpoint,
"credentials": "omit",
"headers": {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
"Accept": "application/json, text/javascript, */*; q=0.01",
url: endpoint,
credentials: "omit",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
Accept: "application/json, text/javascript, */*; q=0.01",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/x-www-form-urlencoded",
"X-Requested-With": "XMLHttpRequest"
"X-Requested-With": "XMLHttpRequest",
},
"referrer": "https://fantasyscotus.net/case/list/",
"method": "GET",
"mode": "cors"
})
.then(res => res.data)
referrer: "https://fantasyscotus.net/case/list/",
method: "GET",
mode: "cors",
}).then((res) => res.data);
//console.log(response)
return response
return response;
}
async function getPredictionsData(caseUrl) {
let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`
let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`;
//console.log(newCaseUrl)
let predictions = await axios({
"url": newCaseUrl,
"credentials": "include",
"headers": {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
"Accept": "application/json, text/javascript, */*; q=0.01",
url: newCaseUrl,
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
Accept: "application/json, text/javascript, */*; q=0.01",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/x-www-form-urlencoded",
"X-Requested-With": "XMLHttpRequest"
"X-Requested-With": "XMLHttpRequest",
},
"referrer": newCaseUrl,
"method": "GET",
"mode": "cors"
})
.then(res => res.data)
referrer: newCaseUrl,
method: "GET",
mode: "cors",
}).then((res) => res.data);
let predictionsAffirm = predictions.filter(prediction => prediction.percent_affirm > 50)
let predictionsAffirm = predictions.filter(
(prediction) => prediction.percent_affirm > 50
);
//console.log(predictions)
//console.log(predictionsAffirm.length/predictions.length)
return ({
return {
numAffirm: predictionsAffirm.length,
proportionAffirm: predictionsAffirm.length/predictions.length,
numForecasts: predictions.length
})
proportionAffirm: predictionsAffirm.length / predictions.length,
numForecasts: predictions.length,
};
}
async function processData(data) {
let events = data.object_list
let historicalPercentageCorrect = data.stats.pcnt_correct
let historicalProbabilityCorrect = Number(historicalPercentageCorrect.replace("%",""))/100
let results = []
for(let event of events){
if(event.accuracy == ''){ // if the thing hasn't already resolved
let predictionData = await getPredictionsData(event.docket_url)
let pAffirm = predictionData.proportionAffirm
let events = data.object_list;
let historicalPercentageCorrect = data.stats.pcnt_correct;
let historicalProbabilityCorrect =
Number(historicalPercentageCorrect.replace("%", "")) / 100;
let results = [];
for (let event of events) {
if (event.accuracy == "") {
let id = `fantasyscotus-${event.id}`;
// if the thing hasn't already resolved
let predictionData = await getPredictionsData(event.docket_url);
let pAffirm = predictionData.proportionAffirm;
//let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
let eventObject = ({
"title": `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
"url": `https://fantasyscotus.net/user-predictions${event.docket_url}`,
"platform": "FantasySCOTUS",
"description": `${(pAffirm*100).toFixed(2)}% (${predictionData.numAffirm} out of ${predictionData.numForecasts}) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of ${event.prediction}. Historically, FantasySCOTUS has chosen the correct side ${historicalPercentageCorrect} of the time.`,
"options": [
let eventObject = {
id: id,
title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,
platform: "FantasySCOTUS",
description: `${(pAffirm * 100).toFixed(2)}% (${
predictionData.numAffirm
} out of ${
predictionData.numForecasts
}) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of ${
event.prediction
}. Historically, FantasySCOTUS has chosen the correct side ${historicalPercentageCorrect} of the time.`,
options: [
{
"name": "Yes",
"probability": pAffirm,
"type": "PROBABILITY"
name: "Yes",
probability: pAffirm,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1-pAffirm,
"type": "PROBABILITY"
}
name: "No",
probability: 1 - pAffirm,
type: "PROBABILITY",
},
],
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(predictionData.numForecasts),
"stars": calculateStars("FantasySCOTUS", ({}))
}
})
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(predictionData.numForecasts),
stars: calculateStars("FantasySCOTUS", {}),
},
};
// console.log(eventObject)
results.push(eventObject)
results.push(eventObject);
}
}
}
return results
return results;
}
/* Body */
export async function fantasyscotus() {
let rawData = await fetchData()
let results = await processData(rawData)
let rawData = await fetchData();
let results = await processData(rawData);
//console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/fantasyscotus-questions.json', string);
await upsert(results, "fantasyscotus-questions")
console.log("Done")
await upsert(results, "fantasyscotus-questions");
console.log("Done");
}
//fantasyscotus()

102
src/platforms/foretold-fetch.js
View File

@ -1,20 +1,26 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let graphQLendpoint = "https://api.foretold.io/graphql"
let highQualityCommunities = ["0104d8e8-07e4-464b-8b32-74ef22b49f21", "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", "cf663021-f87f-4632-ad82-962d889a2d39", "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", "b2412a1d-0aa4-4e37-a12a-0aca9e440a96"]
let graphQLendpoint = "https://api.foretold.io/graphql";
let highQualityCommunities = [
"0104d8e8-07e4-464b-8b32-74ef22b49f21",
"c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2",
"cf663021-f87f-4632-ad82-962d889a2d39",
"47ff5c49-9c20-4f3d-bd57-1897c35cd42d",
"b2412a1d-0aa4-4e37-a12a-0aca9e440a96",
];
/* Support functions */
async function fetchAllCommunityQuestions(communityId) {
let response = await axios({
url: graphQLendpoint,
method: 'POST',
headers: ({ 'Content-Type': 'application/json' }),
data: JSON.stringify(({
method: "POST",
headers: { "Content-Type": "application/json" },
data: JSON.stringify({
query: `
query {
measurables(
@ -38,62 +44,64 @@ async function fetchAllCommunityQuestions(communityId) {
}
}
}
`
})),
`,
}),
})
.then(res => res.data)
.then(res => res.data.measurables.edges)
.then((res) => res.data)
.then((res) => res.data.measurables.edges);
//console.log(response)
return response
return response;
}
/* Body */
export async function foretold(){
let results = []
for(let community of highQualityCommunities){
let questions = await fetchAllCommunityQuestions(community)
questions = questions.map(question => question.node)
questions = questions.filter(question => question.previousAggregate) // Questions without any predictions
questions.forEach(question => {
let options = []
if(question.valueType == "PERCENTAGE"){
let probability = question.previousAggregate.value.percentage
export async function foretold() {
let results = [];
for (let community of highQualityCommunities) {
let questions = await fetchAllCommunityQuestions(community);
questions = questions.map((question) => question.node);
questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
questions.forEach((question) => {
let id = `foretold-${question.id}`;
let options = [];
if (question.valueType == "PERCENTAGE") {
let probability = question.previousAggregate.value.percentage;
options = [
{
"name": "Yes",
"probability": probability/100,
"type": "PROBABILITY"
name: "Yes",
probability: probability / 100,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1-probability/100,
"type": "PROBABILITY"
}
]
name: "No",
probability: 1 - probability / 100,
type: "PROBABILITY",
},
];
}
let result = {
"title": question.name,
"url": `https://www.foretold.io/c/${community}/m/${question.id}`,
"platform": "Foretold",
"description": "",
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Math.floor(Number(question.measurementCount) / 2),
"stars": calculateStars("Foretold", ({ }))
}
id: id,
title: question.name,
url: `https://www.foretold.io/c/${community}/m/${question.id}`,
platform: "Foretold",
description: "",
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Math.floor(Number(question.measurementCount) / 2),
stars: calculateStars("Foretold", {}),
},
/*liquidity: liquidity.toFixed(2),
tradevolume: tradevolume.toFixed(2),
address: obj.address*/
}
};
// console.log(result)
results.push(result)
})
results.push(result);
});
}
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/foretold-questions.json', string);
await upsert(results, "foretold-questions")
console.log("Done")
await upsert(results, "foretold-questions");
console.log("Done");
}
// foretold()

74
src/platforms/givewellopenphil-fetch.js
View File

@ -1,61 +1,69 @@
/* Imports */
import fs from "fs"
import axios from "axios"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let locationData = "./data/"
let locationData = "./data/";
/* Support functions */
async function fetchPage(url) {
let response = await axios({
url: url,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
},
}).then((res) => res.data);
//console.log(response)
return response
return response;
}
/* Body */
async function main() {
let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt")
let data = rawdata.toString().split("\n").filter(url => url != "");
let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt");
let data = rawdata
.toString()
.split("\n")
.filter((url) => url != "");
// console.log(data)
let results = []
let results = [];
for (let url of data) {
// console.log(url)
let page = await fetchPage(url)
let page = await fetchPage(url);
// Title
let titleraw = page.split('<meta name="twitter:title" content="')[1]
let title = titleraw.split('" />')[0]
let titleraw = page.split('<meta name="twitter:title" content="')[1];
let title = titleraw.split('" />')[0];
// Description
let internalforecasts = page.split("<h2").filter(section => section.includes("Internal forecast") || section.includes("internal forecast"))
let description = "<h2 " + internalforecasts[1]
let internalforecasts = page
.split("<h2")
.filter(
(section) =>
section.includes("Internal forecast") ||
section.includes("internal forecast")
);
let description = "<h2 " + internalforecasts[1];
let result = {
"title": title,
"url": url,
"platform": "GiveWell",
"description": description,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("GiveWell/OpenPhilanthropy", ({})),
}
} // Note: This requires some processing afterwards
title: title,
url: url,
platform: "GiveWell",
description: description,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("GiveWell/OpenPhilanthropy", {}),
},
}; // Note: This requires some processing afterwards
// console.log(result)
results.push(result)
results.push(result);
}
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/givewell-questions-unprocessed.json', string);
await upsert(results, "givewell-questions-unprocessed")
await upsert(results, "givewell-questions-unprocessed");
}
main()
main();

14
src/platforms/goodjudgment-fetch.js
View File

@ -6,6 +6,7 @@ import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
import { hash } from "../utils/hash.js";
/* Definitions */
let endpoint = "https://goodjudgment.io/superforecasts/";
@ -63,6 +64,9 @@ export async function goodjudgment() {
for (let table of jsonTable) {
// console.log(table)
let title = table[0]["0"].split("\t\t\t").splice(3)[0];
if (title != undefined) {
title = title.replaceAll("</a>", "");
let id = `goodjudgment-${hash(title)}`;
let description = table
.filter((row) => row["0"].includes("BACKGROUND:"))
.map((row) => row["0"])
@ -81,7 +85,9 @@ export async function goodjudgment() {
let options = table
.filter((row) => "4" in row)
.map((row) => ({
name: row["2"].split('<span class="qTitle">')[1].replace("</span>", ""),
name: row["2"]
.split('<span class="qTitle">')[1]
.replace("</span>", ""),
probability: Number(row["3"].split("%")[0]) / 100,
type: "PROBABILITY",
}));
@ -89,11 +95,12 @@ export async function goodjudgment() {
row[0] ? row[0].toLowerCase().includes("commentary") : false
);
// "Examples of Superforecaster Commentary" / Analysis
// The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list.
analysis = analysis ? analysis[0] : "";
// The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list.
analysis = analysis ? analysis[0] : "";
analysis = analysis ? analysis[0] : ""; // not a duplicate
// console.log(analysis)
let standardObj = {
id: id,
title: title,
url: endpoint,
platform: "Good Judgment",
@ -107,7 +114,6 @@ export async function goodjudgment() {
superforecastercommentary: analysis || "",
},
};
if (standardObj.title != undefined) {
results.push(standardObj);
}
}

285
src/platforms/goodjudmentopen-fetch.js
View File

@ -1,211 +1,242 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
import { Tabletojson } from "tabletojson"
import { calculateStars } from "../utils/stars.js"
import toMarkdown from "../utils/toMarkdown.js"
import { upsert } from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import { getCookie, applyIfCookieExists } from "../utils/getCookies.js";
import { Tabletojson } from "tabletojson";
import { calculateStars } from "../utils/stars.js";
import toMarkdown from "../utils/toMarkdown.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let htmlEndPoint = 'https://www.gjopen.com/questions?page='
let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"]
const DEBUG_MODE = "off" // "on"
const id = x => x
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
let annoyingPromptUrls = [
"https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
"https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters",
"https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
"https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
];
const DEBUG_MODE = "off"; // "on"
const id = (x) => x;
/* Support functions */
async function fetchPage(page, cookie) {
let response = await axios({
url: htmlEndPoint + page,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
Cookie: cookie,
},
}).then((res) => res.data);
//console.log(response)
return response
return response;
}
async function fetchStats(questionUrl, cookie) {
let response = await axios({
url: questionUrl + "/stats",
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie,
'Referer': questionUrl,
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
Cookie: cookie,
Referer: questionUrl,
},
}).then((res) => res.data);
//console.log(response)
// Is binary?
let isbinary = response.includes("binary?&quot;:true")
let isbinary = response.includes("binary?&quot;:true");
let options = []
let options = [];
if (isbinary) {
// Crowd percentage
let htmlElements = response.split("\n")
let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
let htmlElements = response.split("\n");
let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
// console.log(h3Element)
let crowdpercentage = h3Element.split(">")[1].split("<")[0]
let probability = Number(crowdpercentage.replace("%", "")) / 100
options.push(({
let crowdpercentage = h3Element.split(">")[1].split("<")[0];
let probability = Number(crowdpercentage.replace("%", "")) / 100;
options.push(
{
name: "Yes",
probability: probability,
type: "PROBABILITY"
}), ({
type: "PROBABILITY",
},
{
name: "No",
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY"
}))
type: "PROBABILITY",
}
);
} else {
let optionsHtmlElement = "<table" + response.split("tbody")[1] + "table>"
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
let firstTable = tablesAsJson[0]
options = firstTable.map(element => ({
name: element['0'],
probability: Number(element['1'].replace("%", "")) / 100,
type: "PROBABILITY"
}))
let optionsHtmlElement = "<table" + response.split("tbody")[1] + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0];
options = firstTable.map((element) => ({
name: element["0"],
probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY",
}));
//console.log(optionsHtmlElement)
//console.log(options)
}
// Description
let descriptionraw = response.split(`<div id="question-background" class="collapse smb">`)[1]
let descriptionprocessed1 = descriptionraw.split(`</div>`)[0]
let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
let descriptionprocessed3 = descriptionprocessed2.split("\n")
.filter(string => !string.includes("Confused? Check our"))
.join("\n")
let description = descriptionprocessed3
let descriptionraw = response.split(
`<div id="question-background" class="collapse smb">`
)[1];
let descriptionprocessed1 = descriptionraw.split(`</div>`)[0];
let descriptionprocessed2 = toMarkdown(descriptionprocessed1);
let descriptionprocessed3 = descriptionprocessed2
.split("\n")
.filter((string) => !string.includes("Confused? Check our"))
.join("\n");
let description = descriptionprocessed3;
// Number of forecasts
let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0]
let numforecasts = response
.split("prediction_sets_count&quot;:")[1]
.split(",")[0];
//console.log(numforecasts)
// Number of predictors
let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0]
let numforecasters = response
.split("predictors_count&quot;:")[1]
.split(",")[0];
//console.log(numpredictors)
// Calculate the stars
let minProbability = Math.min(...options.map(option => option.probability))
let maxProbability = Math.max(...options.map(option => option.probability))
let minProbability = Math.min(...options.map((option) => option.probability));
let maxProbability = Math.max(...options.map((option) => option.probability));
let result = {
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(numforecasts),
"numforecasters": Number(numforecasters),
"stars": calculateStars("Good Judgment Open", ({ numforecasts, minProbability, maxProbability }))
}
}
return result
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(numforecasts),
numforecasters: Number(numforecasters),
stars: calculateStars("Good Judgment Open", {
numforecasts,
minProbability,
maxProbability,
}),
},
};
return result;
}
function isSignedIn(html){
let isSignedInBool = !( html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") )
function isSignedIn(html) {
let isSignedInBool = !(
html.includes("You need to sign in or sign up before continuing") ||
html.includes("Sign up")
);
// console.log(html)
if(!isSignedInBool){
console.log("Error: Not signed in.")
if (!isSignedInBool) {
console.log("Error: Not signed in.");
}
console.log(`is signed in? ${isSignedInBool}`)
return isSignedInBool
console.log(`is signed in? ${isSignedInBool}`);
return isSignedInBool;
}
function isEnd(html){
let isEndBool = html.includes("No questions match your filter")
if(isEndBool){
function isEnd(html) {
let isEndBool = html.includes("No questions match your filter");
if (isEndBool) {
//console.log(html)
}
console.log(`IsEnd? ${isEndBool}`)
return isEndBool
console.log(`IsEnd? ${isEndBool}`);
return isEndBool;
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
return new Promise((resolve) => setTimeout(resolve, ms));
}
/* Body */
async function goodjudgmentopen_inner(cookie) {
let i = 1
let response = await fetchPage(i, cookie)
let i = 1;
let response = await fetchPage(i, cookie);
let results = []
let init = Date.now()
let results = [];
let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while(!isEnd(response) && isSignedIn(response)){
let htmlLines = response.split("\n")
DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
let h5elements = htmlLines.filter(str => str.includes("<h5> <a href="))
DEBUG_MODE == "on" ? console.log(h5elements) : id()
let j = 0
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n");
DEBUG_MODE == "on" ? htmlLines.forEach((line) => console.log(line)) : id();
let h5elements = htmlLines.filter((str) => str.includes("<h5> <a href="));
DEBUG_MODE == "on" ? console.log(h5elements) : id();
let j = 0;
for (let h5element of h5elements) {
let h5elementSplit = h5element.split('"><span>')
let url = h5elementSplit[0].split('<a href="')[1]
if(!annoyingPromptUrls.includes(url)){
let title = h5elementSplit[1].replace('</span></a></h5>', "")
await sleep(1000 + Math.random() * 1000) // don't be as noticeable
let h5elementSplit = h5element.split('"><span>');
let url = h5elementSplit[0].split('<a href="')[1];
if (!annoyingPromptUrls.includes(url)) {
let title = h5elementSplit[1].replace("</span></a></h5>", "");
await sleep(1000 + Math.random() * 1000); // don't be as noticeable
try {
let moreinfo = await fetchStats(url, cookie)
let moreinfo = await fetchStats(url, cookie);
if (moreinfo.isbinary) {
if (!moreinfo.crowdpercentage) { // then request again.
moreinfo = await fetchStats(url, cookie)
if (!moreinfo.crowdpercentage) {
// then request again.
moreinfo = await fetchStats(url, cookie);
}
}
let question = ({
"title": title,
"url": url,
"platform": "Good Judgment Open",
...moreinfo
})
if(j % 30 == 0 || DEBUG_MODE == "on"){
console.log(`Page #${i}`)
console.log(question)
let questionNumRegex = new RegExp("questions/([0-9]+)");
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
let id = `goodjudmentopen-${questionNum}`;
let question = {
id: id,
title: title,
url: url,
platform: "Good Judgment Open",
...moreinfo,
};
if (j % 30 == 0 || DEBUG_MODE == "on") {
console.log(`Page #${i}`);
console.log(question);
}
// console.log(question)
results.push(question)
results.push(question);
} catch (error) {
console.log(error)
console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
console.log(error);
console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
}
}
j = j+1
j = j + 1;
}
i = i + 1
i = i + 1;
// console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
await sleep(5000 + Math.random() * 1000) // don't be a dick to gjopen server
await sleep(5000 + Math.random() * 1000); // don't be a dick to gjopen server
try {
response = await fetchPage(i, cookie)
response = await fetchPage(i, cookie);
} catch (error) {
console.log(error)
console.log(`We encountered some error when fetching page #${i}, so it won't appear on the final json`)
console.log(error);
console.log(
`We encountered some error when fetching page #${i}, so it won't appear on the final json`
);
}
}
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/goodjudmentopen-questions.json', string);
console.log(results)
if(results.length > 0){
await upsert(results, "goodjudmentopen-questions")
}else{
console.log("Not updating results, as process was not signed in")
console.log(results);
if (results.length > 0) {
await upsert(results, "goodjudmentopen-questions");
} else {
console.log("Not updating results, as process was not signed in");
}
let end = Date.now()
let difference = end - init
console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`)
let end = Date.now();
let difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}
export async function goodjudgmentopen(){
let cookie = process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudmentopen")
await applyIfCookieExists(cookie, goodjudgmentopen_inner)
export async function goodjudgmentopen() {
let cookie =
process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudmentopen");
await applyIfCookieExists(cookie, goodjudgmentopen_inner);
}

307
src/platforms/infer-fetch.js
View File

@ -1,187 +1,201 @@
/* Imports */
import axios from "axios"
import { getCookie, applyIfCookieExists } from "../utils/getCookies.js"
import { Tabletojson } from "tabletojson"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
import axios from "axios";
import { getCookie, applyIfCookieExists } from "../utils/getCookies.js";
import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let htmlEndPoint = 'https://www.infer-pub.com/questions'
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
const DEBUG_MODE = "on"// "off"
const SLEEP_TIME_RANDOM = 7000 // miliseconds
const SLEEP_TIME_EXTRA = 2000
let htmlEndPoint = "https://www.infer-pub.com/questions";
String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace);
};
const DEBUG_MODE = "on"; // "off"
const SLEEP_TIME_RANDOM = 7000; // miliseconds
const SLEEP_TIME_EXTRA = 2000;
/* Support functions */
async function fetchPage(page, cookie) {
console.log(`Page #${page}`)
console.log(`Page #${page}`);
if (page == 1) {
cookie = cookie.split(";")[0] // Interesting that it otherwise doesn't work :(
cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :(
}
let urlEndpoint = `${htmlEndPoint}/?page=${page}`
console.log(urlEndpoint)
let urlEndpoint = `${htmlEndPoint}/?page=${page}`;
console.log(urlEndpoint);
let response = await axios({
url: urlEndpoint,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
Cookie: cookie,
},
}).then((res) => res.data);
// console.log(response)
return response
return response;
}
async function fetchStats(questionUrl, cookie) {
let response = await axios({
url: questionUrl + "/stats",
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie,
'Referer': questionUrl,
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
Cookie: cookie,
Referer: questionUrl,
},
}).then((res) => res.data);
if (response.includes("Sign up or sign in to forecast")) {
throw Error("Not logged in")
throw Error("Not logged in");
}
// Is binary?
let isbinary = response.includes("binary?&quot;:true")
let isbinary = response.includes("binary?&quot;:true");
// console.log(`is binary? ${isbinary}`)
let options = []
let options = [];
if (isbinary) {
// Crowd percentage
let htmlElements = response.split("\n")
let htmlElements = response.split("\n");
// DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
// DEBUG_MODE == "on" ? console.log(h5elements) : id()
let crowdpercentage = h3Element.split(">")[1].split("<")[0]
let probability = Number(crowdpercentage.replace("%", "")) / 100
options.push(({
let crowdpercentage = h3Element.split(">")[1].split("<")[0];
let probability = Number(crowdpercentage.replace("%", "")) / 100;
options.push(
{
name: "Yes",
probability: probability,
type: "PROBABILITY"
}), ({
type: "PROBABILITY",
},
{
name: "No",
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY"
}))
type: "PROBABILITY",
}
);
} else {
try {
let optionsBody = response.split("tbody")[1] // Previously [1], then previously [3] but they added a new table.
let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table.
// console.log(optionsBody)
let optionsHtmlElement = "<table" + optionsBody + "table>"
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
let firstTable = tablesAsJson[0]
options = firstTable.map(element => ({
name: element['0'],
probability: Number(element['1'].replace("%", "")) / 100,
type: "PROBABILITY"
}))
let optionsHtmlElement = "<table" + optionsBody + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0];
options = firstTable.map((element) => ({
name: element["0"],
probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY",
}));
} catch (error) {
let optionsBody = response.split("tbody")[3] // Catch if the error is related to table position
let optionsHtmlElement = "<table" + optionsBody + "table>"
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
let firstTable = tablesAsJson[0]
let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position
let optionsHtmlElement = "<table" + optionsBody + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0];
if (firstTable) {
options = firstTable.map(element => ({
name: element['0'],
probability: Number(element['1'].replace("%", "")) / 100,
type: "PROBABILITY"
}))
options = firstTable.map((element) => ({
name: element["0"],
probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY",
}));
} else {
// New type of question, tricky to parse the options
// Just leave options = [] for now.
// https://www.cset-foretell.com/blog/rolling-question-formats
}
}
}
// Description
let descriptionraw = response.split(`<meta name="description" content="`)[1]
let descriptionprocessed1 = descriptionraw.split(`">`)[0]
let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
let descriptionraw = response.split(`<meta name="description" content="`)[1];
let descriptionprocessed1 = descriptionraw.split(`">`)[0];
let descriptionprocessed2 = descriptionprocessed1.replace(">", "");
let descriptionprocessed3 = descriptionprocessed2.replace(
"To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.",
``
);
// console.log(descriptionprocessed3)
let descriptionprocessed4 = descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n")
let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`)
let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'")
let descriptionprocessed8 = toMarkdown(descriptionprocessed7)
let description = descriptionprocessed8
let descriptionprocessed4 = descriptionprocessed3.replaceAll(
"\r\n\r\n",
"\n"
);
let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n");
let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`);
let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'");
let descriptionprocessed8 = toMarkdown(descriptionprocessed7);
let description = descriptionprocessed8;
// Number of forecasts
//console.log(response)
//console.log(response.split("prediction_sets_count&quot;:")[1])
let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0]
let numforecasts = response
.split("prediction_sets_count&quot;:")[1]
.split(",")[0];
// console.log(numforecasts)
// Number of predictors
let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0]
let numforecasters = response
.split("predictors_count&quot;:")[1]
.split(",")[0];
// console.log(numpredictors)
let result = {
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(numforecasts),
"numforecasters": Number(numforecasters),
"stars": calculateStars("Infer", { numforecasts })
}
}
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(numforecasts),
numforecasters: Number(numforecasters),
stars: calculateStars("Infer", { numforecasts }),
},
};
return result
return result;
}
function isSignedIn(html) {
let isSignedInBool = !(html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up"))
let isSignedInBool = !(
html.includes("You need to sign in or sign up before continuing") ||
html.includes("Sign up")
);
if (!isSignedInBool) {
console.log("Error: Not signed in.")
console.log("Error: Not signed in.");
}
console.log(`Signed in? ${isSignedInBool}`)
return isSignedInBool
console.log(`Signed in? ${isSignedInBool}`);
return isSignedInBool;
}
function isEnd(html) {
let isEndBool = html.includes("No questions match your filter")
let isEndBool = html.includes("No questions match your filter");
if (isEndBool) {
//console.log(html)
}
console.log(`IsEnd? ${isEndBool}`)
return isEndBool
console.log(`IsEnd? ${isEndBool}`);
return isEndBool;
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
return new Promise((resolve) => setTimeout(resolve, ms));
}
/* Body */
async function infer_inner(cookie) {
let i = 1
let response = await fetchPage(i, cookie)
let results = []
let init = Date.now()
let i = 1;
let response = await fetchPage(i, cookie);
let results = [];
let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n")
let htmlLines = response.split("\n");
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter(str => str.includes("https://www.infer-pub.com/questions/"))
let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.infer-pub.com/questions/")
);
// console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log(response)
console.log("questionHrefs: ")
console.log(questionHrefs)
console.log("questionHrefs: ");
console.log(questionHrefs);
}
//console.log("")
@ -191,64 +205,83 @@ async function infer_inner(cookie) {
for (let questionHref of questionHrefs) {
//console.log(h4element)
let elementSplit = questionHref.split('"><span>')
let url = elementSplit[0].split('<a href="')[1]
let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "")
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
let elementSplit = questionHref.split('"><span>');
let url = elementSplit[0].split('<a href="')[1];
let title = elementSplit[1]
.replace("</h4>", "")
.replace("</h5>", "")
.replace("</span></a>", "");
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
let moreinfo = await fetchStats(url, cookie)
let question = ({
"title": title,
"url": url,
"platform": "Infer",
...moreinfo
})
if (i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) {
console.log(`Page #${i}` && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on"))
console.log(question)
let moreinfo = await fetchStats(url, cookie);
let questionNumRegex = new RegExp("questions/([0-9]+)");
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
let id = `infer-${questionNum}`;
let question = {
id: id,
title: title,
url: url,
platform: "Infer",
...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(
`Page #${i}` &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
);
console.log(question);
}
results.push(question)
results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url)
console.log(question)
console.log(url);
console.log(question);
}
} catch (error) {
console.log(error)
console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
console.log(error);
console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
}
}
i++
i++;
//i=Number(i)+1
console.log("Sleeping for ~5secs so as to not be as noticeable to the infer servers")
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
console.log(
"Sleeping for ~5secs so as to not be as noticeable to the infer servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
response = await fetchPage(i, cookie)
response = await fetchPage(i, cookie);
} catch (error) {
console.log(error)
console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
console.log(error);
console.log(
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
);
}
}
// let string = JSON.stringify(results,null, 2)
// fs.writeFileSync('./data/infer-questions.json', string);
// console.log(results)
if (results.length > 0) {
await upsert(results, "infer-questions")
await upsert(results, "infer-questions");
} else {
console.log("Not updating results, as process was not signed in")
console.log("Not updating results, as process was not signed in");
}
let end = Date.now()
let difference = end - init
console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`)
let end = Date.now();
let difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}
export async function infer() {
let cookie = process.env.INFER_COOKIE || getCookie("infer")
await applyIfCookieExists(cookie, infer_inner)
let cookie = process.env.INFER_COOKIE || getCookie("infer");
await applyIfCookieExists(cookie, infer_inner);
}

100
src/platforms/kalshi-fetch.js
View File

@ -1,11 +1,11 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
/* Support functions
async function fetchAllContractInfo(){ // for info which the polymarket graphql API
@ -14,66 +14,76 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql
return response
}
*/
async function fetchAllMarkets() { // for info which the polymarket graphql API
let response = await axios.get(jsonEndpoint).then(response => response.data.markets)
async function fetchAllMarkets() {
// for info which the polymarket graphql API
let response = await axios
.get(jsonEndpoint)
.then((response) => response.data.markets);
// console.log(response)
return response
return response;
}
async function processMarkets(markets) {
let dateNow = new Date().toISOString()
let dateNow = new Date().toISOString();
// console.log(markets)
markets = markets.filter(market => market.close_date > dateNow)
let results = await markets.map(market => {
let probability = market.last_price/100
markets = markets.filter((market) => market.close_date > dateNow);
let results = await markets.map((market) => {
let probability = market.last_price / 100;
let options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
let result = ({
"title": market.title.replaceAll("*", ""),
"url": `https://kalshi.com/markets/${market.ticker_name}`,
"platform": "Kalshi",
"description": `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Kalshi", ({shares_volume: market.volume, interest: market.open_interest})),
"yes_bid": market.yes_bid,
"yes_ask": market.yes_ask,
"spread": Math.abs(market.yes_bid-market.yes_ask),
"shares_volume": market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if.
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
];
let id = `kalshi-${market.id}`;
let result = {
id: id,
title: market.title.replaceAll("*", ""),
url: `https://kalshi.com/markets/${market.ticker_name}`,
platform: "Kalshi",
description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Kalshi", {
shares_volume: market.volume,
interest: market.open_interest,
}),
yes_bid: market.yes_bid,
yes_ask: market.yes_ask,
spread: Math.abs(market.yes_bid - market.yes_ask),
shares_volume: market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if.
// "open_interest": market.open_interest, also in shares
}
})
return result
})
},
};
return result;
});
//console.log(results.length)
// console.log(results.map(result => result.title))
// console.log(results.map(result => result.title).length)
console.log([...new Set(results.map(result => result.title))])
console.log("Number of unique questions: ", [...new Set(results.map(result => result.title))].length)
console.log([...new Set(results.map((result) => result.title))]);
console.log(
"Number of unique questions: ",
[...new Set(results.map((result) => result.title))].length
);
// console.log([...new Set(results.map(result => result.title))].length)
return results //resultsProcessed
return results; //resultsProcessed
}
/* Body */
export async function kalshi() {
let markets = await fetchAllMarkets()
let results = await processMarkets(markets) // somehow needed
let markets = await fetchAllMarkets();
let results = await processMarkets(markets); // somehow needed
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polymarket-questions.json', string);
await upsert(results, "kalshi-questions")
console.log("Done")
await upsert(results, "kalshi-questions");
console.log("Done");
}
// kalshi()

136
src/platforms/ladbrokes-fetch.js
View File

@ -1,136 +0,0 @@
/* Imports */
import axios from "axios"
import fs from "fs"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
/* Definitions */
let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`
let enpointDrillDown = (id) => `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
/* Support functions */
async function fetchUrl(url){
let response = await axios(url, {
"credentials": "include",
"headers": {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0"
},
"method": "GET",
"mode": "cors"
}).then(response => response.data)
return response
}
let processResults = async (json) => {
let results = []
let children = json.SSResponse.children
children.pop()
let ids = children.map(child => child.event.id)
let markets = []
for(let id of ids){
let marketsContainer = await fetchUrl(enpointDrillDown(id))
let marketsObj = marketsContainer.SSResponse.children[0].event
let newMarkets = marketsObj.children
newMarkets = newMarkets.map(market => ({...market.market, parent: marketsObj.name}))
markets.push(...newMarkets)
}
let normalMarkets = markets.filter(market => !market.name.includes("Specials"))
//console.log(normalMarkets)
for(let normalMarket of normalMarkets){
let title = normalMarket.parent + ": " + normalMarket.name
title = title.replace("Boris Johnson Specials", "Boris Johnson") // small kludge
let options = normalMarket.children.map(child => {
let name = child.outcome.name
let priceData = child.outcome.children[0].price
let priceDecimal = Number(priceData.priceDec)
let probability = 1/priceDecimal
let option = ({
"name":name,
"probability": probability,
"type": "PROBABILITY"
})
return option
})
// normalize probabilities
let totalValue = options
.map(element => Number(element.probability))
.reduce((a, b) => (a + b), 0)
options = options.map(element => ({
...element,
probability: Number(element.probability) / totalValue
}))
// Filter very unlikely probabilities: Not here, but on the front end
// options = options.filter(element => element.probability > 0.02)
let obj = ({
"title": title,
"url": "https://sports.ladbrokes.com/sport/politics/outrights",
"platform": "Ladbrokes",
"description": "",
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Ladbrokes", ({}))
}
})
results.push(obj)
}
let specialMarkets = markets.filter(market => market.name.includes("Specials"))
for(let specialMarket of specialMarkets){
//let title = specialMarket.parent + ": " + specialMarket.name
//console.log(title)
specialMarket.children.forEach(child => {
let name = specialMarket.parent.includes("Specials")? child.outcome.name : specialMarket.parent + ": " + child.outcome.name
name = name.replace("Boris Johnson Specials", "Boris Johnson") // small kludge
let priceData = child.outcome.children[0].price
let priceDecimal = Number(priceData.priceDec)
let probability = 1/priceDecimal
let obj = ({
"title": name,
"url": "https://sports.ladbrokes.com/sport/politics/outrights",
"platform": "LadBrokes",
"options": [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
],
"qualityindicators": {
"stars": calculateStars("Ladbrokes", ({}))
}
})
results.push(obj)
})
}
return results
}
/* Body */
export async function ladbrokes() {
let response = await fetchUrl(endpointPolitics)
let results = await processResults(response)
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/ladbrokes-questions.json', string);
await upsert(results, "ladbrokes-questions")
console.log("Done")
}
//ladbrokes()

2
src/platforms/manifoldmarkets-fetch.js
View File

@ -43,6 +43,7 @@ function showStatistics(results) {
async function processPredictions(predictions) {
let results = await predictions.map((prediction) => {
let id = `manifold-${prediction.id}`;
let probability = prediction.probability;
let options = [
{
@ -57,6 +58,7 @@ async function processPredictions(predictions) {
},
];
let result = {
id: id,
title: prediction.question,
url: prediction.url,
platform: "Manifold Markets",

193
src/platforms/metaculus-fetch.js
View File

@ -1,158 +1,161 @@
/* Imports */
import axios from "axios"
import fs from 'fs'
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import axios from "axios";
import fs from "fs";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let jsonEndPoint = 'https://www.metaculus.com/api2/questions/?page='
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
let all_questions = [];
let now = new Date().toISOString()
let now = new Date().toISOString();
let DEBUG_MODE = "off";
/* Support functions */
async function fetchMetaculusQuestions(next) {
// Numbers about a given address: how many, how much, at what price, etc.
let response = await axios(({
let response = await axios({
url: next,
method: 'GET',
headers: ({ 'Content-Type': 'application/json' })
}))
.then(res => res.data)
method: "GET",
headers: { "Content-Type": "application/json" },
}).then((res) => res.data);
// console.log(response)
return response
return response;
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function fetchMetaculusQuestionDescription(slug) {
try {
let response = await axios({
method: 'get',
url: "https://www.metaculus.com" + slug
}).then(response => response.data)
return response
method: "get",
url: "https://www.metaculus.com" + slug,
}).then((response) => response.data);
return response;
} catch (error) {
console.log(error)
console.log(`We encountered some error when attempting to fetch a metaculus page. Trying again`)
await sleep(10000)
console.log(error);
console.log(
`We encountered some error when attempting to fetch a metaculus page. Trying again`
);
await sleep(10000);
try {
let response = await axios({
method: 'get',
url: "https://www.metaculus.com" + slug
}).then(response => response.data)
method: "get",
url: "https://www.metaculus.com" + slug,
}).then((response) => response.data);
// console.log(response)
return response
return response;
} catch (error) {
console.log(`We encountered some error when attempting to fetch a metaculus page.`)
console.log("Error", error)
throw "Giving up"
console.log(
`We encountered some error when attempting to fetch a metaculus page.`
);
console.log("Error", error);
throw "Giving up";
}
}
}
/* Body */
export async function metaculus() {
// let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
// console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
// for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test
let next = "https://www.metaculus.com/api2/questions/"
let i = 1
while(next){
let next = "https://www.metaculus.com/api2/questions/";
let i = 1;
while (next) {
if (i % 20 == 0) {
console.log("Sleeping for 5secs")
await sleep(5000)
console.log("Sleeping for 5secs");
await sleep(5000);
}
console.log(`\nQuery #${i}`)
let metaculusQuestions = await fetchMetaculusQuestions(next)
console.log(`\nQuery #${i}`);
let metaculusQuestions = await fetchMetaculusQuestions(next);
let results = metaculusQuestions.results;
let j=false
let j = false;
for (let result of results) {
if (
(result.publish_time < now) &&
(now < result.resolve_time)
) {
await sleep(5000)
let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
if(!questionPage.includes("A public prediction by")){
if (result.publish_time < now && now < result.resolve_time) {
await sleep(5000);
let questionPage = await fetchMetaculusQuestionDescription(
result.page_url
);
if (!questionPage.includes("A public prediction by")) {
// console.log(questionPage)
let descriptionraw = questionPage.split(`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
let descriptionprocessed1 = descriptionraw.split("</div>")[0]
let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
let description = descriptionprocessed2
let descriptionraw = questionPage.split(
`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`
)[1]; //.split(`<div class="question__content">`)[1]
let descriptionprocessed1 = descriptionraw.split("</div>")[0];
let descriptionprocessed2 = toMarkdown(descriptionprocessed1);
let description = descriptionprocessed2;
let isbinary = result.possibilities.type == "binary"
let options = []
let isbinary = result.possibilities.type == "binary";
let options = [];
if (isbinary) {
let probability = Number(result.community_prediction.full.q2)
let probability = Number(result.community_prediction.full.q2);
options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
}
let interestingInfo = ({
"title": result.title,
"url": "https://www.metaculus.com" + result.page_url,
"platform": "Metaculus",
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(result.number_of_predictions),
"stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions }))
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
"extra": {
"resolution_data": {
"publish_time": result.publish_time,
"resolution": result.resolution,
"close_time": result.close_time,
"resolve_time": result.resolve_time
}
];
}
let id = `metaculus-${result.id}`;
let interestingInfo = {
id: id,
title: result.title,
url: "https://www.metaculus.com" + result.page_url,
platform: "Metaculus",
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(result.number_of_predictions),
stars: calculateStars("Metaculus", {
numforecasts: result.number_of_predictions,
}),
},
extra: {
resolution_data: {
publish_time: result.publish_time,
resolution: result.resolution,
close_time: result.close_time,
resolve_time: result.resolve_time,
},
},
//"status": result.status,
//"publish_time": result.publish_time,
//"close_time": result.close_time,
//"type": result.possibilities.type, // We want binary ones here.
//"last_activity_time": result.last_activity_time,
})
};
if (Number(result.number_of_predictions) >= 10) {
console.log(`- ${interestingInfo.title}`)
all_questions.push(interestingInfo)
if(!j && (i % 20 == 0)){
console.log(interestingInfo)
j = true
console.log(`- ${interestingInfo.title}`);
all_questions.push(interestingInfo);
if ((!j && i % 20 == 0) || DEBUG_MODE == "on") {
console.log(interestingInfo);
j = true;
}
}
}else{
console.log("- [Skipping public prediction]")
}
} else {
console.log("- [Skipping public prediction]");
}
}
next = metaculusQuestions.next
i = i+1
}
next = metaculusQuestions.next;
i = i + 1;
}
// let string = JSON.stringify(all_questions, null, 2)
// fs.writeFileSync('./metaculus-questions.json', string);
await upsert(all_questions, "metaculus-questions")
await upsert(all_questions, "metaculus-questions");
console.log("Done")
console.log("Done");
}
//metaculus()

133
src/platforms/polymarket-fetch.js
View File

@ -1,12 +1,13 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"// "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
let units = 10 ** 6
let graphQLendpoint =
"https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
let units = 10 ** 6;
/* Support functions
async function fetchAllContractInfo(){ // for info which the polymarket graphql API
@ -15,20 +16,24 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql
return response
}
*/
async function fetchAllContractInfo() { // for info which the polymarket graphql API
let response = await axios.get('https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1')
.then(query => query.data);
response = response.filter(res => res.closed != true)
return response
async function fetchAllContractInfo() {
// for info which the polymarket graphql API
let response = await axios
.get(
"https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1"
)
.then((query) => query.data);
response = response.filter((res) => res.closed != true);
return response;
}
async function fetchAllContractData() {
let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7 // last week
let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week
let response = await axios({
url: graphQLendpoint,
method: 'POST',
headers: ({ 'Content-Type': 'application/json' }),
data: JSON.stringify(({
method: "POST",
headers: { "Content-Type": "application/json" },
data: JSON.stringify({
query: `
{
fixedProductMarketMakers(first: 1000
@ -53,23 +58,23 @@ async function fetchAllContractData() {
}
}
}
`
})),
`,
}),
})
.then(res => res.data)
.then(res => res.data.fixedProductMarketMakers)
.then((res) => res.data)
.then((res) => res.data.fixedProductMarketMakers);
// console.log(response)
return response
return response;
}
async function fetch_all() {
let allData = await fetchAllContractData()
let allInfo = await fetchAllContractInfo()
let allData = await fetchAllContractData();
let allInfo = await fetchAllContractInfo();
let infos = ({})
let infos = {};
for (let info of allInfo) {
let address = info.marketMakerAddress
let addressLowerCase = address.toLowerCase()
let address = info.marketMakerAddress;
let addressLowerCase = address.toLowerCase();
//delete info.history
if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long")
infos[addressLowerCase] = {
@ -79,67 +84,75 @@ async function fetch_all() {
description: info.description,
outcomes: info.outcomes,
options: [],
category: info.category
}
category: info.category,
};
}
let results = []
let results = [];
for (let data of allData) {
let addressLowerCase = data.id
let addressLowerCase = data.id;
// console.log(data)
if (infos[addressLowerCase] != undefined) {
// console.log(addressLowerCase)
let info = infos[addressLowerCase]
let numforecasts = Number(data.tradesQuantity)
let tradevolume = (Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) / units
let liquidity = Number(data.liquidityParameter) / units
let id = `polymarket-${addressLowerCase.slice(0, 10)}`;
let info = infos[addressLowerCase];
let numforecasts = Number(data.tradesQuantity);
let tradevolume =
(Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) /
units;
let liquidity = Number(data.liquidityParameter) / units;
// let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2
// let percentage = Number(data.outcomeTokenPrices[0]) * 100
// let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none"
let options = []
let options = [];
for (let outcome in data.outcomeTokenPrices) {
options.push({
"name": info.outcomes[outcome],
"probability": data.outcomeTokenPrices[outcome],
"type": "PROBABILITY"
})
name: info.outcomes[outcome],
probability: data.outcomeTokenPrices[outcome],
type: "PROBABILITY",
});
}
let result = ({
"title": info.title,
"url": info.url,
"platform": "PolyMarket",
"description": info.description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": numforecasts.toFixed(0),
"liquidity": liquidity.toFixed(2),
"tradevolume": tradevolume.toFixed(2),
"stars": calculateStars("Polymarket", ({ liquidity, option: options[0], volume: tradevolume}))
let result = {
id: id,
title: info.title,
url: info.url,
platform: "PolyMarket",
description: info.description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: numforecasts.toFixed(0),
liquidity: liquidity.toFixed(2),
tradevolume: tradevolume.toFixed(2),
stars: calculateStars("Polymarket", {
liquidity,
option: options[0],
volume: tradevolume,
}),
},
extra: {
address: info.address,
},
"extra": {
"address": info.address
}
/*
*/
})
if(info.category != "Sports"){
};
if (info.category != "Sports") {
// console.log(result)
results.push(result)
results.push(result);
}
}
}
return results //resultsProcessed
return results; //resultsProcessed
}
/* Body */
export async function polymarket() {
let results = await fetch_all()
let results = await fetch_all();
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polymarket-questions.json', string);
await upsert(results, "polymarket-questions")
console.log("Done")
await upsert(results, "polymarket-questions");
console.log("Done");
}
// polymarket()

138
src/platforms/predictit-fetch.js
View File

@ -1,114 +1,116 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Support functions */
async function fetchmarkets() {
let response = await axios({
method: 'get',
url: 'https://www.predictit.org/api/marketdata/all/'
})
let openMarkets = response.data.markets.filter(market => market.status == "Open")
return openMarkets
method: "get",
url: "https://www.predictit.org/api/marketdata/all/",
});
let openMarkets = response.data.markets.filter(
(market) => market.status == "Open"
);
return openMarkets;
}
async function fetchmarketrules(market_id) {
let response = await axios({
method: 'get',
url: 'https://www.predictit.org/api/Market/' + market_id
})
return response.data.rule
method: "get",
url: "https://www.predictit.org/api/Market/" + market_id,
});
return response.data.rule;
}
async function fetchmarketvolumes(){
async function fetchmarketvolumes() {
let response = await axios({
method: 'get',
url: "https://predictit-f497e.firebaseio.com/marketStats.json"
})
return response.data
method: "get",
url: "https://predictit-f497e.firebaseio.com/marketStats.json",
});
return response.data;
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
return new Promise((resolve) => setTimeout(resolve, ms));
}
/* Body */
export async function predictit() {
let markets = await fetchmarkets()
let marketVolumes = await fetchmarketvolumes()
let markets = await fetchmarkets();
let marketVolumes = await fetchmarketvolumes();
markets = markets.map(market => ({
markets = markets.map((market) => ({
...market,
TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"]
}))
TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"],
}));
// console.log(markets)
let results = []
let results = [];
for (let market of markets) {
// console.log(market.name)
let id = `predictit-${market.id}`;
let isbinary = market.contracts.length == 1;
await sleep(3000 * (1 + Math.random()))
let descriptionraw = await fetchmarketrules(market.id)
let descriptionprocessed1 = toMarkdown(descriptionraw)
let description = descriptionprocessed1
let shares_volume = market["TotalSharesTraded"]
await sleep(3000 * (1 + Math.random()));
let descriptionraw = await fetchmarketrules(market.id);
let descriptionprocessed1 = toMarkdown(descriptionraw);
let description = descriptionprocessed1;
let shares_volume = market["TotalSharesTraded"];
// let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none"
let options = market.contracts.map(contract => ({
"name": contract.name,
"probability": contract.lastTradePrice,
"type": "PROBABILITY"
}))
let options = market.contracts.map((contract) => ({
name: contract.name,
probability: contract.lastTradePrice,
type: "PROBABILITY",
}));
let totalValue = options
.map(element => Number(element.probability))
.reduce((a, b) => (a + b), 0)
.map((element) => Number(element.probability))
.reduce((a, b) => a + b, 0);
if (options.length != 1 && totalValue > 1) {
options = options.map(element => ({
options = options.map((element) => ({
...element,
probability: Number(element.probability) / totalValue
}))
probability: Number(element.probability) / totalValue,
}));
} else if (options.length == 1) {
let option = options[0]
let probability = option["probability"]
let option = options[0];
let probability = option["probability"];
options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
];
}
let obj = ({
"title": market["name"],
"url": market.url,
"platform": "PredictIt",
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("PredictIt", ({})),
"shares_volume": shares_volume
}
})
let obj = {
id: id,
title: market["name"],
url: market.url,
platform: "PredictIt",
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("PredictIt", {}),
shares_volume: shares_volume,
},
};
// console.log(obj)
results.push(obj)
results.push(obj);
}
//console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/predictit-questions.json', string);
await upsert(results, "predictit-questions")
await upsert(results, "predictit-questions");
console.log("Done")
console.log("Done");
}

83
src/platforms/rootclaim-fetch.js
View File

@ -1,12 +1,13 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let jsonEndpoint = "https://www.rootclaim.com/main_page_stories?number=100&offset=0"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
let jsonEndpoint =
"https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
/* Support functions
async function fetchAllContractInfo(){ // for info which the polymarket graphql API
@ -15,56 +16,60 @@ async function fetchAllContractInfo(){ // for info which the polymarket graphql
return response
}
*/
async function fetchAllRootclaims() { // for info which the polymarket graphql API
let response = await axios.get(jsonEndpoint)
.then(response => response.data)
if(response.length != (response[0]+1)){
console.log(response.length)
console.log(response[0])
async function fetchAllRootclaims() {
// for info which the polymarket graphql API
let response = await axios
.get(jsonEndpoint)
.then((response) => response.data);
if (response.length != response[0] + 1) {
console.log(response.length);
console.log(response[0]);
//throw Error("Rootclaim's backend has changed.")
}
response.shift()
return response
response.shift();
return response;
}
async function fetchAndProcessData() {
let claims = await fetchAllRootclaims()
let results = []
let claims = await fetchAllRootclaims();
let results = [];
for (let claim of claims) {
let options = []
let id = `rootclaim-${claim.slug.toLowerCase()}`;
let options = [];
for (let scenario of claim.scenarios) {
//console.log(scenario)
options.push({
"name": toMarkdown(scenario.text).replace("\n","").replace("&#39;","'"),
"probability": scenario.net_prob / 100,
"type": "PROBABILITY"
})
name: toMarkdown(scenario.text).replace("\n", "").replace("&#39;", "'"),
probability: scenario.net_prob / 100,
type: "PROBABILITY",
});
}
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"
let obj = ({
"title": toMarkdown(claim.question).replace("\n",""),
"url": `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
"platform": "Rootclaim",
"description": toMarkdown(claim.background).replace("&#39;","'"),
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": 1,
"stars": calculateStars("Rootclaim", ({}))
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
let obj = {
id: id,
title: toMarkdown(claim.question).replace("\n", ""),
url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
platform: "Rootclaim",
description: toMarkdown(claim.background).replace("&#39;", "'"),
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: 1,
stars: calculateStars("Rootclaim", {}),
},
};
results.push(obj);
}
})
results.push(obj)
}
return results
return results;
}
/* Body */
export async function rootclaim() {
let results = await fetchAndProcessData()
let results = await fetchAndProcessData();
//console.log(JSON.stringify(results, null, 4))
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('rootclaim-questions.json', string);
await upsert(results, "rootclaim-questions")
console.log("Done")
await upsert(results, "rootclaim-questions");
console.log("Done");
}
//rootclaim()

146
src/platforms/smarkets-fetch.js
View File

@ -1,127 +1,127 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import fs from "fs";
import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */
let htmlEndPointEntrance = 'https://api.smarkets.com/v3/events/'
let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
/* Support functions */
async function fetchEvents(url) {
let response = await axios({
url: htmlEndPointEntrance + url,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
},
}).then((res) => res.data);
// console.log(response)
return response
return response;
}
async function fetchMarkets(eventid) {
let response = await axios({
url: `https://api.smarkets.com/v3/events/${eventid}/markets/`,
method: 'GET',
headers: ({
'Content-Type': 'text/json',
}),
method: "GET",
headers: {
"Content-Type": "text/json",
},
})
.then(res => res.data)
.then(res => res.markets)
return response
.then((res) => res.data)
.then((res) => res.markets);
return response;
}
async function fetchContracts(marketid) {
let response = await axios({
url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
},
}).then((res) => res.data);
// console.log(response)
return response
return response;
}
async function fetchPrices(marketid) {
let response = await axios({
url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
}),
})
.then(res => res.data)
method: "GET",
headers: {
"Content-Type": "text/html",
},
}).then((res) => res.data);
// console.log(response)
return response
return response;
}
/* Body */
export async function smarkets() {
let htmlPath = '?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50'
let htmlPath =
"?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
let events = []
let events = [];
while (htmlPath) {
let data = await fetchEvents(htmlPath)
events.push(...data.events)
htmlPath = data.pagination.next_page
let data = await fetchEvents(htmlPath);
events.push(...data.events);
htmlPath = data.pagination.next_page;
}
// console.log(events)
let markets = []
let markets = [];
for (let event of events) {
// console.log(Date.now())
// console.log(event.name)
let eventMarkets = await fetchMarkets(event.id)
eventMarkets = eventMarkets.map(market => ({ ...market, slug: event.full_slug }))
let eventMarkets = await fetchMarkets(event.id);
eventMarkets = eventMarkets.map((market) => ({
...market,
slug: event.full_slug,
}));
// console.log("Markets fetched")
// console.log(event.id)
// console.log(market)
markets.push(...eventMarkets)
markets.push(...eventMarkets);
//let lastPrices = await fetchPrices(market.id)
}
// console.log(markets)
let results = []
let results = [];
for (let market of markets) {
// console.log("================")
// console.log("Market: ", market)
let name = market.name
let id = `smarkets-${market.id}`;
let name = market.name;
let contracts = await fetchContracts(market.id)
let contracts = await fetchContracts(market.id);
// console.log("Contracts: ", contracts)
let prices = await fetchPrices(market.id)
let prices = await fetchPrices(market.id);
// console.log("Prices: ", prices["last_executed_prices"][market.id])
let options = {}
let options = {};
for (let contract of contracts["contracts"]) {
options[contract.id] = { name: contract.name }
options[contract.id] = { name: contract.name };
}
for (let price of prices["last_executed_prices"][market.id]) {
options[price.contract_id] = {
...options[price.contract_id],
probability: Number(price.last_executed_price),
type: "PROBABILITY"
type: "PROBABILITY",
};
}
}
options = Object.values(options)
options = Object.values(options);
let totalValue = options
.map(element => Number(element.probability))
.reduce((a, b) => (a + b), 0)
.map((element) => Number(element.probability))
.reduce((a, b) => a + b, 0);
options = options.map(element => ({
options = options.map((element) => ({
...element,
probability: Number(element.probability) / totalValue
}))
probability: Number(element.probability) / totalValue,
}));
// console.log(options)
@ -135,26 +135,24 @@ export async function smarkets() {
}
*/
let result = {
"title": name,
"url": "https://smarkets.com/event/" + market.event_id + market.slug,
"platform": "Smarkets",
"description": market.description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"stars": calculateStars("Smarkets", ({}))
}
}
id: id,
title: name,
url: "https://smarkets.com/event/" + market.event_id + market.slug,
platform: "Smarkets",
description: market.description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Smarkets", {}),
},
};
// console.log(result)
results.push(result)
results.push(result);
}
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/smarkets-questions.json', string);
await upsert(results, "smarkets-questions")
await upsert(results, "smarkets-questions");
}
//smarkets()

169
src/platforms/wildeford-fetch.js
View File

@ -1,51 +1,61 @@
/* Imports */
import fs from 'fs'
import fs from "fs";
// import axios from "axios"
import { GoogleSpreadsheet } from "google-spreadsheet"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
import { GoogleSpreadsheet } from "google-spreadsheet";
import { getCookie, applyIfCookieExists } from "../utils/getCookies.js";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js";
import { hash } from "../utils/hash.js";
/* Definitions */
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0" // spreadsheet key is the long id in the sheets URL
const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`;
// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0
/* Support functions */
const formatRow = row => {
let colNames = ["Prediction Date", "Prediction", "Odds", "Actual", "Resolution Date", "Prediction Right?", "Brier Score", "Notes"]
let result = ({})
row.forEach((col,i) => {
result[colNames[i]] = col
})
return result
}
const formatRow = (row) => {
let colNames = [
"Prediction Date",
"Prediction",
"Odds",
"Actual",
"Resolution Date",
"Prediction Right?",
"Brier Score",
"Notes",
];
let result = {};
row.forEach((col, i) => {
result[colNames[i]] = col;
});
return result;
};
async function fetchGoogleDoc(google_api_key){
async function fetchGoogleDoc(google_api_key) {
// https://gist.github.com/micalevisk/9bc831bd4b3e5a3f62b9810330129c59
let results = []
const doc = new GoogleSpreadsheet(SHEET_ID)
doc.useApiKey(google_api_key)
let results = [];
const doc = new GoogleSpreadsheet(SHEET_ID);
doc.useApiKey(google_api_key);
await doc.loadInfo() // loads document properties and worksheets
console.log('>>', doc.title)
await doc.loadInfo(); // loads document properties and worksheets
console.log(">>", doc.title);
const sheet = doc.sheetsByIndex[0]
const rows = await sheet.getRows({ offset:0, })
const sheet = doc.sheetsByIndex[0];
const rows = await sheet.getRows({ offset: 0 });
console.log('# ' +
rows[0]._sheet.headerValues.join(',')
)
console.log("# " + rows[0]._sheet.headerValues.join(","));
let isEnd = false;
for (let i in rows) {
let data = rows[i]._rawData
if(data.length == 0) isEnd = true;
if(!isEnd){
let result = ({...formatRow(data), "url": endpoint + `&range=A${(Number(i) + 2)}`})
let data = rows[i]._rawData;
if (data.length == 0) isEnd = true;
if (!isEnd) {
let result = {
...formatRow(data),
url: endpoint + `&range=A${Number(i) + 2}`,
};
// +2: +1 for the header row, +1 for starting at 1 and not at 0.
// console.log(result)
results.push(result)
results.push(result);
// console.log(rows[i])
// console.log(rows[i]._rawData)
@ -55,66 +65,71 @@ async function fetchGoogleDoc(google_api_key){
// console.log(row._rawData.join(','))
}
// console.log(results)
return(results)
return results;
}
async function processPredictions(predictions){
let currentPredictions = predictions.filter(prediction => prediction["Actual"] == "Unknown" )
let results = currentPredictions.map(prediction => {
let probability = Number(prediction["Odds"].replace("%", ""))/100
async function processPredictions(predictions) {
let currentPredictions = predictions.filter(
(prediction) => prediction["Actual"] == "Unknown"
);
let results = currentPredictions.map((prediction) => {
let title = prediction["Prediction"].replace(" [update]", "");
let id = `wildeford-${hash(title)}`;
let probability = Number(prediction["Odds"].replace("%", "")) / 100;
let options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
let result = ({
"title": prediction["Prediction"],
"url": prediction["url"],
"platform": "Peter Wildeford",
"description": prediction["Notes"] || "",
"options": options,
"timestamp": new Date(Date.parse(prediction["Prediction Date"] + "Z")).toISOString(),
"qualityindicators": {
"stars": calculateStars("Peter Wildeford"),
}
})
return result
})
results = results.map(result => ({...result, title: result.title.replace(" [update]", "")})).reverse()
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
];
let result = {
id: id,
title: title,
url: prediction["url"],
platform: "Peter Wildeford",
description: prediction["Notes"] || "",
options: options,
timestamp: new Date(
Date.parse(prediction["Prediction Date"] + "Z")
).toISOString(),
qualityindicators: {
stars: calculateStars("Peter Wildeford"),
},
};
return result;
});
let uniqueTitles = []
let uniqueResults = []
results.forEach(result => {
if(!uniqueTitles.includes(result.title)) uniqueResults.push(result)
uniqueTitles.push(result.title)
})
return(uniqueResults)
results = results.reverse();
let uniqueTitles = [];
let uniqueResults = [];
results.forEach((result) => {
if (!uniqueTitles.includes(result.title)) uniqueResults.push(result);
uniqueTitles.push(result.title);
});
return uniqueResults;
// console.log(results)
// console.log(results.map(result => result.options))
// processPredictions()
}
// processPredictions()
/* Body */
export async function wildeford_inner(google_api_key) {
let predictions = await fetchGoogleDoc(google_api_key)
let results = await processPredictions(predictions) // somehow needed
let predictions = await fetchGoogleDoc(google_api_key);
let results = await processPredictions(predictions); // somehow needed
// console.log(results.sort((a,b) => (a.title > b.title)))
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string);
await upsert(results, "wildeford-questions")
console.log("Done")
await upsert(results, "wildeford-questions");
console.log("Done");
}
//example()
export async function wildeford(){
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getCookie("google-api") // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
await applyIfCookieExists(GOOGLE_API_KEY, wildeford_inner)
export async function wildeford() {
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getCookie("google-api"); // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
await applyIfCookieExists(GOOGLE_API_KEY, wildeford_inner);
}