chore: Deprecate outdated or inaccessible platforms, but keep the code

This commit is contained in:
NunoSempere 2022-02-04 13:32:53 -05:00
parent ba24ef7722
commit 68a7feb6e6
17 changed files with 55011 additions and 54397 deletions

View File

@ -21,6 +21,7 @@ The following variables are currently needed to run the `master` branch:
- `CSETFORETELL_COOKIE`
- `GOODJUDGMENTOPENCOOKIE`
- `HYPERMINDCOOKIE`
- `SECRET_BETFAIR_ENDPOINT`
- `ALGOLIA_MASTER_API_KEY`, a string of 32 alphanumeric characters, like `6ofolyptm956j9uuev3q4v81vjbqrkp2` (not an actual key)
They can either be stored as process variables (e.g., something that can be accessed as `process.env.<variable name>`), or as text in `src/input/privatekeys.json`, in the same format as `src/input/privatekeys_example.json`.

File diff suppressed because one or more lines are too long

46
specification.json Normal file
View File

@ -0,0 +1,46 @@
{
"title": " String, not optional",
"url": " String. A very few number of platforms have a page for more than one prediction.",
"platform": " String, not optional",
"description": " String, optional",
"options": " Optional. Saved as a string. If this is a binary question, it gets converted to having Yes/No options. The probabilities should be probabilities, i.e., normalized to 1 first.",
"options": [
"description: optional. Some prediction platforms have distributions, rather than probabilities.",
{
"name": "Option 1",
"probability": "Number, 0 to 1",
"type": "for now just 'PROBABILITY', but could include others, like 'DISTRIBUTION'",
},
{
"name": "Option 2",
"probability": "Number, 0 to 1",
"type": "for now just 'PROBABILITY', but could include others, like 'DISTRIBUTION'",
}
],
"timestamp": " Timestamp at which metaforecast fetches the probability",
"qualityindicators": {
"stars": "required, 0 to 5",
"numforecasts": " optional",
"numforecasters": " optional",
"liquidity": "optional",
"tradevolume": "optional",
"volume": "optional",
"address": "optional",
"other properties": "optional",
},
"extra": {
"field_description": "arbitrary extra information",
"country_name": " country.country_name",
"regime_type": " country.regime_type",
"month": " country.month",
"year": " country.year",
"leader_name": " country.leader_name",
"month_risk": " country.month_risk",
"annual_risk": " country.annual_risk",
"risk_change_percent": " country.risk_change_percent",
"regime_years": " country.regime_years",
"leader_years": " country.leader_years",
"country_code": " country.country_code",
"country_abb": " country.country_abb",
"superforecastercommentary": " analysis || "",
}

View File

@ -2,28 +2,7 @@
import fs from 'fs'
import readline from "readline"
import { astralcodexten } from "./platforms/astralcodexten-fetch.js"
import { betfair } from "./platforms/betfair-fetch.js"
import { coupcast } from "./platforms/coupcast-fetch.js"
import { csetforetell } from "./platforms/csetforetell-fetch.js"
import { elicit } from "./platforms/elicit-fetch.js"
import { estimize } from "./platforms/estimize-fetch.js"
import { fantasyscotus } from "./platforms/fantasyscotus-fetch.js"
import { foretold } from "./platforms/foretold-fetch.js"
import { goodjudgment } from "./platforms/goodjudgment-fetch.js"
import { goodjudgmentopen } from "./platforms/goodjudmentopen-fetch.js"
import { hypermind } from "./platforms/hypermind-fetch.js"
import { kalshi } from "./platforms/kalshi-fetch.js"
import { ladbrokes } from "./platforms/ladbrokes-fetch.js"
import { manifoldmarkets } from "./platforms/manifoldmarkets-fetch.js"
import { metaculus } from "./platforms/metaculus-fetch.js"
import { omen } from "./platforms/omen-fetch.js"
import { polymarket } from "./platforms/polymarket-fetch.js"
import { predictit } from "./platforms/predictit-fetch.js"
import { rootclaim } from "./platforms/rootclaim-fetch.js"
import { smarkets } from "./platforms/smarkets-fetch.js"
import { wildeford } from "./platforms/wildeford-fetch.js"
import { williamhill } from "./platforms/williamhill-fetch.js"
import { platformFetchers } from "./utils/platforms.js"
import { mergeEverything } from "./utils/mergeEverything.js"
import { updateHistory } from "./utils/history/updateHistory.js"
import { rebuildAlgoliaDatabase } from "./utils/algolia.js"
@ -31,11 +10,11 @@ import { rebuildNetlifySiteWithNewData } from "./utils/rebuildNetliftySiteWithNe
import { doEverything, tryCatchTryAgain } from "./utils/doEverything.js"
/* Support functions */
let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
let functions = [...platformFetchers, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
let functionNames = functions.map(fun => fun.name)
let whattodoMessage = functionNames
.slice(0, functionNames.length - 5)
.slice(0, platformFetchers.length)
.map((functionName, i) => `[${i}]: Download predictions from ${functionName}`)
.join('\n') +
`\n[${functionNames.length - 5}]: Merge jsons them into one big json (and push it to mongodb database)` +

View File

@ -9,9 +9,9 @@ import {upsert} from "../utils/mongo-wrapper.js"
/* Definitions */
let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv"
var datenow = new Date();
var datenow = new Date();
var currentmonth = datenow.getMonth() + 1;
dd
/* Support functions */
let unique = arr => [...new Set(arr)]
let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));

View File

@ -5,7 +5,9 @@ import { calculateStars } from "../utils/stars.js"
import {upsert} from "../utils/mongo-wrapper.js"
/* Definitions */
let graphQLendpoint = 'https://api.thegraph.com/subgraphs/name/protofire/omen'
let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen"
// "https://gateway.thegraph.com/api/[api-key]/subgraphs/id/0x0503024fcc5e1bd834530e69d592dbb6e8c03968-0"
// 'https://api.thegraph.com/subgraphs/name/protofire/omen'
// https://github.com/protofire/omen-subgraph
// https://thegraph.com/explorer/subgraph/protofire/omen
@ -40,7 +42,7 @@ async function fetchAllContractData() {
})
.then(res => res.data)
.then(res => res.data.fixedProductMarketMakers)
// console.log(response)
console.log(response)
return response
}

View File

@ -0,0 +1,254 @@
/* Imports */
import axios from "axios"
import { getCookie, applyIfCookieExists } from "../utils/getCookies.js"
import { Tabletojson } from "tabletojson"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
/* Definitions */
let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
const DEBUG_MODE = "on"// "off"
const SLEEP_TIME_RANDOM=100//5000 // miliseconds
const SLEEP_TIME_EXTRA=0//1000
/* Support functions */
async function fetchPage(page, cookie) {
console.log(`Page #${page}`)
if (page == 1) {
cookie = cookie.split(";")[0] // Interesting that it otherwise doesn't work :(
}
let urlEndpoint = htmlEndPoint + page
console.log(urlEndpoint)
let response = await axios({
url: urlEndpoint,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie
}),
})
.then(res => res.data)
// console.log(response)
return response
}
async function fetchStats(questionUrl, cookie) {
let response = await axios({
url: questionUrl + "/stats",
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie,
'Referer': questionUrl,
}),
})
.then(res => res.data)
if (response.includes("Sign up or sign in to forecast")) {
throw Error("Not logged in")
}
// Is binary?
let isbinary = response.includes("binary?&quot;:true")
// console.log(`is binary? ${isbinary}`)
let options = []
if (isbinary) {
// Crowd percentage
let htmlElements = response.split("\n")
// DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
// DEBUG_MODE == "on" ? console.log(h5elements) : id()
let crowdpercentage = h3Element.split(">")[1].split("<")[0]
let probability = Number(crowdpercentage.replace("%", "")) / 100
options.push(({
name: "Yes",
probability: probability,
type: "PROBABILITY"
}), ({
name: "No",
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY"
}))
} else {
try {
let optionsBody = response.split("tbody")[1] // Previously [1], then previously [3] but they added a new table.
// console.log(optionsBody)
let optionsHtmlElement = "<table" + optionsBody + "table>"
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
let firstTable = tablesAsJson[0]
options = firstTable.map(element => ({
name: element['0'],
probability: Number(element['1'].replace("%", "")) / 100,
type: "PROBABILITY"
}))
} catch (error) {
let optionsBody = response.split("tbody")[3] // Catch if the error is related to table position
let optionsHtmlElement = "<table" + optionsBody + "table>"
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
let firstTable = tablesAsJson[0]
if (firstTable) {
options = firstTable.map(element => ({
name: element['0'],
probability: Number(element['1'].replace("%", "")) / 100,
type: "PROBABILITY"
}))
} else {
// New type of question, tricky to parse the options
// Just leave options = [] for now.
// https://www.cset-foretell.com/blog/rolling-question-formats
}
}
}
// Description
let descriptionraw = response.split(`<meta name="description" content="`)[1]
let descriptionprocessed1 = descriptionraw.split(`">`)[0]
let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
// console.log(descriptionprocessed3)
let descriptionprocessed4 = descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n")
let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`)
let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'")
let descriptionprocessed8 = toMarkdown(descriptionprocessed7)
let description = descriptionprocessed8
// Number of forecasts
//console.log(response)
//console.log(response.split("prediction_sets_count&quot;:")[1])
let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0]
// console.log(numforecasts)
// Number of predictors
let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0]
// console.log(numpredictors)
let result = {
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(numforecasts),
"numforecasters": Number(numforecasters),
"stars": calculateStars("CSET-foretell", { numforecasts })
}
}
return result
}
function isSignedIn(html) {
let isSignedInBool = !(html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up"))
if (!isSignedInBool) {
console.log("Error: Not signed in.")
}
console.log(`Signed in? ${isSignedInBool}`)
return isSignedInBool
}
function isEnd(html) {
let isEndBool = html.includes("No questions match your filter")
if (isEndBool) {
//console.log(html)
}
console.log(`IsEnd? ${isEndBool}`)
return isEndBool
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/* Body */
async function csetforetell_inner(cookie) {
let i = 1
let response = await fetchPage(i, cookie)
let results = []
let init = Date.now()
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n")
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
// console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log(response)
console.log("questionHrefs: ")
console.log(questionHrefs)
}
//console.log("")
//console.log("")
//console.log(h4elements)
for (let questionHref of questionHrefs) {
//console.log(h4element)
let elementSplit = questionHref.split('"><span>')
let url = elementSplit[0].split('<a href="')[1]
let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "")
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
try {
let moreinfo = await fetchStats(url, cookie)
let question = ({
"title": title,
"url": url,
"platform": "CSET-foretell",
...moreinfo
})
if (i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")) {
console.log(`Page #${i}` && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on"))
console.log(question)
}
results.push(question)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url)
console.log(question)
}
} catch (error) {
console.log(error)
console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
}
}
i++
//i=Number(i)+1
console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
try {
response = await fetchPage(i, cookie)
} catch (error) {
console.log(error)
console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
}
}
// let string = JSON.stringify(results,null, 2)
// fs.writeFileSync('./data/csetforetell-questions.json', string);
// console.log(results)
if (results.length > 0) {
await upsert(results, "csetforetell-questions")
} else {
console.log("Not updating results, as process was not signed in")
}
let end = Date.now()
let difference = end - init
console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`)
}
export async function infer() {
let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell")
await applyIfCookieExists(cookie, csetforetell_inner)
}

View File

@ -40,7 +40,7 @@ async function processMarkets(markets) {
}
]
let result = ({
"title": market.title,
"title": market.title.replaceAll("*", ""),
"url": `https://kalshi.com/markets/${market.ticker_name}`,
"platform": "Kalshi",
"description": `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,

View File

@ -1,29 +1,9 @@
import { astralcodexten } from "../platforms/astralcodexten-fetch.js"
import { betfair } from "../platforms/betfair-fetch.js"
import { coupcast } from "../platforms/coupcast-fetch.js"
import { csetforetell } from "../platforms/csetforetell-fetch.js"
import { elicit } from "../platforms/elicit-fetch.js"
import { estimize } from "../platforms/estimize-fetch.js"
import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js"
import { foretold } from "../platforms/foretold-fetch.js"
import { goodjudgment } from "../platforms/goodjudgment-fetch.js"
import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js"
import { hypermind } from "../platforms/hypermind-fetch.js"
import { kalshi } from "../platforms/kalshi-fetch.js"
import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
import { metaculus } from "../platforms/metaculus-fetch.js"
import { omen } from "../platforms/omen-fetch.js"
import { polymarket } from "../platforms/polymarket-fetch.js"
import { predictit } from "../platforms/predictit-fetch.js"
import { rootclaim } from "../platforms/rootclaim-fetch.js"
import { smarkets } from "../platforms/smarkets-fetch.js"
import { wildeford } from "../platforms/wildeford-fetch.js"
import { williamhill } from "../platforms/williamhill-fetch.js"
import { mergeEverything } from "./mergeEverything.js"
import { updateHistory } from "./history/updateHistory.js"
import { rebuildAlgoliaDatabase } from "./algolia.js"
import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData.js"
import { platformFetchers } from "./platforms.js"
/* Do everything */
function sleep(ms) {
@ -47,7 +27,7 @@ export async function tryCatchTryAgain(fun) {
}
export async function doEverything() {
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
let functions = [...platformFetchers, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
// Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
console.log("")

View File

@ -1,5 +1,5 @@
import { mongoRead, upsert } from "./mongo-wrapper.js";
import { platformNames } from "./platforms.js"
/* Merge everything */
// let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
let sets = [

47
src/utils/platforms.js Normal file
View File

@ -0,0 +1,47 @@
import { betfair } from "../platforms/betfair-fetch.js"
import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js"
import { foretold } from "../platforms/foretold-fetch.js"
import { goodjudgment } from "../platforms/goodjudgment-fetch.js"
import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js"
import { infer } from "../platforms/infer-fetch.js"
import { kalshi } from "../platforms/kalshi-fetch.js"
import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
import { metaculus } from "../platforms/metaculus-fetch.js"
import { polymarket } from "../platforms/polymarket-fetch.js"
import { predictit } from "../platforms/predictit-fetch.js"
import { rootclaim } from "../platforms/rootclaim-fetch.js"
import { smarkets } from "../platforms/smarkets-fetch.js"
import { wildeford } from "../platforms/wildeford-fetch.js"
import { williamhill } from "../platforms/williamhill-fetch.js"
/* Deprecated
import { astralcodexten } from "../platforms/astralcodexten-fetch.js"
import { csetforetell } from "../platforms/csetforetell-fetch.js"
import { elicit } from "../platforms/elicit-fetch.js"
import { estimize } from "../platforms/estimize-fetch.js"
import { hypermind } from "../platforms/hypermind-fetch.js"
import { coupcast } from "../platforms/coupcast-fetch.js"
*/
export const platformFetchers = [betfair, fantasyscotus, foretold, goodjudgment, goodjudgmentopen, infer, ladbrokes, kalshi, manifoldmarkets, metaculus, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill]
export const platformNames = [
"betfair",
"fantasyscotus",
"foretold",
"givewellopenphil",
"goodjudgment",
"goodjudmentopen",
"kalshi",
"ladbrokes",
"manifoldmarkets",
"metaculus",
"polymarket",
"predictit",
"rootclaim",
"smarkets",
"wildeford",
"williamhill",
"xrisk",
];
// deprecated: "astralcodexten", "csetforetell", "coupcast", "elicit", "estimize", "hypermind", "omen", etc

View File

@ -166,13 +166,15 @@ function calculateStarsLadbrokes(data) {
function calculateStarsManifoldMarkets(data) {
let nuno = (data) =>
data.volume7Days > 500 || (data.pool > 500 && data.volume7Days > 200)
data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100)
? 2
: 1;
let eli = (data) => null;
let misha = (data) => null;
let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)])
let starsInteger = Math.round(starsDecimal);
// console.log(data);
// console.log(starsInteger);
return starsInteger;
}