From f69814d8e76b3c093190e2efc40b4c4cbf6167e3 Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Thu, 29 Apr 2021 18:11:45 +0200 Subject: [PATCH] New commit; shouldn't produce history --- src/index.js | 8 +- src/platforms/astralcodexten-fetch.js | 100 +++++++++++++++ src/platforms/coupcast-fetch.js | 172 ++++++++++++++++++++++++++ src/platforms/csetforetell-fetch.js | 4 +- 4 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 src/platforms/astralcodexten-fetch.js create mode 100644 src/platforms/coupcast-fetch.js diff --git a/src/index.js b/src/index.js index e06d182..e429c5c 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,9 @@ /* Imports */ import fs from 'fs' import readline from "readline" + +import {astralcodexten} from "./platforms/astralcodexten-fetch.js" +import {coupcast} from "./platforms/coupcast-fetch.js" import {csetforetell} from "./platforms/csetforetell-fetch.js" import {elicit} from "./platforms/elicit-fetch.js" import {estimize} from "./platforms/estimize-fetch.js" @@ -16,17 +19,18 @@ import {predictit} from "./platforms/predictit-fetch.js" import {omen} from "./platforms/omen-fetch.js" import {smarkets} from "./platforms/smarkets-fetch.js" import {williamhill} from "./platforms/williamhill-fetch.js" + import {mergeEverything} from "./utils/mergeEverything.js" import {addToHistory} from "./utils/addToHistory.js" import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js" import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js" /* Support functions */ -let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything] +let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything] let functionNames = functions.map(fun => fun.name) let whattodoMessage = functionNames - .slice(0,functionNames.length-2) + .slice(0,functionNames.length-4) .map((functionName,i) => `[${i}]: Download predictions from ${functionName}`) .join('\n') + `\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` + diff --git a/src/platforms/astralcodexten-fetch.js b/src/platforms/astralcodexten-fetch.js new file mode 100644 index 0000000..dc7e008 --- /dev/null +++ b/src/platforms/astralcodexten-fetch.js @@ -0,0 +1,100 @@ +/* Imports */ +import fs from 'fs' +import axios from "axios" +import { calculateStars } from "../utils/stars.js" +import { upsert } from "../utils/mongo-wrapper.js" + +/* Definitions */ +let graphQLendpoint = "https://api.foretold.io/graphql" +let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"] + +/* Support functions */ +async function fetchAllCommunityQuestions(communityId) { + let response = await axios({ + url: graphQLendpoint, + method: 'POST', + headers: ({ 'Content-Type': 'application/json' }), + data: JSON.stringify(({ + query: ` + query { + measurables( + channelId: "${communityId}", + states: OPEN, + first: 500 + ){ + total + edges{ + node{ + id + name + valueType + measurementCount + previousAggregate{ + value{ + percentage + } + } + } + } + } + } + ` + })), + }) + .then(res => res.data) + .then(res => res.data.measurables.edges) + //console.log(response) + return response +} + +/* Body */ + +export async function astralcodexten(){ + let results = [] + for(let community of ScottAlexanderPredictions){ + let questions = await fetchAllCommunityQuestions(community) + questions = questions.map(question => question.node) + questions = questions.filter(question => question.previousAggregate) // Questions without any predictions + questions.forEach(question => { + let options = [] + if(question.valueType == "PERCENTAGE"){ + let probability = question.previousAggregate.value.percentage + options = [ + { + "name": "Yes", + "probability": probability/100, + "type": "PROBABILITY" + }, + { + "name": "No", + "probability": 1-probability/100, + "type": "PROBABILITY" + } + ] + } + let result = { + "title": question.name.split(". ")[1], + "url": `https://www.foretold.io/c/${community}/m/${question.id}`, + "platform": "AstralCodexTen", + "description": "...by the end of 2021", + "options": options, + "timestamp": new Date().toISOString(), + "qualityindicators": { + "numforecasts": (question.measurementCount +1) / 2, + "stars": calculateStars("AstralCodexTen", ({ })) + } + /*liquidity: liquidity.toFixed(2), + tradevolume: tradevolume.toFixed(2), + address: obj.address*/ + } + // console.log(result) + results.push(result) + }) + } + // let string = JSON.stringify(results, null, 2) + // fs.writeFileSync('./data/foretold-questions.json', string); + await upsert(results, "astralcodexten-questions") + // console.log(results) + console.log("Done") +} +// astralcodexten() diff --git a/src/platforms/coupcast-fetch.js b/src/platforms/coupcast-fetch.js new file mode 100644 index 0000000..705f0e2 --- /dev/null +++ b/src/platforms/coupcast-fetch.js @@ -0,0 +1,172 @@ +/* Imports */ +import fs from "fs" +import axios from "axios" +import Papa from "papaparse" +import open from "open" +import readline from "readline" +import {calculateStars} from "../utils/stars.js" +import {upsert} from "../utils/mongo-wrapper.js" + +/* Definitions */ +let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_02.csv" +var datenow = new Date(); +var currentmonth = datenow.getMonth() + 1; + +/* Support functions */ +let unique = arr => [...new Set(arr)] +let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +let sanitizeCountryName = (country_name) => { + let sanitized_name + switch(country_name) { + case "Cen African Rep": + sanitized_name = "Central African Republic" + break; + case "Congo-Brz": + sanitized_name = "Republic of the Congo" + break; + case "Congo/Zaire": + sanitized_name = "Democratic Republic of the Congo" + break; + case "Czech Rep": + sanitized_name = "Czech Republic" + break; + case "Dominican Rep": + sanitized_name = "Dominican Republic" + break; + case "Korea North": + sanitized_name = "North Korea" + break; + case "Korea South": + sanitized_name = "South Korea" + break; + case "UKG": + sanitized_name = "UK" + break; + default: + sanitized_name = country_name + } + return sanitized_name +} + + +async function processArray(countryArray) { + let results = [] + for (let country of countryArray) { + let url = `https://www.oneearthfuture.org/activities/coup-cast` + + // We don't really want the prediction for all months; one is enough + // console.log(country.month) + if(Number(country.month) == currentmonth){ + // Monthly + country.country_name = sanitizeCountryName(country.country_name) + let processedPrediction1 = ({ + "title": `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`, + "url": url, + "platform": "CoupCast", + "description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`, + "options": [ + { + "name": "Yes", + "probability": country.month_risk, + "type": "PROBABILITY" + }, + { + "name": "No", + "probability": 1 - country.month_risk, + "type": "PROBABILITY" + } + ], + "timestamp": new Date().toISOString(), + "qualityindicators": { + "stars": calculateStars("Coupcast", ({})) + }, + "extra": { + "country_name": country.country_name, + "regime_type": country.regime_type, + "month": country.month, + "year": country.year, + "leader_name": country.leader_name, + "month_risk": country.month_risk, + "annual_risk": country.annual_risk, + "risk_change_percent": country.risk_change_percent, + "regime_years": country.regime_years, + "leader_years": country.leader_years, + "country_code": country.country_code, + "country_abb": country.country_abb + } + }) + + // Yearly + let processedPrediction2 = ({ + "title": `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`, + "url": url, + "platform": "CoupCast", + "description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`, + "options": [ + { + "name": "Yes", + "probability": country.annual_risk, + "type": "PROBABILITY" + }, + { + "name": "No", + "probability": 1 - country.annual_risk, + "type": "PROBABILITY" + } + ], + "timestamp": new Date().toISOString(), + "qualityindicators": { + "stars": calculateStars("CoupCast", ({})) + }, + "extra": { + "country_name": country.country_name, + "regime_type": country.regime_type, + "month": country.month, + "year": country.year, + "leader_name": country.leader_name, + "month_risk": country.month_risk, + "annual_risk": country.annual_risk, + "risk_change_percent": country.risk_change_percent, + "regime_years": country.regime_years, + "leader_years": country.leader_years, + "country_code": country.country_code, + "country_abb": country.country_abb + } + }) + + results.push(processedPrediction1) + results.push(processedPrediction2) + } + } + // let string = JSON.stringify(results, null, 2) + // fs.writeFileSync('./data/elicit-questions.json', string); + await upsert(results, "coupcast-questions") + // console.log(results) + console.log("Done") +} + +/* Body */ +let filePath = "./data/coupcast-raw-download.csv" // not used right now. + +export async function coupcast() { + let csvContent = await axios.get(coupCastEndpoint) + .then(query => query.data) + await Papa.parse(csvContent, { + header: true, + complete: async (results) => { + console.log('Downloaded', results.data.length, 'records.'); + /* console.log( + JSON.stringify( + unique(results.data.map(country => country.country_name)), + null, + 4 + ) + )*/ + // console.log(results.data) + await processArray(results.data) + } + }); + await sleep(1000) // needed to wait for Papaparse's callback to be executed. +} +// coupcast() diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js index 96f038e..253b215 100644 --- a/src/platforms/csetforetell-fetch.js +++ b/src/platforms/csetforetell-fetch.js @@ -128,7 +128,7 @@ async function csetforetell_inner(cookie){ let htmlLines = response.split("\n") let h4elements = htmlLines.filter(str => str.includes("
') let url = h4elementSplit[0].split('
', "").replace('', "") await sleep(1000 + Math.random()*1000) // don't be as noticeable try{ let moreinfo = await fetchStats(url, cookie)