New commit; shouldn't produce history
This commit is contained in:
parent
f45e302208
commit
f69814d8e7
|
|
@ -1,6 +1,9 @@
|
|||
/* Imports */
|
||||
import fs from 'fs'
|
||||
import readline from "readline"
|
||||
|
||||
import {astralcodexten} from "./platforms/astralcodexten-fetch.js"
|
||||
import {coupcast} from "./platforms/coupcast-fetch.js"
|
||||
import {csetforetell} from "./platforms/csetforetell-fetch.js"
|
||||
import {elicit} from "./platforms/elicit-fetch.js"
|
||||
import {estimize} from "./platforms/estimize-fetch.js"
|
||||
|
|
@ -16,17 +19,18 @@ import {predictit} from "./platforms/predictit-fetch.js"
|
|||
import {omen} from "./platforms/omen-fetch.js"
|
||||
import {smarkets} from "./platforms/smarkets-fetch.js"
|
||||
import {williamhill} from "./platforms/williamhill-fetch.js"
|
||||
|
||||
import {mergeEverything} from "./utils/mergeEverything.js"
|
||||
import {addToHistory} from "./utils/addToHistory.js"
|
||||
import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
|
||||
import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
|
||||
|
||||
/* Support functions */
|
||||
let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything]
|
||||
let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything]
|
||||
let functionNames = functions.map(fun => fun.name)
|
||||
|
||||
let whattodoMessage = functionNames
|
||||
.slice(0,functionNames.length-2)
|
||||
.slice(0,functionNames.length-4)
|
||||
.map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
|
||||
.join('\n') +
|
||||
`\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` +
|
||||
|
|
|
|||
100
src/platforms/astralcodexten-fetch.js
Normal file
100
src/platforms/astralcodexten-fetch.js
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
/* Imports */
|
||||
import fs from 'fs'
|
||||
import axios from "axios"
|
||||
import { calculateStars } from "../utils/stars.js"
|
||||
import { upsert } from "../utils/mongo-wrapper.js"
|
||||
|
||||
/* Definitions */
|
||||
let graphQLendpoint = "https://api.foretold.io/graphql"
|
||||
let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"]
|
||||
|
||||
/* Support functions */
|
||||
async function fetchAllCommunityQuestions(communityId) {
|
||||
let response = await axios({
|
||||
url: graphQLendpoint,
|
||||
method: 'POST',
|
||||
headers: ({ 'Content-Type': 'application/json' }),
|
||||
data: JSON.stringify(({
|
||||
query: `
|
||||
query {
|
||||
measurables(
|
||||
channelId: "${communityId}",
|
||||
states: OPEN,
|
||||
first: 500
|
||||
){
|
||||
total
|
||||
edges{
|
||||
node{
|
||||
id
|
||||
name
|
||||
valueType
|
||||
measurementCount
|
||||
previousAggregate{
|
||||
value{
|
||||
percentage
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
})),
|
||||
})
|
||||
.then(res => res.data)
|
||||
.then(res => res.data.measurables.edges)
|
||||
//console.log(response)
|
||||
return response
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
export async function astralcodexten(){
|
||||
let results = []
|
||||
for(let community of ScottAlexanderPredictions){
|
||||
let questions = await fetchAllCommunityQuestions(community)
|
||||
questions = questions.map(question => question.node)
|
||||
questions = questions.filter(question => question.previousAggregate) // Questions without any predictions
|
||||
questions.forEach(question => {
|
||||
let options = []
|
||||
if(question.valueType == "PERCENTAGE"){
|
||||
let probability = question.previousAggregate.value.percentage
|
||||
options = [
|
||||
{
|
||||
"name": "Yes",
|
||||
"probability": probability/100,
|
||||
"type": "PROBABILITY"
|
||||
},
|
||||
{
|
||||
"name": "No",
|
||||
"probability": 1-probability/100,
|
||||
"type": "PROBABILITY"
|
||||
}
|
||||
]
|
||||
}
|
||||
let result = {
|
||||
"title": question.name.split(". ")[1],
|
||||
"url": `https://www.foretold.io/c/${community}/m/${question.id}`,
|
||||
"platform": "AstralCodexTen",
|
||||
"description": "...by the end of 2021",
|
||||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": (question.measurementCount +1) / 2,
|
||||
"stars": calculateStars("AstralCodexTen", ({ }))
|
||||
}
|
||||
/*liquidity: liquidity.toFixed(2),
|
||||
tradevolume: tradevolume.toFixed(2),
|
||||
address: obj.address*/
|
||||
}
|
||||
// console.log(result)
|
||||
results.push(result)
|
||||
})
|
||||
}
|
||||
// let string = JSON.stringify(results, null, 2)
|
||||
// fs.writeFileSync('./data/foretold-questions.json', string);
|
||||
await upsert(results, "astralcodexten-questions")
|
||||
// console.log(results)
|
||||
console.log("Done")
|
||||
}
|
||||
// astralcodexten()
|
||||
172
src/platforms/coupcast-fetch.js
Normal file
172
src/platforms/coupcast-fetch.js
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/* Imports */
|
||||
import fs from "fs"
|
||||
import axios from "axios"
|
||||
import Papa from "papaparse"
|
||||
import open from "open"
|
||||
import readline from "readline"
|
||||
import {calculateStars} from "../utils/stars.js"
|
||||
import {upsert} from "../utils/mongo-wrapper.js"
|
||||
|
||||
/* Definitions */
|
||||
let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_02.csv"
|
||||
var datenow = new Date();
|
||||
var currentmonth = datenow.getMonth() + 1;
|
||||
|
||||
/* Support functions */
|
||||
let unique = arr => [...new Set(arr)]
|
||||
let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
let sanitizeCountryName = (country_name) => {
|
||||
let sanitized_name
|
||||
switch(country_name) {
|
||||
case "Cen African Rep":
|
||||
sanitized_name = "Central African Republic"
|
||||
break;
|
||||
case "Congo-Brz":
|
||||
sanitized_name = "Republic of the Congo"
|
||||
break;
|
||||
case "Congo/Zaire":
|
||||
sanitized_name = "Democratic Republic of the Congo"
|
||||
break;
|
||||
case "Czech Rep":
|
||||
sanitized_name = "Czech Republic"
|
||||
break;
|
||||
case "Dominican Rep":
|
||||
sanitized_name = "Dominican Republic"
|
||||
break;
|
||||
case "Korea North":
|
||||
sanitized_name = "North Korea"
|
||||
break;
|
||||
case "Korea South":
|
||||
sanitized_name = "South Korea"
|
||||
break;
|
||||
case "UKG":
|
||||
sanitized_name = "UK"
|
||||
break;
|
||||
default:
|
||||
sanitized_name = country_name
|
||||
}
|
||||
return sanitized_name
|
||||
}
|
||||
|
||||
|
||||
async function processArray(countryArray) {
|
||||
let results = []
|
||||
for (let country of countryArray) {
|
||||
let url = `https://www.oneearthfuture.org/activities/coup-cast`
|
||||
|
||||
// We don't really want the prediction for all months; one is enough
|
||||
// console.log(country.month)
|
||||
if(Number(country.month) == currentmonth){
|
||||
// Monthly
|
||||
country.country_name = sanitizeCountryName(country.country_name)
|
||||
let processedPrediction1 = ({
|
||||
"title": `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
|
||||
"url": url,
|
||||
"platform": "CoupCast",
|
||||
"description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
|
||||
"options": [
|
||||
{
|
||||
"name": "Yes",
|
||||
"probability": country.month_risk,
|
||||
"type": "PROBABILITY"
|
||||
},
|
||||
{
|
||||
"name": "No",
|
||||
"probability": 1 - country.month_risk,
|
||||
"type": "PROBABILITY"
|
||||
}
|
||||
],
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"stars": calculateStars("Coupcast", ({}))
|
||||
},
|
||||
"extra": {
|
||||
"country_name": country.country_name,
|
||||
"regime_type": country.regime_type,
|
||||
"month": country.month,
|
||||
"year": country.year,
|
||||
"leader_name": country.leader_name,
|
||||
"month_risk": country.month_risk,
|
||||
"annual_risk": country.annual_risk,
|
||||
"risk_change_percent": country.risk_change_percent,
|
||||
"regime_years": country.regime_years,
|
||||
"leader_years": country.leader_years,
|
||||
"country_code": country.country_code,
|
||||
"country_abb": country.country_abb
|
||||
}
|
||||
})
|
||||
|
||||
// Yearly
|
||||
let processedPrediction2 = ({
|
||||
"title": `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
|
||||
"url": url,
|
||||
"platform": "CoupCast",
|
||||
"description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
|
||||
"options": [
|
||||
{
|
||||
"name": "Yes",
|
||||
"probability": country.annual_risk,
|
||||
"type": "PROBABILITY"
|
||||
},
|
||||
{
|
||||
"name": "No",
|
||||
"probability": 1 - country.annual_risk,
|
||||
"type": "PROBABILITY"
|
||||
}
|
||||
],
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"stars": calculateStars("CoupCast", ({}))
|
||||
},
|
||||
"extra": {
|
||||
"country_name": country.country_name,
|
||||
"regime_type": country.regime_type,
|
||||
"month": country.month,
|
||||
"year": country.year,
|
||||
"leader_name": country.leader_name,
|
||||
"month_risk": country.month_risk,
|
||||
"annual_risk": country.annual_risk,
|
||||
"risk_change_percent": country.risk_change_percent,
|
||||
"regime_years": country.regime_years,
|
||||
"leader_years": country.leader_years,
|
||||
"country_code": country.country_code,
|
||||
"country_abb": country.country_abb
|
||||
}
|
||||
})
|
||||
|
||||
results.push(processedPrediction1)
|
||||
results.push(processedPrediction2)
|
||||
}
|
||||
}
|
||||
// let string = JSON.stringify(results, null, 2)
|
||||
// fs.writeFileSync('./data/elicit-questions.json', string);
|
||||
await upsert(results, "coupcast-questions")
|
||||
// console.log(results)
|
||||
console.log("Done")
|
||||
}
|
||||
|
||||
/* Body */
|
||||
let filePath = "./data/coupcast-raw-download.csv" // not used right now.
|
||||
|
||||
export async function coupcast() {
|
||||
let csvContent = await axios.get(coupCastEndpoint)
|
||||
.then(query => query.data)
|
||||
await Papa.parse(csvContent, {
|
||||
header: true,
|
||||
complete: async (results) => {
|
||||
console.log('Downloaded', results.data.length, 'records.');
|
||||
/* console.log(
|
||||
JSON.stringify(
|
||||
unique(results.data.map(country => country.country_name)),
|
||||
null,
|
||||
4
|
||||
)
|
||||
)*/
|
||||
// console.log(results.data)
|
||||
await processArray(results.data)
|
||||
}
|
||||
});
|
||||
await sleep(1000) // needed to wait for Papaparse's callback to be executed.
|
||||
}
|
||||
// coupcast()
|
||||
|
|
@ -128,7 +128,7 @@ async function csetforetell_inner(cookie){
|
|||
|
||||
let htmlLines = response.split("\n")
|
||||
let h4elements = htmlLines.filter(str => str.includes("<h5><a href=") || str.includes("<h4><a href="))
|
||||
|
||||
|
||||
if(process.env.DEBUG_MODE == "on"){
|
||||
console.log(`Page #${i}`)
|
||||
console.log(response)
|
||||
|
|
@ -142,7 +142,7 @@ async function csetforetell_inner(cookie){
|
|||
let h4elementSplit = h4element.split('"><span>')
|
||||
let url = h4elementSplit[0].split('<a href="')[1]
|
||||
//console.log(url)
|
||||
let title = h4elementSplit[1].replace('</span></a></h4>', "")
|
||||
let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "")
|
||||
await sleep(1000 + Math.random()*1000) // don't be as noticeable
|
||||
try{
|
||||
let moreinfo = await fetchStats(url, cookie)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user