New commit; shouldn't produce history

src/index.js

@@ -1,6 +1,9 @@
 /* Imports */
 import fs from 'fs'
 import readline from "readline"
+
+import {astralcodexten} from "./platforms/astralcodexten-fetch.js"
+import {coupcast} from "./platforms/coupcast-fetch.js"
 import {csetforetell} from "./platforms/csetforetell-fetch.js"
 import {elicit} from "./platforms/elicit-fetch.js"
 import {estimize} from "./platforms/estimize-fetch.js"
@@ -16,17 +19,18 @@ import {predictit} from "./platforms/predictit-fetch.js"
 import {omen} from "./platforms/omen-fetch.js"
 import {smarkets} from "./platforms/smarkets-fetch.js"
 import {williamhill} from "./platforms/williamhill-fetch.js"
+
 import {mergeEverything} from "./utils/mergeEverything.js"
 import {addToHistory} from "./utils/addToHistory.js"
 import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
 import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
 
 /* Support functions */
-let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything]
+let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything]
 let functionNames =  functions.map(fun => fun.name)
 
 let whattodoMessage = functionNames
-    .slice(0,functionNames.length-2)
+    .slice(0,functionNames.length-4)
     .map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
     .join('\n') +
   `\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` + 

src/platforms/astralcodexten-fetch.js

@@ -0,0 +1,100 @@
+/* Imports */
+import fs from 'fs'
+import axios from "axios"
+import { calculateStars } from "../utils/stars.js"
+import { upsert } from "../utils/mongo-wrapper.js"
+
+/* Definitions */
+let graphQLendpoint = "https://api.foretold.io/graphql"
+let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"]
+
+/* Support functions */
+async function fetchAllCommunityQuestions(communityId) {
+  let response = await axios({
+    url: graphQLendpoint,
+    method: 'POST',
+    headers: ({ 'Content-Type': 'application/json' }),
+    data: JSON.stringify(({
+      query: `
+      query {
+        measurables(
+          channelId: "${communityId}",
+          states: OPEN,
+          first: 500
+        ){
+		      total
+          edges{
+            node{
+              id
+              name
+              valueType
+              measurementCount
+              previousAggregate{
+                value{
+                  percentage
+                }
+              }
+            }
+          }
+        }
+      }
+      `
+    })),
+  })
+    .then(res => res.data)
+    .then(res => res.data.measurables.edges)
+  //console.log(response)
+  return response
+}
+
+/* Body */
+
+export async function astralcodexten(){
+  let results = []
+  for(let community of ScottAlexanderPredictions){
+    let questions = await fetchAllCommunityQuestions(community)
+    questions = questions.map(question => question.node)
+    questions = questions.filter(question => question.previousAggregate) // Questions without any predictions
+    questions.forEach(question => {
+      let options = []
+      if(question.valueType == "PERCENTAGE"){
+        let probability = question.previousAggregate.value.percentage
+        options = [
+          {
+            "name": "Yes",
+            "probability": probability/100,
+            "type": "PROBABILITY"
+          },
+          {
+            "name": "No",
+            "probability": 1-probability/100,
+            "type": "PROBABILITY"
+          }
+        ]
+      }
+      let result = {
+          "title": question.name.split(". ")[1],
+          "url": `https://www.foretold.io/c/${community}/m/${question.id}`,
+          "platform": "AstralCodexTen",
+          "description": "...by the end of 2021",
+          "options": options,
+          "timestamp": new Date().toISOString(),
+          "qualityindicators": {
+            "numforecasts": (question.measurementCount +1) / 2,
+            "stars": calculateStars("AstralCodexTen", ({  }))
+            }
+          /*liquidity: liquidity.toFixed(2),
+          tradevolume: tradevolume.toFixed(2),
+          address: obj.address*/
+      }
+      // console.log(result)
+      results.push(result)
+    })
+  }
+  // let string = JSON.stringify(results, null, 2)
+  // fs.writeFileSync('./data/foretold-questions.json', string);
+  await upsert(results, "astralcodexten-questions")
+  // console.log(results)
+  console.log("Done")
+}
+// astralcodexten()

src/platforms/coupcast-fetch.js

@@ -0,0 +1,172 @@
+/* Imports */
+import fs from "fs"
+import axios from "axios"
+import Papa from "papaparse"
+import open from "open"
+import readline from "readline"
+import {calculateStars} from "../utils/stars.js"
+import {upsert} from "../utils/mongo-wrapper.js"
+
+/* Definitions */
+let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_02.csv"
+var datenow = new Date();
+var currentmonth = datenow.getMonth() + 1; 
+
+/* Support functions */
+let unique = arr => [...new Set(arr)]
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+let sanitizeCountryName = (country_name) => {
+  let sanitized_name
+  switch(country_name) {
+    case "Cen African Rep":
+      sanitized_name = "Central African Republic"
+      break;
+    case "Congo-Brz":
+      sanitized_name = "Republic of the Congo"
+      break;
+    case "Congo/Zaire":
+      sanitized_name = "Democratic Republic of the Congo"
+      break;
+    case "Czech Rep":
+      sanitized_name = "Czech Republic"
+      break;
+    case "Dominican Rep":
+      sanitized_name = "Dominican Republic"
+      break;
+    case "Korea North":
+      sanitized_name = "North Korea"
+      break;
+    case "Korea South":
+      sanitized_name = "South Korea"
+      break;
+    case "UKG":
+      sanitized_name = "UK"
+      break;
+    default:
+      sanitized_name = country_name
+  }
+  return sanitized_name
+}
+
+
+async function processArray(countryArray) {
+  let results = []
+  for (let country of countryArray) {
+    let url = `https://www.oneearthfuture.org/activities/coup-cast`
+
+    // We don't really want the prediction for all months; one is enough
+    // console.log(country.month)
+    if(Number(country.month) == currentmonth){
+      // Monthly
+      country.country_name = sanitizeCountryName(country.country_name)
+      let processedPrediction1 = ({
+        "title": `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
+        "url": url,
+        "platform": "CoupCast",
+        "description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years.  ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
+        "options": [
+          {
+            "name": "Yes",
+            "probability": country.month_risk,
+            "type": "PROBABILITY"
+          },
+          {
+            "name": "No",
+            "probability": 1 - country.month_risk,
+            "type": "PROBABILITY"
+          }
+        ],
+        "timestamp": new Date().toISOString(),
+        "qualityindicators": {
+          "stars": calculateStars("Coupcast", ({}))
+        },
+        "extra": {
+          "country_name": country.country_name,
+          "regime_type": country.regime_type,
+          "month": country.month,
+          "year": country.year,
+          "leader_name": country.leader_name,
+          "month_risk": country.month_risk,
+          "annual_risk": country.annual_risk,
+          "risk_change_percent": country.risk_change_percent,
+          "regime_years": country.regime_years,
+          "leader_years": country.leader_years,
+          "country_code": country.country_code,
+          "country_abb": country.country_abb
+        }
+      })
+      
+      // Yearly
+      let processedPrediction2 = ({
+        "title": `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
+        "url": url,
+        "platform": "CoupCast",
+        "description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
+        "options": [
+          {
+            "name": "Yes",
+            "probability": country.annual_risk,
+            "type": "PROBABILITY"
+          },
+          {
+            "name": "No",
+            "probability": 1 - country.annual_risk,
+            "type": "PROBABILITY"
+          }
+        ],
+        "timestamp": new Date().toISOString(),
+        "qualityindicators": {
+          "stars": calculateStars("CoupCast", ({}))
+        },
+        "extra": {
+          "country_name": country.country_name,
+          "regime_type": country.regime_type,
+          "month": country.month,
+          "year": country.year,
+          "leader_name": country.leader_name,
+          "month_risk": country.month_risk,
+          "annual_risk": country.annual_risk,
+          "risk_change_percent": country.risk_change_percent,
+          "regime_years": country.regime_years,
+          "leader_years": country.leader_years,
+          "country_code": country.country_code,
+          "country_abb": country.country_abb
+        }
+      })
+
+      results.push(processedPrediction1)
+      results.push(processedPrediction2)
+      }
+  }
+  // let string = JSON.stringify(results, null, 2)
+  // fs.writeFileSync('./data/elicit-questions.json', string);
+  await upsert(results, "coupcast-questions")
+  // console.log(results)
+  console.log("Done")
+}
+
+/* Body */
+let filePath = "./data/coupcast-raw-download.csv" // not used right now.
+
+export async function coupcast() {
+  let csvContent = await axios.get(coupCastEndpoint)
+    .then(query => query.data)
+  await Papa.parse(csvContent, {
+    header: true,
+    complete: async (results) => {
+      console.log('Downloaded', results.data.length, 'records.');
+      /* console.log(
+        JSON.stringify(
+          unique(results.data.map(country => country.country_name)),
+          null, 
+          4
+        )
+      )*/
+      // console.log(results.data)
+      await processArray(results.data)
+    }
+  });
+  await sleep(1000) // needed to wait for Papaparse's callback to be executed. 
+}
+// coupcast()

src/platforms/csetforetell-fetch.js

@@ -128,7 +128,7 @@ async function csetforetell_inner(cookie){
     
     let htmlLines = response.split("\n")
     let h4elements = htmlLines.filter(str => str.includes("<h5><a href=") || str.includes("<h4><a href=")) 
-
+  
     if(process.env.DEBUG_MODE == "on"){
       console.log(`Page #${i}`)
       console.log(response)
@@ -142,7 +142,7 @@ async function csetforetell_inner(cookie){
       let h4elementSplit = h4element.split('"><span>')
       let url = h4elementSplit[0].split('<a href="')[1]
       //console.log(url)
-      let title = h4elementSplit[1].replace('</span></a></h4>', "")
+      let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "")
       await sleep(1000 + Math.random()*1000) // don't be as noticeable
       try{
         let moreinfo = await fetchStats(url, cookie)