feat: regiggled metaculus predictions to deal with new interface

package.json

@@ -5,7 +5,9 @@
   "main": "src/index.js",
   "scripts": {
     "start": "node src/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "reload": "heroku run:detached node src/utils/doEverythingForScheduler.js"
+
   },
   "repository": {
     "type": "git",

src/platforms/astralcodexten-fetch.js

@@ -93,6 +93,7 @@ export async function astralcodexten(){
   }
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/foretold-questions.json', string);
+  // console.log(JSON.stringify(results, null, 2))
   await upsert(results, "astralcodexten-questions")
   // console.log(results)
   console.log("Done")

src/platforms/metaculus-fetch.js

@@ -79,61 +79,66 @@ export async function metaculus() {
         (result.publish_time < now) &&
         (now < result.resolve_time)
       ) {
-        await sleep(5000)
+        await sleep(5000) 
         let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
-        let descriptionraw = questionPage.split(`<div  class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
+        if(!questionPage.includes("A public prediction by")){
-        let descriptionprocessed1 = descriptionraw.split("</div>")[0]
+          let descriptionraw = questionPage.split(`<div  class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
-        let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
+          let descriptionprocessed1 = descriptionraw.split("</div>")[0]
-        let description = descriptionprocessed2
+          let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
-
+          let description = descriptionprocessed2
-        let isbinary = result.possibilities.type == "binary"
+  
-        let options = []
+          let isbinary = result.possibilities.type == "binary"
-        if (isbinary) {
+          let options = []
-          let probability = Number(result.community_prediction.full.q2)
+          if (isbinary) {
-          options = [
+            let probability = Number(result.community_prediction.full.q2)
-            {
+            options = [
-              "name": "Yes",
+              {
-              "probability": probability,
+                "name": "Yes",
-              "type": "PROBABILITY"
+                "probability": probability,
-            },
+                "type": "PROBABILITY"
-            {
+              },
-              "name": "No",
+              {
-              "probability": 1 - probability,
+                "name": "No",
-              "type": "PROBABILITY"
+                "probability": 1 - probability,
+                "type": "PROBABILITY"
+              }
+            ]
+          }
+          let interestingInfo = ({
+            "title": result.title,
+            "url": "https://www.metaculus.com" + result.page_url,
+            "platform": "Metaculus",
+            "description": description,
+            "options": options,
+            "timestamp": new Date().toISOString(),
+            "qualityindicators": {
+              "numforecasts": Number(result.number_of_predictions),
+              "resolution_data": {
+                "publish_time": result.publish_time,
+                "resolution": result.resolution,
+                "close_time": result.close_time,
+                "resolve_time": result.resolve_time                  
+              },
+              "stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions }))
+            }
+            //"status": result.status,
+            //"publish_time": result.publish_time,
+            //"close_time": result.close_time,
+            //"type": result.possibilities.type, // We want binary ones here.
+            //"last_activity_time": result.last_activity_time,
+          })
+          if (Number(result.number_of_predictions) >= 10) {
+            console.log(`- ${interestingInfo.title}`)
+            all_questions.push(interestingInfo)
+            if(!j && (i % 20 == 0)){
+              console.log(interestingInfo)
+              j = true
             }
-          ]
-        }
-        let interestingInfo = ({
-          "title": result.title,
-          "url": "https://www.metaculus.com" + result.page_url,
-          "platform": "Metaculus",
-          "description": description,
-          "options": options,
-          "timestamp": new Date().toISOString(),
-          "qualityindicators": {
-            "numforecasts": Number(result.number_of_predictions),
-            "resolution_data": {
-              "publish_time": result.publish_time,
-              "resolution": result.resolution,
-              "close_time": result.close_time,
-              "resolve_time": result.resolve_time                  
-            },
-            "stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions }))
-          }
-          //"status": result.status,
-          //"publish_time": result.publish_time,
-          //"close_time": result.close_time,
-          //"type": result.possibilities.type, // We want binary ones here.
-          //"last_activity_time": result.last_activity_time,
-        })
-        if (Number(result.number_of_predictions) >= 10) {
-          console.log(`- ${interestingInfo.title}`)
-          all_questions.push(interestingInfo)
-          if(!j && (i % 20 == 0)){
-            console.log(interestingInfo)
-            j = true
           }
+        }else{
+          console.log("- [Skipping public prediction]")
         }
+        
 
       }
     }
@@ -142,7 +147,7 @@ export async function metaculus() {
   }
 
   // let string = JSON.stringify(all_questions, null, 2)
-  // fs.writeFileSync('./data/metaculus-questions.json', string);
+  fs.writeFileSync('./metaculus-questions.json', string);
   await upsert(all_questions, "metaculus-questions")
 
   console.log("Done")

src/utils/downloadFrontpage.js

@@ -2,9 +2,9 @@ import fs from "fs"
 
 import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
 
-let filename = 'frontpage.json'
+let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json'
 let shuffle = (array) => {
-  // https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array
+  // https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
   let currentIndex = array.length,  randomIndex;
 
   // While there remain elements to shuffle...
@@ -32,7 +32,7 @@ let main = async () => {
   
   let string = JSON.stringify(json, null, 2)
   fs.writeFileSync(filename, string);
-  console.log(`File downloaded to ./${filename}`)
+  console.log(`File downloaded to ${filename}`)
 
   let end = Date.now()
   let difference = end - init