feat: regiggled metaculus predictions to deal with new interface

package.json

@@ -5,7 +5,9 @@
   "main": "src/index.js",
   "scripts": {
     "start": "node src/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "reload": "heroku run:detached node src/utils/doEverythingForScheduler.js"
+
   },
   "repository": {
     "type": "git",

src/platforms/astralcodexten-fetch.js

@@ -93,6 +93,7 @@ export async function astralcodexten(){
   }
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/foretold-questions.json', string);
+  // console.log(JSON.stringify(results, null, 2))
   await upsert(results, "astralcodexten-questions")
   // console.log(results)
   console.log("Done")

src/platforms/metaculus-fetch.js

@@ -81,6 +81,7 @@ export async function metaculus() {
       ) {
         await sleep(5000) 
         let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
+        if(!questionPage.includes("A public prediction by")){
           let descriptionraw = questionPage.split(`<div  class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
           let descriptionprocessed1 = descriptionraw.split("</div>")[0]
           let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
@@ -134,6 +135,10 @@ export async function metaculus() {
               j = true
             }
           }
+        }else{
+          console.log("- [Skipping public prediction]")
+        }
+        
 
       }
     }
@@ -142,7 +147,7 @@ export async function metaculus() {
   }
 
   // let string = JSON.stringify(all_questions, null, 2)
-  // fs.writeFileSync('./data/metaculus-questions.json', string);
+  fs.writeFileSync('./metaculus-questions.json', string);
   await upsert(all_questions, "metaculus-questions")
 
   console.log("Done")

src/utils/downloadFrontpage.js

@@ -2,9 +2,9 @@ import fs from "fs"
 
 import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
 
-let filename = 'frontpage.json'
+let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json'
 let shuffle = (array) => {
-  // https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array
+  // https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
   let currentIndex = array.length,  randomIndex;
 
   // While there remain elements to shuffle...
@@ -32,7 +32,7 @@ let main = async () => {
   
   let string = JSON.stringify(json, null, 2)
   fs.writeFileSync(filename, string);
-  console.log(`File downloaded to ./${filename}`)
+  console.log(`File downloaded to ${filename}`)
 
   let end = Date.now()
   let difference = end - init