Test loop hypothesis

Procfile

@@ -1 +1,3 @@
-worker: node src/utils/doEverythingForScheduler.js
+// worker: node src/utils/doEverythingForScheduler.js
+worker: node src/utils/testLoopHypothesis.js
+

src/platforms/goodjudmentopen-fetch.js

@@ -8,6 +8,7 @@ import { upsert } from "../utils/mongo-wrapper.js"
 
 /* Definitions */
 let htmlEndPoint = 'https://www.gjopen.com/questions?page='
+let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"]
 
 /* Support functions */
 
@@ -27,10 +28,8 @@ function getcookie() {
     process.exit()
   }
   */
-  
 }
 
-
 async function fetchPage(page, cookie) {
   let response = await axios({
     url: htmlEndPoint + page,
@@ -147,9 +146,11 @@ export async function goodjudgmentopen() {
     // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
     let h5elements = htmlLines.filter(str => str.includes("<h5><a href="))
+    let j = 0
     for (let h5element of h5elements) {
       let h5elementSplit = h5element.split('"><span>')
       let url = h5elementSplit[0].split('<a href="')[1]
+      if(!annoyingPromptUrls.includes(url)){
         let title = h5elementSplit[1].replace('</span></a></h5>', "")
         await sleep(1000 + Math.random() * 1000) // don't be as noticeable
         try {
@@ -165,7 +166,7 @@ export async function goodjudgmentopen() {
             "platform": "Good Judgment Open",
             ...moreinfo
           })
-        if(i % 10 == 0){
+          if(j % 10 == 0){
             console.log(`Page #${i}`)
             console.log(question)
           }
@@ -176,6 +177,8 @@ export async function goodjudgmentopen() {
           console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
         }
       }
+      j = j+1
+    }
     i = i + 1
     // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
     await sleep(5000 + Math.random() * 1000) // don't be a dick to gjopen server

src/platforms/predictit-fetch.js

@@ -84,7 +84,7 @@ export async function predictit() {
       }
       
     })
-    console.log(obj)
+    // console.log(obj)
     results.push(obj)
   }
   //console.log(results)

src/utils/doEverything.js

@@ -35,8 +35,25 @@ export async function tryCatchTryAgain (fun) {
 export async function doEverything(){
     let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything]
 
+    console.log("")
+    console.log("")
+    console.log("")
+    console.log("")
+    console.log("================================")
+    console.log("STARTING UP")
+    console.log("================================")
+    console.log("")
+    console.log("")
+    console.log("")
+    console.log("")
+
     for(let fun of functions){
+        console.log("")
+        console.log("")
+        console.log("****************************")
         console.log(fun.name)
+        console.log("****************************")
         await tryCatchTryAgain(fun)
+        console.log("****************************")
     }
 }

src/utils/mongo-wrapper.js

@@ -30,7 +30,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
         const myDocument = await collection.findOne(filter);
     
         // Print to the console
-        console.log(myDocument.contentsArray.slice(0,10));
+        console.log(myDocument.contentsArray.slice(0,3));
       } catch (err) {
         console.log(err.stack);
       }

src/utils/testLoopHypothesis.js

@@ -0,0 +1,9 @@
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+let loop = async () => {
+  for (let i = 0; i < 6; i++) {
+    console.log(i)
+    await sleep(1000)
+  } 
+}
+loop()