Test loop hypothesis

Procfile

@@ -1 +1,3 @@
-worker: node src/utils/doEverythingForScheduler.js
+// worker: node src/utils/doEverythingForScheduler.js
+worker: node src/utils/testLoopHypothesis.js
+

src/platforms/goodjudmentopen-fetch.js

@@ -8,6 +8,7 @@ import { upsert } from "../utils/mongo-wrapper.js"
 
 /* Definitions */
 let htmlEndPoint = 'https://www.gjopen.com/questions?page='
+let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"]
 
 /* Support functions */
 
@@ -27,10 +28,8 @@ function getcookie() {
     process.exit()
   }
   */
-  
 }
 
-
 async function fetchPage(page, cookie) {
   let response = await axios({
     url: htmlEndPoint + page,
@@ -147,34 +146,38 @@ export async function goodjudgmentopen() {
     // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
     let h5elements = htmlLines.filter(str => str.includes("<h5><a href="))
+    let j = 0
     for (let h5element of h5elements) {
       let h5elementSplit = h5element.split('"><span>')
       let url = h5elementSplit[0].split('<a href="')[1]
-      let title = h5elementSplit[1].replace('</span></a></h5>', "")
-      await sleep(1000 + Math.random() * 1000) // don't be as noticeable
-      try {
-        let moreinfo = await fetchStats(url, cookie)
-        if (moreinfo.isbinary) {
-          if (!moreinfo.crowdpercentage) { // then request again.
-            moreinfo = await fetchStats(url, cookie)
+      if(!annoyingPromptUrls.includes(url)){
+        let title = h5elementSplit[1].replace('</span></a></h5>', "")
+        await sleep(1000 + Math.random() * 1000) // don't be as noticeable
+        try {
+          let moreinfo = await fetchStats(url, cookie)
+          if (moreinfo.isbinary) {
+            if (!moreinfo.crowdpercentage) { // then request again.
+              moreinfo = await fetchStats(url, cookie)
+            }
           }
+          let question = ({
+            "title": title,
+            "url": url,
+            "platform": "Good Judgment Open",
+            ...moreinfo
+          })
+          if(j % 10 == 0){
+            console.log(`Page #${i}`)
+            console.log(question)
+          }
+          // console.log(question)
+          results.push(question)
+        } catch (error) {
+          console.log(error)
+          console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
         }
-        let question = ({
-          "title": title,
-          "url": url,
-          "platform": "Good Judgment Open",
-          ...moreinfo
-        })
-        if(i % 10 == 0){
-          console.log(`Page #${i}`)
-          console.log(question)
-        }
-        // console.log(question)
-        results.push(question)
-      } catch (error) {
-        console.log(error)
-        console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
       }
+      j = j+1
     }
     i = i + 1
     // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")

src/platforms/predictit-fetch.js

@@ -84,7 +84,7 @@ export async function predictit() {
       }
       
     })
-    console.log(obj)
+    // console.log(obj)
     results.push(obj)
   }
   //console.log(results)

src/utils/doEverything.js

@@ -35,8 +35,25 @@ export async function tryCatchTryAgain (fun) {
 export async function doEverything(){
     let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything]
 
+    console.log("")
+    console.log("")
+    console.log("")
+    console.log("")
+    console.log("================================")
+    console.log("STARTING UP")
+    console.log("================================")
+    console.log("")
+    console.log("")
+    console.log("")
+    console.log("")
+
     for(let fun of functions){
+        console.log("")
+        console.log("")
+        console.log("****************************")
         console.log(fun.name)
+        console.log("****************************")
         await tryCatchTryAgain(fun)
+        console.log("****************************")
     }
 }

src/utils/mongo-wrapper.js

@@ -30,7 +30,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
         const myDocument = await collection.findOne(filter);
     
         // Print to the console
-        console.log(myDocument.contentsArray.slice(0,10));
+        console.log(myDocument.contentsArray.slice(0,3));
       } catch (err) {
         console.log(err.stack);
       }

src/utils/testLoopHypothesis.js

@@ -0,0 +1,9 @@
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+let loop = async () => {
+  for (let i = 0; i < 6; i++) {
+    console.log(i)
+    await sleep(1000)
+  } 
+}
+loop()