fix: Keeping up with cultivate labs

src/platforms/csetforetell-fetch.js

@@ -55,8 +55,9 @@ async function fetchStats(questionUrl, cookie) {
   if (isbinary) {
     // Crowd percentage
     let htmlElements = response.split("\n")
-    // console.log(htmlElements)
+    // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
     let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
+    // DEBUG_MODE == "on" ? console.log(h5elements) : id()
     let crowdpercentage = h3Element.split(">")[1].split("<")[0]
     let probability = Number(crowdpercentage.replace("%", "")) / 100
     options.push(({
@@ -171,6 +172,9 @@ async function csetforetell_inner(cookie) {
 
     let htmlLines = response.split("\n")
     let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
+    //let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
+    // console.log(questionHrefs)
+
 
     if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
       //console.log(response)

src/platforms/goodjudmentopen-fetch.js

@@ -10,6 +10,8 @@ import { upsert } from "../utils/mongo-wrapper.js"
 /* Definitions */
 let htmlEndPoint = 'https://www.gjopen.com/questions?page='
 let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"]
+const DEBUG_MODE = "off" // "on"
+const id = x => x
 
 /* Support functions */
 
@@ -136,13 +138,15 @@ function sleep(ms) {
 async function goodjudgmentopen_inner(cookie) {
   let i = 1
   let response = await fetchPage(i, cookie)
+  
   let results = []
   let init = Date.now()
   // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while(!isEnd(response) && isSignedIn(response)){
-    // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
+    DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
     let h5elements = htmlLines.filter(str => str.includes("<h5> <a href="))
+    DEBUG_MODE == "on" ? console.log(h5elements) : id()
     let j = 0
     for (let h5element of h5elements) {
       let h5elementSplit = h5element.split('"><span>')
@@ -163,7 +167,7 @@ async function goodjudgmentopen_inner(cookie) {
             "platform": "Good Judgment Open",
             ...moreinfo
           })
-          if(j % 30 == 0){
+          if(j % 30 == 0 || DEBUG_MODE == "on"){
             console.log(`Page #${i}`)
             console.log(question)
           }
@@ -189,7 +193,7 @@ async function goodjudgmentopen_inner(cookie) {
   }
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/goodjudmentopen-questions.json', string);
-
+  console.log(results)
   if(results.length > 0){
     await upsert(results, "goodjudmentopen-questions")
   }else{

src/platforms/wildeford-fetch.js

@@ -43,7 +43,7 @@ async function fetchGoogleDoc(google_api_key){
 		let data = rows[i]._rawData
 		if(data.length == 0) isEnd = true;
 		if(!isEnd){
-			let result = ({...formatRow(data), "url": endpoint + `&range=A${i + 2}`})
+			let result = ({...formatRow(data), "url": endpoint + `&range=A${(Number(i) + 2)}`})
 			// +2: +1 for the header row, +1 for starting at 1 and not at 0.
 			// console.log(result)
 			results.push(result)