From 1462580d44b9c84fd675998021db9f42024f97f0 Mon Sep 17 00:00:00 2001
From: NunoSempere <nuno.sempere@gmail.com>
Date: Fri, 5 Nov 2021 13:45:55 +0000
Subject: [PATCH] fix: Keeping up with cultivate labs

---
 heroku                                 |  0
 src/platforms/csetforetell-fetch.js    |  8 ++++++--
 src/platforms/goodjudmentopen-fetch.js | 12 ++++++++----
 src/platforms/wildeford-fetch.js       |  2 +-
 4 files changed, 15 insertions(+), 7 deletions(-)
 create mode 100644 heroku

diff --git a/heroku b/heroku
new file mode 100644
index 0000000..e69de29
diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js
index e43b3ad..ade253f 100644
--- a/src/platforms/csetforetell-fetch.js
+++ b/src/platforms/csetforetell-fetch.js
@@ -55,8 +55,9 @@ async function fetchStats(questionUrl, cookie) {
   if (isbinary) {
     // Crowd percentage
     let htmlElements = response.split("\n")
-    // console.log(htmlElements)
+    // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
     let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
+    // DEBUG_MODE == "on" ? console.log(h5elements) : id()
     let crowdpercentage = h3Element.split(">")[1].split("<")[0]
     let probability = Number(crowdpercentage.replace("%", "")) / 100
     options.push(({
@@ -170,7 +171,10 @@ async function csetforetell_inner(cookie) {
   while (!isEnd(response) && isSignedIn(response)) {
 
     let htmlLines = response.split("\n")
-    let h4elements = htmlLines.filter(str => str.includes("<h5><a href=") || str.includes("<h4><a href="))
+    let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
+    //let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
+    // console.log(questionHrefs)
+
 
     if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
       //console.log(response)
diff --git a/src/platforms/goodjudmentopen-fetch.js b/src/platforms/goodjudmentopen-fetch.js
index f28c0be..d9691f0 100644
--- a/src/platforms/goodjudmentopen-fetch.js
+++ b/src/platforms/goodjudmentopen-fetch.js
@@ -10,6 +10,8 @@ import { upsert } from "../utils/mongo-wrapper.js"
 /* Definitions */
 let htmlEndPoint = 'https://www.gjopen.com/questions?page='
 let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"]
+const DEBUG_MODE = "off" // "on"
+const id = x => x
 
 /* Support functions */
 
@@ -136,13 +138,15 @@ function sleep(ms) {
 async function goodjudgmentopen_inner(cookie) {
   let i = 1
   let response = await fetchPage(i, cookie)
+  
   let results = []
   let init = Date.now()
   // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while(!isEnd(response) && isSignedIn(response)){
-    // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
-    let h5elements = htmlLines.filter(str => str.includes("<h5><a href="))
+    DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
+    let h5elements = htmlLines.filter(str => str.includes("<h5> <a href="))
+    DEBUG_MODE == "on" ? console.log(h5elements) : id()
     let j = 0
     for (let h5element of h5elements) {
       let h5elementSplit = h5element.split('"><span>')
@@ -163,7 +167,7 @@ async function goodjudgmentopen_inner(cookie) {
             "platform": "Good Judgment Open",
             ...moreinfo
           })
-          if(j % 30 == 0){
+          if(j % 30 == 0 || DEBUG_MODE == "on"){
             console.log(`Page #${i}`)
             console.log(question)
           }
@@ -189,7 +193,7 @@ async function goodjudgmentopen_inner(cookie) {
   }
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/goodjudmentopen-questions.json', string);
-
+  console.log(results)
   if(results.length > 0){
     await upsert(results, "goodjudmentopen-questions")
   }else{
diff --git a/src/platforms/wildeford-fetch.js b/src/platforms/wildeford-fetch.js
index b9f491c..30abb0c 100644
--- a/src/platforms/wildeford-fetch.js
+++ b/src/platforms/wildeford-fetch.js
@@ -43,7 +43,7 @@ async function fetchGoogleDoc(google_api_key){
 		let data = rows[i]._rawData
 		if(data.length == 0) isEnd = true;
 		if(!isEnd){
-			let result = ({...formatRow(data), "url": endpoint + `&range=A${i + 2}`})
+			let result = ({...formatRow(data), "url": endpoint + `&range=A${(Number(i) + 2)}`})
 			// +2: +1 for the header row, +1 for starting at 1 and not at 0.
 			// console.log(result)
 			results.push(result)