From d733804a72e8e5e853d2d07a3b46fa242ec83c96 Mon Sep 17 00:00:00 2001
From: NunoSempere <nuno.sempere@gmail.com>
Date: Fri, 25 Jun 2021 18:00:00 +0200
Subject: [PATCH] Fixed annoying cset bug

---
 src/platforms/csetforetell-fetch.js | 37 +++++++++++++++++++----------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js
index 9f9c77f..50115e7 100644
--- a/src/platforms/csetforetell-fetch.js
+++ b/src/platforms/csetforetell-fetch.js
@@ -9,15 +9,18 @@ import {upsert} from "../utils/mongo-wrapper.js"
 /* Definitions */
 let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
 String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
-
+const DEBUG_MODE = "off" // "on"
 /* Support functions */
 
 async function fetchPage(page, cookie){
+  console.log(page)
   if(page==1){
     cookie=cookie.split(";")[0] // Interesting that it otherwise doesn't work :(
   }
+  let urlEndpoint = htmlEndPoint+page
+  console.log(urlEndpoint)
   let response  = await axios({
-    url: htmlEndPoint+page,
+    url: urlEndpoint,
     method: 'GET',
     headers: ({ 
     'Content-Type': 'text/html',
@@ -123,7 +126,12 @@ async function fetchStats(questionUrl, cookie){
 }
 
 function isEnd(html){
-  return html.includes("No questions match your filter")
+  let isEndBool = html.includes("No questions match your filter")
+  if(isEndBool){
+    //console.log(html)
+  }
+  console.log(`IsEnd? ${isEndBool}`)
+  return isEndBool
 }
 
 function sleep(ms) {
@@ -133,7 +141,7 @@ function sleep(ms) {
 /* Body */
 
 async function csetforetell_inner(cookie){
-  let i=0
+  let i=1
   let response = await fetchPage(i, cookie)
   let results = []
   let init = Date.now()
@@ -143,8 +151,9 @@ async function csetforetell_inner(cookie){
     let htmlLines = response.split("\n")
     let h4elements = htmlLines.filter(str => str.includes("<h5><a href=") || str.includes("<h4><a href=")) 
 
-    if(process.env.DEBUG_MODE == "on"){
-      console.log(response)
+    if(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on"){
+      //console.log(response)
+      console.log(h4elements)
     }
 
     //console.log("")
@@ -152,6 +161,7 @@ async function csetforetell_inner(cookie){
     //console.log(h4elements)
     
     for(let h4element of h4elements){
+      //console.log(h4element)
 
       let h4elementSplit = h4element.split('"><span>')
       let url = h4elementSplit[0].split('<a href="')[1]
@@ -166,12 +176,13 @@ async function csetforetell_inner(cookie){
             "platform": "CSET-foretell",
             ...moreinfo
           })
-          if(i % 30 == 0){
-            console.log(`Page #${i}`)
+          if(i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")){
+            console.log(`Page #${i}` && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on"))
             console.log(question)
           }
           results.push(question)
-          if(process.env.DEBUG_MODE == "on"){
+          if(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on"){
+            console.log(url)
             console.log(question)
           }
 
@@ -179,11 +190,12 @@ async function csetforetell_inner(cookie){
         console.log(error)
         console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
       }
-
-      i=i+1
     }
     
-    // console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
+    i++
+    //i=Number(i)+1
+
+    console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
     await sleep(5000 + Math.random()*1000) // don't be as noticeable
     
     try{
@@ -198,7 +210,6 @@ async function csetforetell_inner(cookie){
   // console.log(results)
   await upsert(results, "csetforetell-questions")
   
-  
   let end = Date.now()
   let difference = end-init
   console.log(`Took ${difference/1000} seconds, or ${difference/(1000*60)} minutes.`)