Simplified prints

src/index.js

@@ -76,5 +76,5 @@ let commandLineUtility  = async () => {
 
 // console.log("1")
 // console.log(process.argv)
-// commandLineUtility()
+commandLineUtility()
 // doEverything()

src/platforms/csetforetell-fetch.js

@@ -92,7 +92,7 @@ async function fetchStats(questionUrl, cookie){
   let descriptionprocessed1 = descriptionraw.split(`">`)[0]
   let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
   let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
-  console.log(descriptionprocessed3)
+  // console.log(descriptionprocessed3)
   let descriptionprocessed4=descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
   let descriptionprocessed5=descriptionprocessed4.replaceAll("\n\n", "\n")  
   let descriptionprocessed6=descriptionprocessed5.replaceAll(""", `"`)
@@ -137,9 +137,9 @@ export async function csetforetell(){
   let response = await fetchPage(i, cookie)
   let results = []
   let init = Date.now()
-  console.log("Downloading... This might take a couple of minutes. Results will be shown.")
+  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while(!isEnd(response)){
-    console.log(`Page #${i}`)
+    // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
     let h4elements = htmlLines.filter(str => str.includes("<h4><a href=")) 
     for(let h4element of h4elements){
@@ -160,7 +160,10 @@ export async function csetforetell(){
             "platform": "CSET-foretell",
             ...moreinfo
           })
+          if(i % 10 == 0){
+            console.log(`Page #${i}`)
             console.log(question)
+          }
           results.push(question)
       } catch(error){
         console.log(error)
@@ -168,12 +171,13 @@ export async function csetforetell(){
       }
     }
     i=i+1
-    console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
+    // console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
     await sleep(5000 + Math.random()*1000) // don't be as noticeable
     
     try{
       response = await fetchPage(i, cookie)
     }catch(error){
+      console.log(error)
       console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
     }
   }

src/platforms/estimize-fetch.js

@@ -28,7 +28,7 @@ export async function estimize(){
         results.push(standardObj)
       }
   }
-  console.log(results)
+  // console.log(results)
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/estimize-questions.json', string);
   await upsert(results, "estimize-questions")

src/platforms/fantasyscotus-fetch.js

@@ -91,7 +91,7 @@ async function processData(data) {
           "stars": calculateStars("FantasySCOTUS", ({}))
         }
       })
-      console.log(eventObject)
+      // console.log(eventObject)
       results.push(eventObject)
     }
 

src/platforms/foretold-fetch.js

@@ -87,7 +87,7 @@ export async function foretold(){
           tradevolume: tradevolume.toFixed(2),
           address: obj.address*/
       }
-      console.log(result)
+      // console.log(result)
       results.push(result)
     })
   }

src/platforms/givewellopenphil-fetch.js

@@ -27,10 +27,10 @@ async function fetchPage(url) {
 async function main() {
   let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt")
   let data = rawdata.toString().split("\n").filter(url => url != "");
-  console.log(data)
+  // console.log(data)
   let results = []
   for (let url of data) {
-    console.log(url)
+    // console.log(url)
     let page = await fetchPage(url)
 
     // Title
@@ -51,7 +51,7 @@ async function main() {
         "stars": calculateStars("GiveWell/OpenPhilanthropy", ({})),
       }
     } // Note: This requires some processing afterwards
-    console.log(result)
+    // console.log(result)
     results.push(result)
   }
   // let string = JSON.stringify(results, null, 2)

src/platforms/goodjudgment-fetch.js

@@ -22,7 +22,6 @@ export async function goodjudgment() {
     jsonTable.shift() // deletes first element
     jsonTable.pop() // deletes last element
     if (endpoint == endpoints[1]) jsonTable.pop() // pop again
-    console.log(jsonTable)
     //console.log(jsonTable)
     for (let table of jsonTable) {
       let title = table[0]['0']
@@ -66,7 +65,7 @@ export async function goodjudgment() {
       results.push(standardObj)
     }
   }
-  console.log(results)
+  // console.log(results.slice(0,10))
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/goodjudgment-questions.json', string);
   await upsert(results, "goodjudgment-questions")

src/platforms/goodjudmentopen-fetch.js

@@ -27,6 +27,7 @@ function getcookie() {
     process.exit()
   }
   */
+  
 }
 
 
@@ -141,9 +142,9 @@ export async function goodjudgmentopen() {
   let response = await fetchPage(i, cookie)
   let results = []
   let init = Date.now()
-  console.log("Downloading... This might take a couple of minutes. Results will be shown.")
+  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while (!isEnd(response)) {
-    console.log(`Page #${i}`)
+    // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
     let h5elements = htmlLines.filter(str => str.includes("<h5><a href="))
     for (let h5element of h5elements) {
@@ -164,7 +165,11 @@ export async function goodjudgmentopen() {
           "platform": "Good Judgment Open",
           ...moreinfo
         })
+        if(i % 10 == 0){
+          console.log(`Page #${i}`)
           console.log(question)
+        }
+        // console.log(question)
         results.push(question)
       } catch (error) {
         console.log(error)
@@ -172,7 +177,7 @@ export async function goodjudgmentopen() {
       }
     }
     i = i + 1
-    console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
+    // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
     await sleep(5000 + Math.random() * 1000) // don't be a dick to gjopen server
 
     try {

src/platforms/hypermind-fetch.js

@@ -201,7 +201,7 @@ export async function hypermind() {
   console.log("AI in 2023")
   await sleep(1000 + Math.random() * 1000)
   let results4 = await fetchHypermindData4()
-  console.log(results4)
+  // console.log(results4)
   let results4processed = results2.map(res => {
     let description = res.props.details
     return ({

src/platforms/ladbrokes-fetch.js

@@ -127,7 +127,7 @@ let processResults = async (json) => {
 export async function ladbrokes() {
   let response = await fetchUrl(endpointPolitics)
   let results = await processResults(response)
-  console.log(results)
+  // console.log(results)
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/ladbrokes-questions.json', string);
   await upsert(results, "ladbrokes-questions")

src/platforms/metaculus-fetch.js

@@ -19,7 +19,7 @@ async function fetchMetaculusQuestions(next) {
     headers: ({ 'Content-Type': 'application/json' })
   }))
     .then(res => res.data)
-  console.log(response)
+  // console.log(response)
   return response
 }
 
@@ -36,6 +36,7 @@ async function fetchMetaculusQuestionDescription(slug) {
     }).then(response => response.data)
     return response
   } catch (error) {
+    console.log(error)
     console.log(`We encountered some error when attempting to fetch a metaculus page. Trying again`)
     await sleep(10000)
     try {
@@ -125,7 +126,7 @@ export async function metaculus() {
           //"last_activity_time": result.last_activity_time,
         })
         if (Number(result.number_of_predictions) >= 10) {
-          console.log(interestingInfo)
+          // console.log(interestingInfo)
           all_questions.push(interestingInfo)
         }
 

src/platforms/omen-fetch.js

@@ -53,7 +53,7 @@ async function fetch_all() {
       data.usdLiquidityMeasure != '0' &
       data.resolutionTimestamp == null &
       data.question.title != "ssdds") {
-      console.log(data)
+      // console.log(data)
       // console.log(data.usdLiquidityMeasure)
       let options = data.outcomeTokenMarginalPrices.map((price, slotnum) => {
         let name = `Option ${slotnum}`
@@ -77,7 +77,7 @@ async function fetch_all() {
           "stars": calculateStars("Omen", ({}))
         }
       }
-      console.log(obj)
+      // console.log(obj)
       results.push(obj)
     }
 

src/platforms/polymarket-fetch.js

@@ -83,7 +83,7 @@ async function fetch_all() {
   }
   for (let data of allData) {
     let addressLowerCase = data.id
-    console.log(data)
+    // console.log(data)
     if (combinedObj[addressLowerCase] != undefined) {
       // console.log(addressLowerCase)
       let obj = combinedObj[addressLowerCase]
@@ -129,7 +129,7 @@ async function fetch_all() {
 /* Body */
 export async function polymarket() {
   let results = await fetch_all()
-  console.log(results)
+  // console.log(results)
   // console.log(result)
   // let string = JSON.stringify(result, null, 2)
   // fs.writeFileSync('./data/polymarket-questions.json', string);

src/platforms/predictit-fetch.js

@@ -31,7 +31,7 @@ function sleep(ms) {
 /* Body */
 export async function predictit() {
   let response = await fetchmarkets()
-  console.log(response)
+  // console.log(response)
   let results = []
   for (let market of response) {
     let isbinary = market.contracts.length == 1;

src/utils/mongo-wrapper.js

@@ -30,7 +30,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
         const myDocument = await collection.findOne(filter);
     
         // Print to the console
-        console.log(myDocument);
+        console.log(myDocument.contentsArray.slice(0,10));
       } catch (err) {
         console.log(err.stack);
       }
@@ -73,5 +73,6 @@ export async function mongoRead (documentName, collectionName="metaforecastColle
     finally {
       await client.close();
     }
+    console.log(documentContents.slice(0,10));
     return documentContents
 }