Simplified prints

src/index.js

@@ -76,5 +76,5 @@ let commandLineUtility  = async () => {
 
 // console.log("1")
 // console.log(process.argv)
-// commandLineUtility()
+commandLineUtility()
 // doEverything()

src/platforms/csetforetell-fetch.js

@@ -41,7 +41,7 @@ async function fetchPage(page, cookie){
     }),
   })
   .then(res => res.data)
-  //console.log(response)
+  // console.log(response)
   return response
 }
 
@@ -92,7 +92,7 @@ async function fetchStats(questionUrl, cookie){
   let descriptionprocessed1 = descriptionraw.split(`">`)[0]
   let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
   let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
-  console.log(descriptionprocessed3)
+  // console.log(descriptionprocessed3)
   let descriptionprocessed4=descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
   let descriptionprocessed5=descriptionprocessed4.replaceAll("\n\n", "\n")  
   let descriptionprocessed6=descriptionprocessed5.replaceAll(""", `"`)
@@ -101,11 +101,11 @@ async function fetchStats(questionUrl, cookie){
   let description = descriptionprocessed8
   // Number of forecasts
   let numforecasts = response.split("prediction_sets_count":")[1].split(",")[0]
-  //console.log(numforecasts)
+  // console.log(numforecasts)
   
   // Number of predictors
   let numforecasters = response.split("predictors_count":")[1].split(",")[0]
-  //console.log(numpredictors)
+  // console.log(numpredictors)
   
   let result = {
     "description": description, 
@@ -137,9 +137,9 @@ export async function csetforetell(){
   let response = await fetchPage(i, cookie)
   let results = []
   let init = Date.now()
-  console.log("Downloading... This might take a couple of minutes. Results will be shown.")
+  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while(!isEnd(response)){
-    console.log(`Page #${i}`)
+    // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
     let h4elements = htmlLines.filter(str => str.includes("<h4><a href=")) 
     for(let h4element of h4elements){
@@ -160,7 +160,10 @@ export async function csetforetell(){
             "platform": "CSET-foretell",
             ...moreinfo
           })
-          console.log(question)
+          if(i % 10 == 0){
+            console.log(`Page #${i}`)
+            console.log(question)
+          }
           results.push(question)
       } catch(error){
         console.log(error)
@@ -168,12 +171,13 @@ export async function csetforetell(){
       }
     }
     i=i+1
-    console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
+    // console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
     await sleep(5000 + Math.random()*1000) // don't be as noticeable
     
     try{
       response = await fetchPage(i, cookie)
     }catch(error){
+      console.log(error)
       console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
     }
   }

src/platforms/estimize-fetch.js

@@ -28,7 +28,7 @@ export async function estimize(){
         results.push(standardObj)
       }
   }
-  console.log(results)
+  // console.log(results)
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/estimize-questions.json', string);
   await upsert(results, "estimize-questions")

src/platforms/fantasyscotus-fetch.js

@@ -91,7 +91,7 @@ async function processData(data) {
           "stars": calculateStars("FantasySCOTUS", ({}))
         }
       })
-      console.log(eventObject)
+      // console.log(eventObject)
       results.push(eventObject)
     }
 

src/platforms/foretold-fetch.js

@@ -87,7 +87,7 @@ export async function foretold(){
           tradevolume: tradevolume.toFixed(2),
           address: obj.address*/
       }
-      console.log(result)
+      // console.log(result)
       results.push(result)
     })
   }

src/platforms/givewellopenphil-fetch.js

@@ -27,10 +27,10 @@ async function fetchPage(url) {
 async function main() {
   let rawdata = fs.readFileSync("./src/input/givewellopenphil-urls.txt")
   let data = rawdata.toString().split("\n").filter(url => url != "");
-  console.log(data)
+  // console.log(data)
   let results = []
   for (let url of data) {
-    console.log(url)
+    // console.log(url)
     let page = await fetchPage(url)
 
     // Title
@@ -51,7 +51,7 @@ async function main() {
         "stars": calculateStars("GiveWell/OpenPhilanthropy", ({})),
       }
     } // Note: This requires some processing afterwards
-    console.log(result)
+    // console.log(result)
     results.push(result)
   }
   // let string = JSON.stringify(results, null, 2)

src/platforms/goodjudgment-fetch.js

@@ -22,7 +22,6 @@ export async function goodjudgment() {
     jsonTable.shift() // deletes first element
     jsonTable.pop() // deletes last element
     if (endpoint == endpoints[1]) jsonTable.pop() // pop again
-    console.log(jsonTable)
     //console.log(jsonTable)
     for (let table of jsonTable) {
       let title = table[0]['0']
@@ -66,7 +65,7 @@ export async function goodjudgment() {
       results.push(standardObj)
     }
   }
-  console.log(results)
+  // console.log(results.slice(0,10))
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/goodjudgment-questions.json', string);
   await upsert(results, "goodjudgment-questions")

src/platforms/goodjudmentopen-fetch.js

@@ -27,6 +27,7 @@ function getcookie() {
     process.exit()
   }
   */
+  
 }
 
 
@@ -141,9 +142,9 @@ export async function goodjudgmentopen() {
   let response = await fetchPage(i, cookie)
   let results = []
   let init = Date.now()
-  console.log("Downloading... This might take a couple of minutes. Results will be shown.")
+  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
   while (!isEnd(response)) {
-    console.log(`Page #${i}`)
+    // console.log(`Page #${i}`)
     let htmlLines = response.split("\n")
     let h5elements = htmlLines.filter(str => str.includes("<h5><a href="))
     for (let h5element of h5elements) {
@@ -164,7 +165,11 @@ export async function goodjudgmentopen() {
           "platform": "Good Judgment Open",
           ...moreinfo
         })
-        console.log(question)
+        if(i % 10 == 0){
+          console.log(`Page #${i}`)
+          console.log(question)
+        }
+        // console.log(question)
         results.push(question)
       } catch (error) {
         console.log(error)
@@ -172,7 +177,7 @@ export async function goodjudgmentopen() {
       }
     }
     i = i + 1
-    console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
+    // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
     await sleep(5000 + Math.random() * 1000) // don't be a dick to gjopen server
 
     try {

src/platforms/hypermind-fetch.js

@@ -201,7 +201,7 @@ export async function hypermind() {
   console.log("AI in 2023")
   await sleep(1000 + Math.random() * 1000)
   let results4 = await fetchHypermindData4()
-  console.log(results4)
+  // console.log(results4)
   let results4processed = results2.map(res => {
     let description = res.props.details
     return ({

src/platforms/ladbrokes-fetch.js

@@ -127,7 +127,7 @@ let processResults = async (json) => {
 export async function ladbrokes() {
   let response = await fetchUrl(endpointPolitics)
   let results = await processResults(response)
-  console.log(results)
+  // console.log(results)
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/ladbrokes-questions.json', string);
   await upsert(results, "ladbrokes-questions")

src/platforms/metaculus-fetch.js

@@ -19,7 +19,7 @@ async function fetchMetaculusQuestions(next) {
     headers: ({ 'Content-Type': 'application/json' })
   }))
     .then(res => res.data)
-  console.log(response)
+  // console.log(response)
   return response
 }
 
@@ -36,6 +36,7 @@ async function fetchMetaculusQuestionDescription(slug) {
     }).then(response => response.data)
     return response
   } catch (error) {
+    console.log(error)
     console.log(`We encountered some error when attempting to fetch a metaculus page. Trying again`)
     await sleep(10000)
     try {
@@ -76,7 +77,7 @@ export async function metaculus() {
         (result.publish_time < now) &&
         (now < result.resolve_time)
       ) {
-        //console.log(result)
+        // console.log(result)
         await sleep(1000)
         let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
         let descriptionraw = questionPage.split(`<div class="question__content">`)[1]
@@ -125,7 +126,7 @@ export async function metaculus() {
           //"last_activity_time": result.last_activity_time,
         })
         if (Number(result.number_of_predictions) >= 10) {
-          console.log(interestingInfo)
+          // console.log(interestingInfo)
           all_questions.push(interestingInfo)
         }
 

src/platforms/omen-fetch.js

@@ -40,7 +40,7 @@ async function fetchAllContractData() {
   })
     .then(res => res.data)
     .then(res => res.data.fixedProductMarketMakers)
-  //console.log(response)
+  // console.log(response)
   return response
 }
 
@@ -53,8 +53,8 @@ async function fetch_all() {
       data.usdLiquidityMeasure != '0' &
       data.resolutionTimestamp == null &
       data.question.title != "ssdds") {
-      console.log(data)
-      //console.log(data.usdLiquidityMeasure)
+      // console.log(data)
+      // console.log(data.usdLiquidityMeasure)
       let options = data.outcomeTokenMarginalPrices.map((price, slotnum) => {
         let name = `Option ${slotnum}`
         if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 0) name = "Yes"
@@ -77,7 +77,7 @@ async function fetch_all() {
           "stars": calculateStars("Omen", ({}))
         }
       }
-      console.log(obj)
+      // console.log(obj)
       results.push(obj)
     }
 

src/platforms/polymarket-fetch.js

@@ -58,7 +58,7 @@ async function fetchAllContractData() {
   })
     .then(res => res.data)
     .then(res => res.data.fixedProductMarketMakers)
-  //console.log(response)
+  // console.log(response)
   return response
 }
 
@@ -83,9 +83,9 @@ async function fetch_all() {
   }
   for (let data of allData) {
     let addressLowerCase = data.id
-    console.log(data)
+    // console.log(data)
     if (combinedObj[addressLowerCase] != undefined) {
-      //console.log(addressLowerCase)
+      // console.log(addressLowerCase)
       let obj = combinedObj[addressLowerCase]
       let numforecasts = data.tradesQuantity
       let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2
@@ -122,14 +122,14 @@ async function fetch_all() {
     }
   }
   let result = Object.values(combinedObj)
-  //console.log(result)
+  // console.log(result)
   return result
 }
 
 /* Body */
 export async function polymarket() {
   let results = await fetch_all()
-  console.log(results)
+  // console.log(results)
   // console.log(result)
   // let string = JSON.stringify(result, null, 2)
   // fs.writeFileSync('./data/polymarket-questions.json', string);

src/platforms/predictit-fetch.js

@@ -31,7 +31,7 @@ function sleep(ms) {
 /* Body */
 export async function predictit() {
   let response = await fetchmarkets()
-  console.log(response)
+  // console.log(response)
   let results = []
   for (let market of response) {
     let isbinary = market.contracts.length == 1;

src/platforms/smarkets-fetch.js

@@ -19,7 +19,7 @@ async function fetchEvents(url) {
     }),
   })
     .then(res => res.data)
-  //console.log(response)
+  // console.log(response)
   return response
 }
 
@@ -45,7 +45,7 @@ async function fetchContracts(marketid) {
     }),
   })
     .then(res => res.data)
-  //console.log(response)
+  // console.log(response)
   return response
 
 }
@@ -59,7 +59,7 @@ async function fetchPrices(marketid) {
     }),
   })
     .then(res => res.data)
-  //console.log(response)
+  // console.log(response)
   return response
 
 }
@@ -75,32 +75,32 @@ export async function smarkets() {
     events.push(...data.events)
     htmlPath = data.pagination.next_page
   }
-  //console.log(events)
+  // console.log(events)
 
   let markets = []
   for (let event of events) {
-    //console.log(Date.now())
-    //console.log(event.name)
+    // console.log(Date.now())
+    // console.log(event.name)
     let eventMarkets = await fetchMarkets(event.id)
     eventMarkets = eventMarkets.map(market => ({ ...market, slug: event.full_slug }))
-    //console.log("Markets fetched")
-    //console.log(event.id)
-    //console.log(market)
+    // console.log("Markets fetched")
+    // console.log(event.id)
+    // console.log(market)
     markets.push(...eventMarkets)
     //let lastPrices = await fetchPrices(market.id)
   }
-  //console.log(markets)
+  // console.log(markets)
 
   let results = []
   for (let market of markets) {
-    //console.log("================")
-    //console.log("Market: ", market)
+    // console.log("================")
+    // console.log("Market: ", market)
     let name = market.name
 
     let contracts = await fetchContracts(market.id)
-    //console.log("Contracts: ", contracts)
+    // console.log("Contracts: ", contracts)
     let prices = await fetchPrices(market.id)
-    //console.log("Prices: ", prices["last_executed_prices"][market.id])
+    // console.log("Prices: ", prices["last_executed_prices"][market.id])
 
     let options = {}
     for (let contract of contracts["contracts"]) {
@@ -123,7 +123,7 @@ export async function smarkets() {
       probability: Number(element.probability) / totalValue
     }))
 
-    //console.log(options)
+    // console.log(options)
 
     /*
     if(contracts["contracts"].length == 2){
@@ -146,10 +146,10 @@ export async function smarkets() {
       }
       
     }
-    //console.log(result)
+    // console.log(result)
     results.push(result)
   }
-  //console.log(results)
+  // console.log(results)
 
   // let string = JSON.stringify(results, null, 2)
   // fs.writeFileSync('./data/smarkets-questions.json', string);

src/utils/mongo-wrapper.js

@@ -30,7 +30,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
         const myDocument = await collection.findOne(filter);
     
         // Print to the console
-        console.log(myDocument);
+        console.log(myDocument.contentsArray.slice(0,10));
       } catch (err) {
         console.log(err.stack);
       }
@@ -73,5 +73,6 @@ export async function mongoRead (documentName, collectionName="metaforecastColle
     finally {
       await client.close();
     }
+    console.log(documentContents.slice(0,10));
     return documentContents
 }