Reinstated history functionality. This is for all platforms except for Metaculus (and Estimize). I'll see how it goes, and i particular, if I exceed the limit for July. If I do, the thing to do is to have a history for every platform for every month, rather than a history for all platforms for every month. It would make things slightly messier, though

src/index.js

@@ -20,14 +20,13 @@ import {predictit} from "./platforms/predictit-fetch.js"
 import {rootclaim} from "./platforms/rootclaim-fetch.js"
 import {smarkets} from "./platforms/smarkets-fetch.js"
 import {williamhill} from "./platforms/williamhill-fetch.js"
-
 import {mergeEverything} from "./utils/mergeEverything.js"
-import {addToHistory} from "./utils/addToHistory.js"
+import {updateHistory} from "./utils/history/updateHistory.js"
 import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
 import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
 
 /* Support functions */
-let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything]
+let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything]
 let functionNames =  functions.map(fun => fun.name)
 
 let whattodoMessage = functionNames
@@ -35,7 +34,7 @@ let whattodoMessage = functionNames
     .map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
     .join('\n') +
   `\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` + 
-  `\n[${functionNames.length-3}]: Add to history file` + 
+  `\n[${functionNames.length-3}]: Update history` + 
   `\n[${functionNames.length-2}]: Rebuild netlify site with new data` + 
   // `\n[${functionNames.length-1}]: Add to history` + 
   `\n[${functionNames.length-1}]: All of the above` + 

src/utils/doEverything.js

@@ -17,7 +17,7 @@ import {rootclaim} from "../platforms/rootclaim-fetch.js"
 import {smarkets} from "../platforms/smarkets-fetch.js"
 import {williamhill} from "../platforms/williamhill-fetch.js"
 import {mergeEverything} from "./mergeEverything.js"
-import {addToHistory} from "./addToHistory.js"
+import {updateHistory} from "./history/updateHistory.js"
 import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js"
 
 /* Do everything */
@@ -37,7 +37,7 @@ export async function tryCatchTryAgain (fun) {
     }
 }
 export async function doEverything(){
-    let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, /*addToHistory,*/rebuildNetlifySiteWithNewData] 
+    let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData] 
     // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
 
     console.log("")

src/utils/history/addToHistory.js

@@ -1,13 +1,17 @@
 import { writeFileSync } from "fs"
-import { mongoReadWithReadCredentials, upsert } from "./mongo-wrapper.js"
+import { mongoReadWithReadCredentials, upsert } from "../mongo-wrapper.js"
 let mongoRead = mongoReadWithReadCredentials
 let isEmptyArray = arr => arr.length == 0
 
 export async function addToHistory(){
-  // throw new Error("Not today")
-  let currentJSON = await mongoRead("metaforecasts")
+  let currentDate = new Date()
+  let dateUpToMonth = currentDate.toISOString().slice(0,7).replace("-", "_")
+
+  let currentJSONwithMetaculus = await mongoRead("metaforecasts")
+  let currentJSON = currentJSONwithMetaculus.filter(element => element.platform != "Metaculus" && element.platform != "Estimize") // without Metaculus
+  // console.log(currentJSON.slice(0,20))
   // console.log(currentJSON)
-  let historyJSON = await mongoRead("metaforecast_history")
+  let historyJSON = await mongoRead(`metaforecast_history_${dateUpToMonth}`,"metaforecastHistory")
   // console.log(historyJSON)
 
   let currentForecastsWithAHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
@@ -55,7 +59,8 @@ export async function addToHistory(){
     newHistoryJSON.push(newHistoryElement)
   }
 
-  upsert(newHistoryJSON, "metaforecast_history")
+  await upsert(newHistoryJSON, `metaforecast_history_${dateUpToMonth}`, "metaforecastHistory")
+
   // console.log(newHistoryJSON.slice(0,5))
   // writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
   // writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
@@ -63,6 +68,6 @@ export async function addToHistory(){
   /*
   let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
   */
-  console.log(new Date().toISOString())
+  //console.log(new Date().toISOString())
 }
-// addToHistory()
+// updateHistory()

src/utils/history/createHistoryForMonth.js

@@ -0,0 +1,27 @@
+import { mongoRead, upsert } from "../mongo-wrapper.js"
+
+export async function createHistoryForMonth(){
+    let currentDate = new Date()
+    let dateUpToMonth = currentDate.toISOString().slice(0,7).replace("-", "_")
+    let metaforecasts = await mongoRead("metaforecasts")
+    let metaforecastsHistorySeed = metaforecasts.map(element => {
+        // let moreoriginsdata = element.author ? ({author: element.author}) : ({})
+        return ({
+            title: element.title,
+            url: element.url,
+            platform: element.platform,
+            moreoriginsdata: element.moreoriginsdata || {},
+            description: element.description,
+            history: [{
+                timestamp: element.timestamp,
+                options: element.options,
+                qualityindicators: element.qualityindicators
+            }],
+            extra: element.extra || {}
+         })
+    }).filter(element => element.platform != "Metaculus" && element.platform != "Estimize")
+    //console.log(metaforecastsHistorySeed)
+    await upsert(metaforecastsHistorySeed, `metaforecast_history_${dateUpToMonth}`, "metaforecastHistory")
+
+}
+////createInitialHistory()

src/utils/history/old/addToHistory_old.js

@@ -0,0 +1,68 @@
+import { writeFileSync } from "fs"
+import { mongoReadWithReadCredentials, upsert } from "../mongo-wrapper.js"
+let mongoRead = mongoReadWithReadCredentials
+let isEmptyArray = arr => arr.length == 0
+
+export async function addToHistory(){
+  // throw new Error("Not today")
+  let currentJSON = await mongoRead("metaforecasts")
+  // console.log(currentJSON)
+  let historyJSON = await mongoRead("metaforecast_history")
+  // console.log(historyJSON)
+
+  let currentForecastsWithAHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
+  // console.log(currentForecastsWithAHistory)
+
+  let currentForecastsWithoutAHistory = currentJSON.filter(element => isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
+  // console.log(currentForecastsWithoutAHistory)
+  
+  // Add both types of forecast
+  let newHistoryJSON = []
+  for(let historyElement of historyJSON){
+    let correspondingNewElementArray = currentForecastsWithAHistory.filter(element => historyElement.title == element.title && historyElement.url == element.url )
+    // console.log(correspondingNewElement)
+    if(!isEmptyArray(correspondingNewElementArray)){
+      let correspondingNewElement = correspondingNewElementArray[0]
+      let timeStampOfNewElement = correspondingNewElement.timestamp
+      let doesHistoryAlreadyContainElement = historyElement.history.map(element => element.timestamp).includes(timeStampOfNewElement)
+      if(!doesHistoryAlreadyContainElement){
+          let historyWithNewElement = historyElement["history"].concat({
+              "timestamp": correspondingNewElement.timestamp,
+              "options": correspondingNewElement.options,
+              "qualityindicators": correspondingNewElement.qualityindicators
+            })
+          let newHistoryElement = {...correspondingNewElement, "history": historyWithNewElement}
+            // If some element (like the description) changes, we keep the new one.
+          newHistoryJSON.push(newHistoryElement)
+      }else{
+          newHistoryJSON.push(historyElement)
+      }
+    }else{
+      // console.log(historyElement)
+      newHistoryJSON.push(historyElement)
+    }
+  }
+
+  for(let currentForecast of currentForecastsWithoutAHistory){
+    let newHistoryElement = ({...currentForecast, "history": [{
+      "timestamp": currentForecast.timestamp,
+      "options": currentForecast.options,
+      "qualityindicators": currentForecast.qualityindicators
+    }]})
+    delete newHistoryElement.timestamp
+    delete newHistoryElement.options
+    delete newHistoryElement.qualityindicators
+    newHistoryJSON.push(newHistoryElement)
+  }
+
+  upsert(newHistoryJSON, "metaforecast_history")
+  // console.log(newHistoryJSON.slice(0,5))
+  // writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
+  // writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
+  //console.log(newHistoryJSON)
+  /*
+  let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
+  */
+  console.log(new Date().toISOString())
+}
+// addToHistory()

src/utils/history/old/createInitialHistoryWithMetaculus.js

@@ -1,4 +1,4 @@
-import { mongoRead, upsert } from "./mongo-wrapper.js"
+import { mongoRead, upsert } from "../mongo-wrapper.js"
 
 let createInitialHistory = async () => {
     let metaforecasts = await mongoRead("metaforecasts")

src/utils/history/updateHistory.js

@@ -0,0 +1,14 @@
+import { addToHistory } from "./addToHistory.js"
+import { createHistoryForMonth } from "./createHistoryForMonth.js"
+
+export async function updateHistory(){
+    let currentDate = new Date()
+    let dayOfMonth = currentDate.getDate()
+    if(dayOfMonth == 1){
+        console.log(`Creating history for the month ${currentDate.toISOString().slice(0,7)}`)
+        await createHistoryForMonth()
+    }else{
+        console.log(`Updating history for ${currentDate.toISOString()}`)
+        await addToHistory()
+    }
+}

src/utils/mongo-wrapper.js

@@ -2,6 +2,39 @@ import pkg from 'mongodb';
 const {MongoClient} = pkg;
 import {getCookie} from "./getCookies.js"
 
+function roughSizeOfObject( object ) {
+  var objectList = [];
+  var stack = [ object ];
+  var bytes = 0;
+
+  while ( stack.length ) {
+    var value = stack.pop();
+    if ( typeof value === 'boolean' ) {
+      bytes += 4;
+    }
+    else if ( typeof value === 'string' ) {
+      bytes += value.length * 2;
+    }
+    else if ( typeof value === 'number' ) {
+      bytes += 8;
+    }
+    else if
+    (
+      typeof value === 'object'
+      && objectList.indexOf( value ) === -1
+    )
+    {
+      objectList.push( value );
+
+      for( var i in value ) {
+          stack.push( value[ i ] );
+      }
+    }
+  }
+  let megaBytes = bytes / (1024)**2
+  let megaBytesRounded = Math.round(megaBytes*10)/10
+  return megaBytesRounded;
+}
 
 export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
     const url = process.env.MONGODB_URL || getCookie("mongodb");
@@ -32,8 +65,9 @@ export async function upsert (contents, documentName, collectionName="metaforeca
         const myDocument = await collection.findOne(filter);
     
         // Print to the console
-        console.log(myDocument.contentsArray.slice(0,1
-        ));
+        console.log(`Updating document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(contents)} MB`)
+        console.log("Sample: ")
+        console.log(JSON.stringify(myDocument.contentsArray.slice(0,1), null, 4));
       } catch (err) {
         console.log(err.stack);
       }
@@ -117,3 +151,38 @@ export async function mongoReadWithReadCredentials (documentName, collectionName
     // console.log(documentContents.slice(0,1));
     return documentContents
 }
+
+export async function mongoGetAllElements(databaseName = "metaforecastDatabase", collectionName="metaforecastCollection"){
+  const url = process.env.MONGODB_URL || getCookie("mongodb");
+  const client = new MongoClient(url, {
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
+  });
+
+  try {
+      await client.connect();
+      console.log(`Connected correctly to server`);
+      const db = client.db(databaseName);
+  
+      // Use the collection "data"
+      const collection = db.collection(collectionName);
+  
+      // Search options
+      const query = ({});
+      const options = ({});
+  
+      // Insert a single document, wait for promise so we can read it back
+      // const p = await collection.insertOne(metaforecastDocument);
+      const documents = await collection.find().toArray()
+      let documentNames = documents.map(document => ({name: document.name, roughSizeMBs: roughSizeOfObject(document)}));
+      console.log(documentNames)
+    }catch(error){
+      console.log(error)
+    }
+    finally {
+      await client.close();
+    }
+
+}
+//mongoGetAllElements()
+//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory")