From 216be068a9e3d3ef9ce4e54aa611e308cc9500aa Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Sun, 27 Jun 2021 23:54:10 +0200 Subject: [PATCH] Reinstated history functionality. This is for all platforms except for Metaculus (and Estimize). I'll see how it goes, and i particular, if I exceed the limit for July. If I do, the thing to do is to have a history for every platform for every month, rather than a history for all platforms for every month. It would make things slightly messier, though --- src/index.js | 7 +- src/utils/doEverything.js | 4 +- src/utils/{ => history}/addToHistory.js | 19 +++-- src/utils/history/createHistoryForMonth.js | 27 +++++++ src/utils/history/old/addToHistory_old.js | 68 +++++++++++++++++ .../old/createInitialHistoryWithMetaculus.js} | 2 +- src/utils/history/updateHistory.js | 14 ++++ src/utils/mongo-wrapper.js | 73 ++++++++++++++++++- 8 files changed, 198 insertions(+), 16 deletions(-) rename src/utils/{ => history}/addToHistory.js (81%) create mode 100644 src/utils/history/createHistoryForMonth.js create mode 100644 src/utils/history/old/addToHistory_old.js rename src/utils/{createInitialHistory.js => history/old/createInitialHistoryWithMetaculus.js} (91%) create mode 100644 src/utils/history/updateHistory.js diff --git a/src/index.js b/src/index.js index ec4d406..327ceb4 100644 --- a/src/index.js +++ b/src/index.js @@ -20,14 +20,13 @@ import {predictit} from "./platforms/predictit-fetch.js" import {rootclaim} from "./platforms/rootclaim-fetch.js" import {smarkets} from "./platforms/smarkets-fetch.js" import {williamhill} from "./platforms/williamhill-fetch.js" - import {mergeEverything} from "./utils/mergeEverything.js" -import {addToHistory} from "./utils/addToHistory.js" +import {updateHistory} from "./utils/history/updateHistory.js" import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js" import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js" /* Support functions */ -let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything] +let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything] let functionNames = functions.map(fun => fun.name) let whattodoMessage = functionNames @@ -35,7 +34,7 @@ let whattodoMessage = functionNames .map((functionName,i) => `[${i}]: Download predictions from ${functionName}`) .join('\n') + `\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` + - `\n[${functionNames.length-3}]: Add to history file` + + `\n[${functionNames.length-3}]: Update history` + `\n[${functionNames.length-2}]: Rebuild netlify site with new data` + // `\n[${functionNames.length-1}]: Add to history` + `\n[${functionNames.length-1}]: All of the above` + diff --git a/src/utils/doEverything.js b/src/utils/doEverything.js index 4a280d9..440dcfe 100644 --- a/src/utils/doEverything.js +++ b/src/utils/doEverything.js @@ -17,7 +17,7 @@ import {rootclaim} from "../platforms/rootclaim-fetch.js" import {smarkets} from "../platforms/smarkets-fetch.js" import {williamhill} from "../platforms/williamhill-fetch.js" import {mergeEverything} from "./mergeEverything.js" -import {addToHistory} from "./addToHistory.js" +import {updateHistory} from "./history/updateHistory.js" import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js" /* Do everything */ @@ -37,7 +37,7 @@ export async function tryCatchTryAgain (fun) { } } export async function doEverything(){ - let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, /*addToHistory,*/rebuildNetlifySiteWithNewData] + let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData] // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku. console.log("") diff --git a/src/utils/addToHistory.js b/src/utils/history/addToHistory.js similarity index 81% rename from src/utils/addToHistory.js rename to src/utils/history/addToHistory.js index d77e08f..e36bf55 100644 --- a/src/utils/addToHistory.js +++ b/src/utils/history/addToHistory.js @@ -1,13 +1,17 @@ import { writeFileSync } from "fs" -import { mongoReadWithReadCredentials, upsert } from "./mongo-wrapper.js" +import { mongoReadWithReadCredentials, upsert } from "../mongo-wrapper.js" let mongoRead = mongoReadWithReadCredentials let isEmptyArray = arr => arr.length == 0 export async function addToHistory(){ - // throw new Error("Not today") - let currentJSON = await mongoRead("metaforecasts") + let currentDate = new Date() + let dateUpToMonth = currentDate.toISOString().slice(0,7).replace("-", "_") + + let currentJSONwithMetaculus = await mongoRead("metaforecasts") + let currentJSON = currentJSONwithMetaculus.filter(element => element.platform != "Metaculus" && element.platform != "Estimize") // without Metaculus + // console.log(currentJSON.slice(0,20)) // console.log(currentJSON) - let historyJSON = await mongoRead("metaforecast_history") + let historyJSON = await mongoRead(`metaforecast_history_${dateUpToMonth}`,"metaforecastHistory") // console.log(historyJSON) let currentForecastsWithAHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url ))) @@ -55,7 +59,8 @@ export async function addToHistory(){ newHistoryJSON.push(newHistoryElement) } - upsert(newHistoryJSON, "metaforecast_history") + await upsert(newHistoryJSON, `metaforecast_history_${dateUpToMonth}`, "metaforecastHistory") + // console.log(newHistoryJSON.slice(0,5)) // writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2)) // writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json") @@ -63,6 +68,6 @@ export async function addToHistory(){ /* let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url ))) */ - console.log(new Date().toISOString()) + //console.log(new Date().toISOString()) } -// addToHistory() +// updateHistory() diff --git a/src/utils/history/createHistoryForMonth.js b/src/utils/history/createHistoryForMonth.js new file mode 100644 index 0000000..fa5fdc5 --- /dev/null +++ b/src/utils/history/createHistoryForMonth.js @@ -0,0 +1,27 @@ +import { mongoRead, upsert } from "../mongo-wrapper.js" + +export async function createHistoryForMonth(){ + let currentDate = new Date() + let dateUpToMonth = currentDate.toISOString().slice(0,7).replace("-", "_") + let metaforecasts = await mongoRead("metaforecasts") + let metaforecastsHistorySeed = metaforecasts.map(element => { + // let moreoriginsdata = element.author ? ({author: element.author}) : ({}) + return ({ + title: element.title, + url: element.url, + platform: element.platform, + moreoriginsdata: element.moreoriginsdata || {}, + description: element.description, + history: [{ + timestamp: element.timestamp, + options: element.options, + qualityindicators: element.qualityindicators + }], + extra: element.extra || {} + }) + }).filter(element => element.platform != "Metaculus" && element.platform != "Estimize") + //console.log(metaforecastsHistorySeed) + await upsert(metaforecastsHistorySeed, `metaforecast_history_${dateUpToMonth}`, "metaforecastHistory") + +} +////createInitialHistory() \ No newline at end of file diff --git a/src/utils/history/old/addToHistory_old.js b/src/utils/history/old/addToHistory_old.js new file mode 100644 index 0000000..7789b41 --- /dev/null +++ b/src/utils/history/old/addToHistory_old.js @@ -0,0 +1,68 @@ +import { writeFileSync } from "fs" +import { mongoReadWithReadCredentials, upsert } from "../mongo-wrapper.js" +let mongoRead = mongoReadWithReadCredentials +let isEmptyArray = arr => arr.length == 0 + +export async function addToHistory(){ + // throw new Error("Not today") + let currentJSON = await mongoRead("metaforecasts") + // console.log(currentJSON) + let historyJSON = await mongoRead("metaforecast_history") + // console.log(historyJSON) + + let currentForecastsWithAHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url ))) + // console.log(currentForecastsWithAHistory) + + let currentForecastsWithoutAHistory = currentJSON.filter(element => isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url ))) + // console.log(currentForecastsWithoutAHistory) + + // Add both types of forecast + let newHistoryJSON = [] + for(let historyElement of historyJSON){ + let correspondingNewElementArray = currentForecastsWithAHistory.filter(element => historyElement.title == element.title && historyElement.url == element.url ) + // console.log(correspondingNewElement) + if(!isEmptyArray(correspondingNewElementArray)){ + let correspondingNewElement = correspondingNewElementArray[0] + let timeStampOfNewElement = correspondingNewElement.timestamp + let doesHistoryAlreadyContainElement = historyElement.history.map(element => element.timestamp).includes(timeStampOfNewElement) + if(!doesHistoryAlreadyContainElement){ + let historyWithNewElement = historyElement["history"].concat({ + "timestamp": correspondingNewElement.timestamp, + "options": correspondingNewElement.options, + "qualityindicators": correspondingNewElement.qualityindicators + }) + let newHistoryElement = {...correspondingNewElement, "history": historyWithNewElement} + // If some element (like the description) changes, we keep the new one. + newHistoryJSON.push(newHistoryElement) + }else{ + newHistoryJSON.push(historyElement) + } + }else{ + // console.log(historyElement) + newHistoryJSON.push(historyElement) + } + } + + for(let currentForecast of currentForecastsWithoutAHistory){ + let newHistoryElement = ({...currentForecast, "history": [{ + "timestamp": currentForecast.timestamp, + "options": currentForecast.options, + "qualityindicators": currentForecast.qualityindicators + }]}) + delete newHistoryElement.timestamp + delete newHistoryElement.options + delete newHistoryElement.qualityindicators + newHistoryJSON.push(newHistoryElement) + } + + upsert(newHistoryJSON, "metaforecast_history") + // console.log(newHistoryJSON.slice(0,5)) + // writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2)) + // writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json") + //console.log(newHistoryJSON) + /* + let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url ))) + */ + console.log(new Date().toISOString()) +} +// addToHistory() diff --git a/src/utils/createInitialHistory.js b/src/utils/history/old/createInitialHistoryWithMetaculus.js similarity index 91% rename from src/utils/createInitialHistory.js rename to src/utils/history/old/createInitialHistoryWithMetaculus.js index 0362e28..e1cff01 100644 --- a/src/utils/createInitialHistory.js +++ b/src/utils/history/old/createInitialHistoryWithMetaculus.js @@ -1,4 +1,4 @@ -import { mongoRead, upsert } from "./mongo-wrapper.js" +import { mongoRead, upsert } from "../mongo-wrapper.js" let createInitialHistory = async () => { let metaforecasts = await mongoRead("metaforecasts") diff --git a/src/utils/history/updateHistory.js b/src/utils/history/updateHistory.js new file mode 100644 index 0000000..f8e1bae --- /dev/null +++ b/src/utils/history/updateHistory.js @@ -0,0 +1,14 @@ +import { addToHistory } from "./addToHistory.js" +import { createHistoryForMonth } from "./createHistoryForMonth.js" + +export async function updateHistory(){ + let currentDate = new Date() + let dayOfMonth = currentDate.getDate() + if(dayOfMonth == 1){ + console.log(`Creating history for the month ${currentDate.toISOString().slice(0,7)}`) + await createHistoryForMonth() + }else{ + console.log(`Updating history for ${currentDate.toISOString()}`) + await addToHistory() + } +} \ No newline at end of file diff --git a/src/utils/mongo-wrapper.js b/src/utils/mongo-wrapper.js index 7744e63..e319e23 100644 --- a/src/utils/mongo-wrapper.js +++ b/src/utils/mongo-wrapper.js @@ -2,6 +2,39 @@ import pkg from 'mongodb'; const {MongoClient} = pkg; import {getCookie} from "./getCookies.js" +function roughSizeOfObject( object ) { + var objectList = []; + var stack = [ object ]; + var bytes = 0; + + while ( stack.length ) { + var value = stack.pop(); + if ( typeof value === 'boolean' ) { + bytes += 4; + } + else if ( typeof value === 'string' ) { + bytes += value.length * 2; + } + else if ( typeof value === 'number' ) { + bytes += 8; + } + else if + ( + typeof value === 'object' + && objectList.indexOf( value ) === -1 + ) + { + objectList.push( value ); + + for( var i in value ) { + stack.push( value[ i ] ); + } + } + } + let megaBytes = bytes / (1024)**2 + let megaBytesRounded = Math.round(megaBytes*10)/10 + return megaBytesRounded; +} export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){ const url = process.env.MONGODB_URL || getCookie("mongodb"); @@ -32,8 +65,9 @@ export async function upsert (contents, documentName, collectionName="metaforeca const myDocument = await collection.findOne(filter); // Print to the console - console.log(myDocument.contentsArray.slice(0,1 - )); + console.log(`Updating document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(contents)} MB`) + console.log("Sample: ") + console.log(JSON.stringify(myDocument.contentsArray.slice(0,1), null, 4)); } catch (err) { console.log(err.stack); } @@ -117,3 +151,38 @@ export async function mongoReadWithReadCredentials (documentName, collectionName // console.log(documentContents.slice(0,1)); return documentContents } + +export async function mongoGetAllElements(databaseName = "metaforecastDatabase", collectionName="metaforecastCollection"){ + const url = process.env.MONGODB_URL || getCookie("mongodb"); + const client = new MongoClient(url, { + useNewUrlParser: true, + useUnifiedTopology: true, + }); + + try { + await client.connect(); + console.log(`Connected correctly to server`); + const db = client.db(databaseName); + + // Use the collection "data" + const collection = db.collection(collectionName); + + // Search options + const query = ({}); + const options = ({}); + + // Insert a single document, wait for promise so we can read it back + // const p = await collection.insertOne(metaforecastDocument); + const documents = await collection.find().toArray() + let documentNames = documents.map(document => ({name: document.name, roughSizeMBs: roughSizeOfObject(document)})); + console.log(documentNames) + }catch(error){ + console.log(error) + } + finally { + await client.close(); + } + +} +//mongoGetAllElements() +//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory") \ No newline at end of file