Reinstated history functionality. This is for all platforms except for Metaculus (and Estimize). I'll see how it goes, and i particular, if I exceed the limit for July. If I do, the thing to do is to have a history for every platform for every month, rather than a history for all platforms for every month. It would make things slightly messier, though

This commit is contained in:
NunoSempere 2021-06-27 23:54:10 +02:00
parent c5bf99cee8
commit 216be068a9
8 changed files with 198 additions and 16 deletions

View File

@ -20,14 +20,13 @@ import {predictit} from "./platforms/predictit-fetch.js"
import {rootclaim} from "./platforms/rootclaim-fetch.js"
import {smarkets} from "./platforms/smarkets-fetch.js"
import {williamhill} from "./platforms/williamhill-fetch.js"
import {mergeEverything} from "./utils/mergeEverything.js"
import {addToHistory} from "./utils/addToHistory.js"
import {updateHistory} from "./utils/history/updateHistory.js"
import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
/* Support functions */
let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, addToHistory, rebuildNetlifySiteWithNewData, doEverything]
let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything]
let functionNames = functions.map(fun => fun.name)
let whattodoMessage = functionNames
@ -35,7 +34,7 @@ let whattodoMessage = functionNames
.map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
.join('\n') +
`\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` +
`\n[${functionNames.length-3}]: Add to history file` +
`\n[${functionNames.length-3}]: Update history` +
`\n[${functionNames.length-2}]: Rebuild netlify site with new data` +
// `\n[${functionNames.length-1}]: Add to history` +
`\n[${functionNames.length-1}]: All of the above` +

View File

@ -17,7 +17,7 @@ import {rootclaim} from "../platforms/rootclaim-fetch.js"
import {smarkets} from "../platforms/smarkets-fetch.js"
import {williamhill} from "../platforms/williamhill-fetch.js"
import {mergeEverything} from "./mergeEverything.js"
import {addToHistory} from "./addToHistory.js"
import {updateHistory} from "./history/updateHistory.js"
import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js"
/* Do everything */
@ -37,7 +37,7 @@ export async function tryCatchTryAgain (fun) {
}
}
export async function doEverything(){
let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, /*addToHistory,*/rebuildNetlifySiteWithNewData]
let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData]
// Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
console.log("")

View File

@ -1,13 +1,17 @@
import { writeFileSync } from "fs"
import { mongoReadWithReadCredentials, upsert } from "./mongo-wrapper.js"
import { mongoReadWithReadCredentials, upsert } from "../mongo-wrapper.js"
let mongoRead = mongoReadWithReadCredentials
let isEmptyArray = arr => arr.length == 0
export async function addToHistory(){
// throw new Error("Not today")
let currentJSON = await mongoRead("metaforecasts")
let currentDate = new Date()
let dateUpToMonth = currentDate.toISOString().slice(0,7).replace("-", "_")
let currentJSONwithMetaculus = await mongoRead("metaforecasts")
let currentJSON = currentJSONwithMetaculus.filter(element => element.platform != "Metaculus" && element.platform != "Estimize") // without Metaculus
// console.log(currentJSON.slice(0,20))
// console.log(currentJSON)
let historyJSON = await mongoRead("metaforecast_history")
let historyJSON = await mongoRead(`metaforecast_history_${dateUpToMonth}`,"metaforecastHistory")
// console.log(historyJSON)
let currentForecastsWithAHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
@ -55,7 +59,8 @@ export async function addToHistory(){
newHistoryJSON.push(newHistoryElement)
}
upsert(newHistoryJSON, "metaforecast_history")
await upsert(newHistoryJSON, `metaforecast_history_${dateUpToMonth}`, "metaforecastHistory")
// console.log(newHistoryJSON.slice(0,5))
// writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
// writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
@ -63,6 +68,6 @@ export async function addToHistory(){
/*
let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
*/
console.log(new Date().toISOString())
//console.log(new Date().toISOString())
}
// addToHistory()
// updateHistory()

View File

@ -0,0 +1,27 @@
import { mongoRead, upsert } from "../mongo-wrapper.js"
export async function createHistoryForMonth(){
let currentDate = new Date()
let dateUpToMonth = currentDate.toISOString().slice(0,7).replace("-", "_")
let metaforecasts = await mongoRead("metaforecasts")
let metaforecastsHistorySeed = metaforecasts.map(element => {
// let moreoriginsdata = element.author ? ({author: element.author}) : ({})
return ({
title: element.title,
url: element.url,
platform: element.platform,
moreoriginsdata: element.moreoriginsdata || {},
description: element.description,
history: [{
timestamp: element.timestamp,
options: element.options,
qualityindicators: element.qualityindicators
}],
extra: element.extra || {}
})
}).filter(element => element.platform != "Metaculus" && element.platform != "Estimize")
//console.log(metaforecastsHistorySeed)
await upsert(metaforecastsHistorySeed, `metaforecast_history_${dateUpToMonth}`, "metaforecastHistory")
}
////createInitialHistory()

View File

@ -0,0 +1,68 @@
import { writeFileSync } from "fs"
import { mongoReadWithReadCredentials, upsert } from "../mongo-wrapper.js"
let mongoRead = mongoReadWithReadCredentials
let isEmptyArray = arr => arr.length == 0
export async function addToHistory(){
// throw new Error("Not today")
let currentJSON = await mongoRead("metaforecasts")
// console.log(currentJSON)
let historyJSON = await mongoRead("metaforecast_history")
// console.log(historyJSON)
let currentForecastsWithAHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
// console.log(currentForecastsWithAHistory)
let currentForecastsWithoutAHistory = currentJSON.filter(element => isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
// console.log(currentForecastsWithoutAHistory)
// Add both types of forecast
let newHistoryJSON = []
for(let historyElement of historyJSON){
let correspondingNewElementArray = currentForecastsWithAHistory.filter(element => historyElement.title == element.title && historyElement.url == element.url )
// console.log(correspondingNewElement)
if(!isEmptyArray(correspondingNewElementArray)){
let correspondingNewElement = correspondingNewElementArray[0]
let timeStampOfNewElement = correspondingNewElement.timestamp
let doesHistoryAlreadyContainElement = historyElement.history.map(element => element.timestamp).includes(timeStampOfNewElement)
if(!doesHistoryAlreadyContainElement){
let historyWithNewElement = historyElement["history"].concat({
"timestamp": correspondingNewElement.timestamp,
"options": correspondingNewElement.options,
"qualityindicators": correspondingNewElement.qualityindicators
})
let newHistoryElement = {...correspondingNewElement, "history": historyWithNewElement}
// If some element (like the description) changes, we keep the new one.
newHistoryJSON.push(newHistoryElement)
}else{
newHistoryJSON.push(historyElement)
}
}else{
// console.log(historyElement)
newHistoryJSON.push(historyElement)
}
}
for(let currentForecast of currentForecastsWithoutAHistory){
let newHistoryElement = ({...currentForecast, "history": [{
"timestamp": currentForecast.timestamp,
"options": currentForecast.options,
"qualityindicators": currentForecast.qualityindicators
}]})
delete newHistoryElement.timestamp
delete newHistoryElement.options
delete newHistoryElement.qualityindicators
newHistoryJSON.push(newHistoryElement)
}
upsert(newHistoryJSON, "metaforecast_history")
// console.log(newHistoryJSON.slice(0,5))
// writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
// writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
//console.log(newHistoryJSON)
/*
let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
*/
console.log(new Date().toISOString())
}
// addToHistory()

View File

@ -1,4 +1,4 @@
import { mongoRead, upsert } from "./mongo-wrapper.js"
import { mongoRead, upsert } from "../mongo-wrapper.js"
let createInitialHistory = async () => {
let metaforecasts = await mongoRead("metaforecasts")

View File

@ -0,0 +1,14 @@
import { addToHistory } from "./addToHistory.js"
import { createHistoryForMonth } from "./createHistoryForMonth.js"
export async function updateHistory(){
let currentDate = new Date()
let dayOfMonth = currentDate.getDate()
if(dayOfMonth == 1){
console.log(`Creating history for the month ${currentDate.toISOString().slice(0,7)}`)
await createHistoryForMonth()
}else{
console.log(`Updating history for ${currentDate.toISOString()}`)
await addToHistory()
}
}

View File

@ -2,6 +2,39 @@ import pkg from 'mongodb';
const {MongoClient} = pkg;
import {getCookie} from "./getCookies.js"
function roughSizeOfObject( object ) {
var objectList = [];
var stack = [ object ];
var bytes = 0;
while ( stack.length ) {
var value = stack.pop();
if ( typeof value === 'boolean' ) {
bytes += 4;
}
else if ( typeof value === 'string' ) {
bytes += value.length * 2;
}
else if ( typeof value === 'number' ) {
bytes += 8;
}
else if
(
typeof value === 'object'
&& objectList.indexOf( value ) === -1
)
{
objectList.push( value );
for( var i in value ) {
stack.push( value[ i ] );
}
}
}
let megaBytes = bytes / (1024)**2
let megaBytesRounded = Math.round(megaBytes*10)/10
return megaBytesRounded;
}
export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
const url = process.env.MONGODB_URL || getCookie("mongodb");
@ -32,8 +65,9 @@ export async function upsert (contents, documentName, collectionName="metaforeca
const myDocument = await collection.findOne(filter);
// Print to the console
console.log(myDocument.contentsArray.slice(0,1
));
console.log(`Updating document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(contents)} MB`)
console.log("Sample: ")
console.log(JSON.stringify(myDocument.contentsArray.slice(0,1), null, 4));
} catch (err) {
console.log(err.stack);
}
@ -117,3 +151,38 @@ export async function mongoReadWithReadCredentials (documentName, collectionName
// console.log(documentContents.slice(0,1));
return documentContents
}
export async function mongoGetAllElements(databaseName = "metaforecastDatabase", collectionName="metaforecastCollection"){
const url = process.env.MONGODB_URL || getCookie("mongodb");
const client = new MongoClient(url, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
try {
await client.connect();
console.log(`Connected correctly to server`);
const db = client.db(databaseName);
// Use the collection "data"
const collection = db.collection(collectionName);
// Search options
const query = ({});
const options = ({});
// Insert a single document, wait for promise so we can read it back
// const p = await collection.insertOne(metaforecastDocument);
const documents = await collection.find().toArray()
let documentNames = documents.map(document => ({name: document.name, roughSizeMBs: roughSizeOfObject(document)}));
console.log(documentNames)
}catch(error){
console.log(error)
}
finally {
await client.close();
}
}
//mongoGetAllElements()
//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory")