diff --git a/src/platforms/coupcast-fetch.js b/src/platforms/coupcast-fetch.js index 673077b..5025207 100644 --- a/src/platforms/coupcast-fetch.js +++ b/src/platforms/coupcast-fetch.js @@ -135,9 +135,10 @@ async function processArray(countryArray) { } }) - results.push(processedPrediction1) + // results.push(processedPrediction1) + // Not pushing monthly results.push(processedPrediction2) - } + } } // let string = JSON.stringify(results, null, 2) // fs.writeFileSync('./data/elicit-questions.json', string); diff --git a/src/platforms/hypermind-fetch.js b/src/platforms/hypermind-fetch.js index 0905d74..76016da 100644 --- a/src/platforms/hypermind-fetch.js +++ b/src/platforms/hypermind-fetch.js @@ -158,6 +158,7 @@ async function hypermind_inner(cookie) { } // console.log(resultsTotal) console.log(resultsTotalUnique) + console.log(resultsTotalUnique.length, "results") // let string = JSON.stringify(resultsTotalUnique, null, 2) // fs.writeFileSync('./data/hypermind-questions.json', string); await upsert(resultsTotalUnique, "hypermind-questions") diff --git a/src/utils/algolia.js b/src/utils/algolia.js index 75ebafc..73c7800 100644 --- a/src/utils/algolia.js +++ b/src/utils/algolia.js @@ -3,42 +3,34 @@ import {getCookie} from "./getCookies.js" import fs from "fs" import { mongoReadWithReadCredentials } from "./mongo-wrapper.js" +import { mergeEverythingInner } from './mergeEverything.js'; let cookie = process.env.ALGOLIA_MASTER_API_KEY || getCookie("algolia") const client = algoliasearch('96UD3NTQ7L', cookie); // delete this when committing const index = client.initIndex('metaforecast'); -export async function rebuildAlgoliaDatabase(){ - let records = await mongoReadWithReadCredentials("metaforecasts") - // let string = JSON.stringify(json, null, 2) - // fs.writeFileSync('metaforecasts.json', string); - records = records.map((record, index) => ({...record, has_numforecasts: record.numforecasts ? true : false, objectID: index}) ) - // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ - /* - index.clearObjects().wait().then(response => { - console.log(response) - }); - */ +export async function rebuildAlgoliaDatabase(){ + console.log("Doing this the hard way") + let records = await mergeEverythingInner() + records = records.map((record, index) => ({...record, has_numforecasts: record.numforecasts ? true : false, objectID: index}) ) + // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ if(index.exists()){ console.log("Index exists") index.replaceAllObjects(records, { safe:true }).catch(error => console.log(error)) console.log(`Pushed ${records.length} records. Algolia will update asynchronously`) } - - - /*await index.clearObjects() - console.log("Past data") - - setTimeout(function(){ - index.saveObjects(records, { autoGenerateObjectIDIfNotExist: true }).then(() => - console.log("algolia search: done") - ).catch(error => { - console.log("algolia search: error", error) - }) - alert('hello'); - }, 60*1000); // 1 minute seconds - */ } -//rebuildAlgoliaDatabase() + +export async function rebuildAlgoliaDatabaseTheEasyWay(){ + let records = await mongoReadWithReadCredentials("metaforecasts") + records = records.map((record, index) => ({...record, has_numforecasts: record.numforecasts ? true : false, objectID: index}) ) + // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ + + if(index.exists()){ + console.log("Index exists") + index.replaceAllObjects(records, { safe:true }).catch(error => console.log(error)) + console.log(`Pushed ${records.length} records. Algolia will update asynchronously`) + } +} \ No newline at end of file diff --git a/src/utils/mergeEverything.js b/src/utils/mergeEverything.js index e9e56e6..1e58761 100644 --- a/src/utils/mergeEverything.js +++ b/src/utils/mergeEverything.js @@ -1,16 +1,23 @@ import { mongoRead, upsert } from "./mongo-wrapper.js" /* Merge everything */ -let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", "estimize", "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "williamhill", "xrisk"] +let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "williamhill", "xrisk"] let suffix = "-questions" -export async function mergeEverything(){ +export async function mergeEverythingInner(){ let merged = [] for(let set of sets){ let json = await mongoRead(set+suffix) + console.log(`${set} has ${json.length} questions`) merged = merged.concat(json) } let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")})) - await upsert( mergedprocessed,"metaforecasts") + console.log(`In total, there are ${mergedprocessed.length} questions`) + return mergedprocessed +} + +export async function mergeEverything(){ + let merged = await mergeEverythingInner() + await upsert( merged,"metaforecasts") console.log("Done") } diff --git a/src/utils/mongo-wrapper.js b/src/utils/mongo-wrapper.js index 46db5cf..9b54c41 100644 --- a/src/utils/mongo-wrapper.js +++ b/src/utils/mongo-wrapper.js @@ -53,17 +53,19 @@ export async function upsert (contents, documentName, collectionName="metaforeca "timestamp": new Date().toISOString(), "contentsArray": contents }) + // Create a filter const filter = { "name": documentName }; // Insert a single document, wait for promise so we can read it back // const p = await collection.insertOne(metaforecastDocument); await collection.replaceOne(filter, document, { upsert: true }); - + console.log(`Pushed document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(document)} MB`) + // Find one document const myDocument = await collection.findOne(filter); // Print to the console - console.log(`Updating document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(contents)} MB`) + console.log(`Received document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(contents)} MB`) console.log("Sample: ") console.log(JSON.stringify(myDocument.contentsArray.slice(0,1), null, 4)); } catch (err) {