From 81eb03e029ae4fefbfa755d8c4dd9953472f386a Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Thu, 3 Mar 2022 16:55:32 -0500 Subject: [PATCH] feat: Migration from mongo to pg --- src/database/database-wrapper.js | 8 ++-- src/platforms/metaculus-fetch.js | 2 +- src/utils/algolia.js | 74 ++++++++++++++++++++++---------- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/database/database-wrapper.js b/src/database/database-wrapper.js index 25ef095..087faae 100644 --- a/src/database/database-wrapper.js +++ b/src/database/database-wrapper.js @@ -81,7 +81,7 @@ export async function databaseRead({ group }) { "metaforecastHistory", "metaforecastDatabase" ); - // responsePg = await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. + responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. break; default: mongoDocName = `${group}-questions`; @@ -93,7 +93,7 @@ export async function databaseRead({ group }) { responsePg = await pgRead({ schema: "latest", tableName: group }); } - response = responseMongo; // responsePg + response = responsePg; // responseMongo; return response; } // databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase") @@ -133,7 +133,7 @@ export async function databaseReadWithReadCredentials({ group }) { "metaforecastHistory", "metaforecastDatabase" ); - // responsePg = await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. + responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. break; default: mongoDocName = `${group}-questions`; @@ -149,7 +149,7 @@ export async function databaseReadWithReadCredentials({ group }) { displayPossibleResponses(responseMongo, responsePg); } - response = responseMongo; // responsePg + response = responsePg; // responseMongo; return response; } //= ; diff --git a/src/platforms/metaculus-fetch.js b/src/platforms/metaculus-fetch.js index 7443e70..bd6a29d 100644 --- a/src/platforms/metaculus-fetch.js +++ b/src/platforms/metaculus-fetch.js @@ -7,7 +7,6 @@ import { databaseUpsert } from "../database/database-wrapper.js"; /* Definitions */ let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; -let all_questions = []; let now = new Date().toISOString(); let DEBUG_MODE = "off"; /* Support functions */ @@ -63,6 +62,7 @@ export async function metaculus() { // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test + let all_questions = []; let next = "https://www.metaculus.com/api2/questions/"; let i = 1; while (next) { diff --git a/src/utils/algolia.js b/src/utils/algolia.js index 7b162ac..8e3fe35 100644 --- a/src/utils/algolia.js +++ b/src/utils/algolia.js @@ -1,37 +1,65 @@ -import algoliasearch from 'algoliasearch'; -import fs from "fs" -import { getSecret } from "./getSecrets.js" -import { databaseReadWithReadCredentials } from "../database/database-wrapper.js" -import { mergeEverythingInner } from '../flow/mergeEverything.js'; - -let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia") -const client = algoliasearch('96UD3NTQ7L', cookie); -const index = client.initIndex('metaforecast'); +import algoliasearch from "algoliasearch"; +import fs from "fs"; +import { getSecret } from "./getSecrets.js"; +import { databaseReadWithReadCredentials } from "../database/database-wrapper.js"; +import { mergeEverythingInner } from "../flow/mergeEverything.js"; +let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia"); +const client = algoliasearch("96UD3NTQ7L", cookie); +const index = client.initIndex("metaforecast"); export async function rebuildAlgoliaDatabaseTheHardWay() { - console.log("Doing this the hard way") - let records = await mergeEverythingInner() - records = records.map((record, index) => ({ ...record, has_numforecasts: record.numforecasts ? true : false, objectID: index })) - // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ + console.log("Doing this the hard way"); + let records = await mergeEverythingInner(); + records = records.map((record, index) => ({ + ...record, + has_numforecasts: record.numforecasts ? true : false, + objectID: index, + })); + // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ if (index.exists()) { - console.log("Index exists") - index.replaceAllObjects(records, { safe: true }).catch(error => console.log(error)) - console.log(`Pushed ${records.length} records. Algolia will update asynchronously`) + console.log("Index exists"); + index + .replaceAllObjects(records, { safe: true }) + .catch((error) => console.log(error)); + console.log( + `Pushed ${records.length} records. Algolia will update asynchronously` + ); } } +let getoptionsstringforsearch = (record) => { + let result = ""; + if (!!record.options && record.options.length > 0) { + result = record.options + .map((option) => option.name || null) + .filter((x) => x != null) + .join(", "); + } + return result; +}; + export async function rebuildAlgoliaDatabaseTheEasyWay() { - let records = await databaseReadWithReadCredentials({ group: "combined" }) - records = records.map((record, index) => ({ ...record, has_numforecasts: record.numforecasts ? true : false, objectID: index })) - // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ + let records = await databaseReadWithReadCredentials({ group: "combined" }); + + records = records.map((record, index) => ({ + ...record, + has_numforecasts: record.numforecasts ? true : false, + objectID: index, + optionsstringforsearch: getoptionsstringforsearch(record), + })); + // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ if (index.exists()) { - console.log("Index exists") - index.replaceAllObjects(records, { safe: true }).catch(error => console.log(error)) - console.log(`Pushed ${records.length} records. Algolia will update asynchronously`) + console.log("Index exists"); + index + .replaceAllObjects(records, { safe: true }) + .catch((error) => console.log(error)); + console.log( + `Pushed ${records.length} records. Algolia will update asynchronously` + ); } } -export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay//rebuildAlgoliaDatabaseTheHardWay \ No newline at end of file +export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay