quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Migration from mongo to pg

Browse Source
This commit is contained in:
NunoSempere 2022-03-03 16:55:32 -05:00
parent 6043f56a1b
commit 81eb03e029
3 changed files with 56 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/database/database-wrapper.js
View File

@ -81,7 +81,7 @@ export async function databaseRead({ group }) {
"metaforecastHistory", "metaforecastHistory",
"metaforecastDatabase" "metaforecastDatabase"
); );
// responsePg = await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
break; break;
default: default:
mongoDocName = `${group}-questions`; mongoDocName = `${group}-questions`;
@ -93,7 +93,7 @@ export async function databaseRead({ group }) {
responsePg = await pgRead({ schema: "latest", tableName: group }); responsePg = await pgRead({ schema: "latest", tableName: group });
} }
response = responseMongo; // responsePg response = responsePg; // responseMongo;
return response; return response;
} }
// databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase") // databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
@ -133,7 +133,7 @@ export async function databaseReadWithReadCredentials({ group }) {
"metaforecastHistory", "metaforecastHistory",
"metaforecastDatabase" "metaforecastDatabase"
); );
// responsePg = await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
break; break;
default: default:
mongoDocName = `${group}-questions`; mongoDocName = `${group}-questions`;
@ -149,7 +149,7 @@ export async function databaseReadWithReadCredentials({ group }) {
displayPossibleResponses(responseMongo, responsePg); displayPossibleResponses(responseMongo, responsePg);
} }
response = responseMongo; // responsePg response = responsePg; // responseMongo;
return response; return response;
} }
//= ; //= ;

2
src/platforms/metaculus-fetch.js
View File

@ -7,7 +7,6 @@ import { databaseUpsert } from "../database/database-wrapper.js";
/* Definitions */ /* Definitions */
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
let all_questions = [];
let now = new Date().toISOString(); let now = new Date().toISOString();
let DEBUG_MODE = "off"; let DEBUG_MODE = "off";
/* Support functions */ /* Support functions */
@ -63,6 +62,7 @@ export async function metaculus() {
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
// console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`) // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
// for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test
let all_questions = [];
let next = "https://www.metaculus.com/api2/questions/"; let next = "https://www.metaculus.com/api2/questions/";
let i = 1; let i = 1;
while (next) { while (next) {

74
src/utils/algolia.js
View File

@ -1,37 +1,65 @@
import algoliasearch from 'algoliasearch'; import algoliasearch from "algoliasearch";
import fs from "fs" import fs from "fs";
import { getSecret } from "./getSecrets.js" import { getSecret } from "./getSecrets.js";
import { databaseReadWithReadCredentials } from "../database/database-wrapper.js" import { databaseReadWithReadCredentials } from "../database/database-wrapper.js";
import { mergeEverythingInner } from '../flow/mergeEverything.js'; import { mergeEverythingInner } from "../flow/mergeEverything.js";
let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia")
const client = algoliasearch('96UD3NTQ7L', cookie);
const index = client.initIndex('metaforecast');
let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia");
const client = algoliasearch("96UD3NTQ7L", cookie);
const index = client.initIndex("metaforecast");
export async function rebuildAlgoliaDatabaseTheHardWay() { export async function rebuildAlgoliaDatabaseTheHardWay() {
console.log("Doing this the hard way") console.log("Doing this the hard way");
let records = await mergeEverythingInner() let records = await mergeEverythingInner();
records = records.map((record, index) => ({ ...record, has_numforecasts: record.numforecasts ? true : false, objectID: index })) records = records.map((record, index) => ({
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ ...record,
has_numforecasts: record.numforecasts ? true : false,
objectID: index,
}));
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
if (index.exists()) { if (index.exists()) {
console.log("Index exists") console.log("Index exists");
index.replaceAllObjects(records, { safe: true }).catch(error => console.log(error)) index
console.log(`Pushed ${records.length} records. Algolia will update asynchronously`) .replaceAllObjects(records, { safe: true })
.catch((error) => console.log(error));
console.log(
`Pushed ${records.length} records. Algolia will update asynchronously`
);
} }
} }
let getoptionsstringforsearch = (record) => {
let result = "";
if (!!record.options && record.options.length > 0) {
result = record.options
.map((option) => option.name || null)
.filter((x) => x != null)
.join(", ");
}
return result;
};
export async function rebuildAlgoliaDatabaseTheEasyWay() { export async function rebuildAlgoliaDatabaseTheEasyWay() {
let records = await databaseReadWithReadCredentials({ group: "combined" }) let records = await databaseReadWithReadCredentials({ group: "combined" });
records = records.map((record, index) => ({ ...record, has_numforecasts: record.numforecasts ? true : false, objectID: index }))
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/ records = records.map((record, index) => ({
...record,
has_numforecasts: record.numforecasts ? true : false,
objectID: index,
optionsstringforsearch: getoptionsstringforsearch(record),
}));
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
if (index.exists()) { if (index.exists()) {
console.log("Index exists") console.log("Index exists");
index.replaceAllObjects(records, { safe: true }).catch(error => console.log(error)) index
console.log(`Pushed ${records.length} records. Algolia will update asynchronously`) .replaceAllObjects(records, { safe: true })
.catch((error) => console.log(error));
console.log(
`Pushed ${records.length} records. Algolia will update asynchronously`
);
} }
} }
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay//rebuildAlgoliaDatabaseTheHardWay export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay