quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Migration from mongo to pg

Browse Source
This commit is contained in:
NunoSempere 2022-03-03 16:55:32 -05:00
parent 6043f56a1b
commit 81eb03e029
3 changed files with 56 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/database/database-wrapper.js
View File

@ -81,7 +81,7 @@ export async function databaseRead({ group }) {
"metaforecastHistory",
"metaforecastDatabase"
);
// responsePg = await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
break;
default:
mongoDocName = `${group}-questions`;
@ -93,7 +93,7 @@ export async function databaseRead({ group }) {
responsePg = await pgRead({ schema: "latest", tableName: group });
}
response = responseMongo; // responsePg
response = responsePg; // responseMongo;
return response;
}
// databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
@ -133,7 +133,7 @@ export async function databaseReadWithReadCredentials({ group }) {
"metaforecastHistory",
"metaforecastDatabase"
);
// responsePg = await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
break;
default:
mongoDocName = `${group}-questions`;
@ -149,7 +149,7 @@ export async function databaseReadWithReadCredentials({ group }) {
displayPossibleResponses(responseMongo, responsePg);
}
response = responseMongo; // responsePg
response = responsePg; // responseMongo;
return response;
}
//= ;

2
src/platforms/metaculus-fetch.js
View File

@ -7,7 +7,6 @@ import { databaseUpsert } from "../database/database-wrapper.js";
/* Definitions */
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
let all_questions = [];
let now = new Date().toISOString();
let DEBUG_MODE = "off";
/* Support functions */
@ -63,6 +62,7 @@ export async function metaculus() {
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
// console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
// for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test
let all_questions = [];
let next = "https://www.metaculus.com/api2/questions/";
let i = 1;
while (next) {

70
src/utils/algolia.js
View File

@ -1,37 +1,65 @@
import algoliasearch from 'algoliasearch';
import fs from "fs"
import { getSecret } from "./getSecrets.js"
import { databaseReadWithReadCredentials } from "../database/database-wrapper.js"
import { mergeEverythingInner } from '../flow/mergeEverything.js';
let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia")
const client = algoliasearch('96UD3NTQ7L', cookie);
const index = client.initIndex('metaforecast');
import algoliasearch from "algoliasearch";
import fs from "fs";
import { getSecret } from "./getSecrets.js";
import { databaseReadWithReadCredentials } from "../database/database-wrapper.js";
import { mergeEverythingInner } from "../flow/mergeEverything.js";
let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia");
const client = algoliasearch("96UD3NTQ7L", cookie);
const index = client.initIndex("metaforecast");
export async function rebuildAlgoliaDatabaseTheHardWay() {
console.log("Doing this the hard way")
let records = await mergeEverythingInner()
records = records.map((record, index) => ({ ...record, has_numforecasts: record.numforecasts ? true : false, objectID: index }))
console.log("Doing this the hard way");
let records = await mergeEverythingInner();
records = records.map((record, index) => ({
...record,
has_numforecasts: record.numforecasts ? true : false,
objectID: index,
}));
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
if (index.exists()) {
console.log("Index exists")
index.replaceAllObjects(records, { safe: true }).catch(error => console.log(error))
console.log(`Pushed ${records.length} records. Algolia will update asynchronously`)
console.log("Index exists");
index
.replaceAllObjects(records, { safe: true })
.catch((error) => console.log(error));
console.log(
`Pushed ${records.length} records. Algolia will update asynchronously`
);
}
}
let getoptionsstringforsearch = (record) => {
let result = "";
if (!!record.options && record.options.length > 0) {
result = record.options
.map((option) => option.name || null)
.filter((x) => x != null)
.join(", ");
}
return result;
};
export async function rebuildAlgoliaDatabaseTheEasyWay() {
let records = await databaseReadWithReadCredentials({ group: "combined" })
records = records.map((record, index) => ({ ...record, has_numforecasts: record.numforecasts ? true : false, objectID: index }))
let records = await databaseReadWithReadCredentials({ group: "combined" });
records = records.map((record, index) => ({
...record,
has_numforecasts: record.numforecasts ? true : false,
objectID: index,
optionsstringforsearch: getoptionsstringforsearch(record),
}));
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
if (index.exists()) {
console.log("Index exists")
index.replaceAllObjects(records, { safe: true }).catch(error => console.log(error))
console.log(`Pushed ${records.length} records. Algolia will update asynchronously`)
console.log("Index exists");
index
.replaceAllObjects(records, { safe: true })
.catch((error) => console.log(error));
console.log(
`Pushed ${records.length} records. Algolia will update asynchronously`
);
}
}
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay//rebuildAlgoliaDatabaseTheHardWay
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay