feat: data in public schema; merge history tables

src/backend/database/database-wrapper.ts

@@ -1,30 +0,0 @@
-import { pgRead, pgReadWithReadCredentials, pgUpsert } from "./pg-wrapper";
-
-export async function databaseUpsert({ contents, group }) {
-  // No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
-  // (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){
-  const tableName = group === "history" ? "h2022" : group;
-  await pgUpsert({ contents, tableName });
-}
-
-const readWithReader = async (
-  group: string,
-  reader: (opts: { tableName: string }) => Promise<any>
-) => {
-  const tableName = group === "history" ? "h2022" : group;
-  const response = await reader({ tableName });
-
-  console.log("Postgres: ");
-  console.log(response.slice(0, 2));
-  console.log("");
-
-  return response;
-};
-
-export async function databaseRead({ group }) {
-  return await readWithReader(group, pgRead);
-}
-
-export async function databaseReadWithReadCredentials({ group }) {
-  return await readWithReader(group, pgReadWithReadCredentials);
-}

src/backend/database/pg-wrapper.ts

@@ -5,30 +5,11 @@ import { hash } from "../utils/hash";
 import { measureTime } from "../utils/measureTime";
 import { roughSizeOfObject } from "../utils/roughSize";
 
-// Definitions
+const platformTableNames = platforms.map((platform) => platform.name);
-const year = Number(new Date().toISOString().slice(0, 4));
+
-const allowed_years = [year, year + 1].map((year) => `h${year}`); // tables can't begin with number
+const forecastTableNames = [...platformTableNames, "combined", "history"];
-const allowed_months = [...Array(12).keys()]
+
-  .map((x) => x + 1)
+const allTableNames = [...forecastTableNames, "dashboards", "frontpage"];
-  .map((x) => (String(x).length == 1 ? `0${x}` : x));
-const allowed_year_month_histories = [].concat(
-  ...allowed_years.map((year) =>
-    allowed_months.map((month) => `${year}_${month}`)
-  )
-); // h2022_01
-const tableNamesWhitelistLatest = [
-  "combined",
-  ...platforms.map((platform) => platform.name),
-];
-const tableNamesWhiteListHistory = [
-  ...allowed_years,
-  ...allowed_year_month_histories,
-];
-const tableWhiteList = [
-  ...tableNamesWhitelistLatest,
-  ...tableNamesWhiteListHistory,
-  "dashboards",
-];
 
 /* Postgres database connection code */
 const databaseURL = process.env.DIGITALOCEAN_POSTGRES;
@@ -107,20 +88,6 @@ async function pgInitializeScaffolding() {
   }
   let YOLO = false;
   if (YOLO) {
-    console.log("Create schemas");
-    await runPgCommand({
-      command: `CREATE SCHEMA IF NOT EXISTS public`,
-      pool: readWritePool,
-    });
-    console.log("");
-
-    console.log("Set search path");
-    await runPgCommand({
-      command: `SET search_path TO public;`,
-      pool: readWritePool,
-    });
-    console.log("");
-
     console.log("Set public user permissions");
     await setPermissionsForPublicUser();
     console.log("");
@@ -148,21 +115,19 @@ async function pgInitializeLatest() {
   let YOLO = false;
   if (YOLO) {
     console.log("Create tables & their indexes");
-    let schema = "latest";
+    for (const table of platformTableNames) {
-    for (let table of tableNamesWhitelistLatest) {
       await runPgCommand({
-        command: dropTable(schema),
+        command: dropTable(table),
         pool: readWritePool,
       });
       await runPgCommand({
-        command: buildMetaforecastTable(schema),
+        command: buildMetaforecastTable(table),
         pool: readWritePool,
       });
       await runPgCommand({
-        command: createUniqueIndex(schema),
+        command: createUniqueIndex(table),
         pool: readWritePool,
       });
-      //}
     }
     console.log("");
   } else {
@@ -224,25 +189,19 @@ let buildHistoryTable = (table: string) => `CREATE TABLE ${table} (
 export async function pgInitializeHistories() {
   let YOLO = false;
   if (YOLO) {
-    console.log("Drop all previous history tables (Danger!)");
+    console.log("Create history table & index");
-    console.log("TODO - drop history tables"); // hope we won't need it until we get proper migrations
-    console.log("");
-
-    console.log("Create tables & their indexes");
-    for (let table of tableNamesWhiteListHistory) {
     await runPgCommand({
-        command: dropTable(table),
+      command: dropTable("history"),
       pool: readWritePool,
     });
     await runPgCommand({
-        command: buildHistoryTable(table),
+      command: buildHistoryTable("history"),
       pool: readWritePool,
     });
     await runPgCommand({
-        command: createIndex(table), // Not unique!!
+      command: createIndex("history"), // Not unique!!
       pool: readWritePool,
     });
-    }
     console.log("");
   } else {
     console.log(
@@ -289,16 +248,15 @@ async function pgReadWithPool({
   tableName: string;
   pool: Pool;
 }) {
-  if (tableWhiteList.includes(tableName)) {
+  if (!allTableNames.includes(tableName)) {
-    let command = `SELECT * from ${tableName}`;
-    let response = await runPgCommand({ command, pool });
-    let results = response.results;
-    return results;
-  } else {
     throw Error(
       `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
     );
   }
+  let command = `SELECT * from ${tableName}`;
+  let response = await runPgCommand({ command, pool });
+  let results = response.results;
+  return results;
 }
 
 export async function pgRead({ tableName }: { tableName: string }) {
@@ -344,7 +302,7 @@ export async function pgBulkInsert({
   tableName: string;
   client: PoolClient;
 }) {
-  if (!tableWhiteList.includes(tableName)) {
+  if (!forecastTableNames.includes(tableName)) {
     throw Error(
       `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
     );
@@ -404,9 +362,8 @@ export async function pgBulkInsert({
   }
 }
 
-export async function pgInsertIntoDashboard({ datum, tableName }) {
+export async function pgInsertIntoDashboard({ datum }) {
-  if (tableWhiteList.includes(tableName)) {
+  let text = `INSERT INTO dashboards VALUES($1, $2, $3, $4, $5, $6, $7)`;
-    let text = `INSERT INTO ${tableName} VALUES($1, $2, $3, $4, $5, $6, $7)`;
   let timestamp = datum.timestamp || new Date().toISOString();
   timestamp = timestamp.slice(0, 19).replace("T", " ");
   let values = [
@@ -429,11 +386,6 @@ export async function pgInsertIntoDashboard({ datum, tableName }) {
   }
   // console.log(result)
   return result;
-  } else {
-    throw Error(
-      `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
-    );
-  }
 }
 /* For reference
 	  id text,
@@ -461,10 +413,16 @@ pgInsertIntoDashboard({
   tableName: "dashboards",
 });
 */
-export async function pgUpsert({ contents, tableName }) {
+export async function pgUpsert({
-  if (!tableWhiteList.includes(tableName)) {
+  contents,
-    console.log("tableWhiteList:");
+  tableName,
-    console.log(tableWhiteList);
+  replace,
+}: {
+  contents: Forecast[];
+  tableName: string;
+  replace: boolean;
+}) {
+  if (!forecastTableNames.includes(tableName)) {
     throw Error(
       `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
     );
@@ -474,19 +432,12 @@ export async function pgUpsert({ contents, tableName }) {
     const client = await readWritePool.connect();
     try {
       await client.query("BEGIN");
-      if (schema === "latest") {
+      if (replace) {
-        client.query(`DELETE FROM latest.${tableName}`);
+        client.query(`DELETE FROM ${tableName}`);
       }
       console.log(
         `Upserting ${contents.length} rows into postgres table ${tableName}.`
       );
-      console.log(
-        `Expected to take ${Number((contents.length * 831.183) / 4422).toFixed(
-          2
-        )} seconds or ${Number((contents.length * 13.85305) / 4422).toFixed(
-          2
-        )} minutes`
-      );
 
       await pgBulkInsert({ data: contents, tableName, client });
       console.log(

src/backend/flow/history/updateHistory.ts

@@ -4,6 +4,7 @@ export async function updateHistory() {
   let latest = await pgReadWithReadCredentials({ tableName: "combined" });
   await pgUpsert({
     contents: latest,
-    tableName: "h2022",
+    tableName: "history",
+    replace: false,
   });
 }

src/backend/flow/mergeEverything.ts

@@ -23,6 +23,6 @@ export async function mergeEverythingInner() {
 
 export async function mergeEverything() {
   let merged = await mergeEverythingInner();
-  await pgUpsert({ contents: merged, tableName: "combined" });
+  await pgUpsert({ contents: merged, tableName: "combined", replace: true });
   console.log("Done");
 }

src/backend/manual/manualDownload.ts

@@ -2,10 +2,10 @@ import "dotenv/config";
 
 import fs from "fs";
 
-import { databaseReadWithReadCredentials } from "../database/database-wrapper";
+import { pgReadWithReadCredentials } from "../database/pg-wrapper";
 
 let main = async () => {
-  let json = await databaseReadWithReadCredentials({ group: "combined" });
+  let json = await pgReadWithReadCredentials({ tableName: "combined" });
   let string = JSON.stringify(json, null, 2);
   let filename = "metaforecasts.json";
   fs.writeFileSync(filename, string);

src/backend/manual/noSchemaMigrate.ts

@@ -0,0 +1,38 @@
+import "dotenv/config";
+
+import { readWritePool } from "../database/pg-wrapper";
+import { platforms } from "../platforms";
+
+const migrate = async () => {
+  const client = await readWritePool.connect();
+
+  const execQuery = async (q: string) => {
+    console.log(q);
+    await client.query(q);
+  };
+
+  try {
+    await client.query("BEGIN");
+    const copyTable = async (from: string, to: string) => {
+      await execQuery(`DROP TABLE IF EXISTS ${to}`);
+      await execQuery(`CREATE TABLE ${to} (LIKE ${from} INCLUDING ALL)`);
+      await execQuery(`INSERT INTO ${to} SELECT * FROM ${from}`);
+    };
+
+    for (const platform of platforms) {
+      await copyTable(`latest.${platform.name}`, platform.name);
+    }
+    await copyTable("latest.dashboards", "dashboards");
+    await copyTable("latest.combined", "combined");
+    await copyTable("latest.frontpage", "frontpage");
+    await copyTable("history.h2022", "history");
+    await client.query("COMMIT");
+  } catch (e) {
+    await client.query("ROLLBACK");
+    throw e;
+  } finally {
+    client.release();
+  }
+};
+
+migrate();

src/backend/platforms/example-fetch.ts

@@ -1,8 +1,8 @@
 /* Imports */
 import axios from "axios";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { calculateStars } from "../utils/stars";
+import { Platform } from "./";
 
 /* Definitions */
 let endpoint = "https://example.com/";
@@ -59,12 +59,11 @@ async function processPredictions(predictions) {
 
 /* Body */
 
-export async function example() {
+export const example: Platform = {
+  name: "example",
+  async fetcher() {
     let data = await fetchData();
     let results = await processPredictions(data); // somehow needed
-  // console.log(results)
+    return results;
-  // let string = JSON.stringify(results, null, 2)
+  },
-  await databaseUpsert({ contents: results, group: "example" });
+};
-  console.log("Done");
-}
-//example()

src/backend/platforms/givewellopenphil.ts

@@ -2,7 +2,6 @@
 import axios from "axios";
 import fs from "fs";
 
-import { databaseUpsert } from "../database/database-wrapper";
 import { calculateStars } from "../utils/stars";
 import { Platform } from "./";
 
@@ -59,10 +58,10 @@ async function main1() {
     // console.log(result)
     results.push(result);
   }
-  await databaseUpsert({
+  // await databaseUpsert({
-    contents: results,
+  //   contents: results,
-    group: "givewell-questions-unprocessed",
+  //   group: "givewell-questions-unprocessed",
-  });
+  // });
 }
 
 export const givewellopenphil: Platform = {

src/backend/platforms/index.ts

@@ -1,4 +1,4 @@
-import { databaseUpsert } from "../database/database-wrapper";
+import { pgUpsert } from "../database/pg-wrapper";
 import { betfair } from "./betfair";
 import { fantasyscotus } from "./fantasyscotus";
 import { foretold } from "./foretold";
@@ -112,7 +112,11 @@ export const processPlatform = async (platform: Platform) => {
   }
   let results = await platform.fetcher();
   if (results && results.length) {
-    await databaseUpsert({ contents: results, group: platform.name });
+    await pgUpsert({
+      contents: results,
+      tableName: platform.name,
+      replace: true,
+    });
     console.log("Done");
   } else {
     console.log(`Platform ${platform.name} didn't return any results`);

src/backend/utils/algolia.ts

@@ -1,6 +1,6 @@
 import algoliasearch from "algoliasearch";
 
-import { databaseReadWithReadCredentials } from "../database/database-wrapper";
+import { pgReadWithReadCredentials } from "../database/pg-wrapper";
 import { mergeEverythingInner } from "../flow/mergeEverything";
 
 let cookie = process.env.ALGOLIA_MASTER_API_KEY;
@@ -41,8 +41,8 @@ let getoptionsstringforsearch = (record: any) => {
 };
 
 export async function rebuildAlgoliaDatabaseTheEasyWay() {
-  let records: any[] = await databaseReadWithReadCredentials({
+  let records: any[] = await pgReadWithReadCredentials({
-    group: "combined",
+    tableName: "combined",
   });
 
   records = records.map((record, index: number) => ({

src/backend/utils/evaluations/pullForecastsToCSVForRating.ts

@@ -1,7 +1,7 @@
 /* Imports */
 import fs from "fs";
 
-import { databaseReadWithReadCredentials } from "../../database/database-wrapper";
+import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
 
 /* Definitions */
 
@@ -24,7 +24,7 @@ let main = async () => {
     "PredictIt",
     "Rootclaim",
   ];
-  let json = await databaseReadWithReadCredentials({ group: "combined" });
+  let json = await pgReadWithReadCredentials({ tableName: "combined" });
   console.log(json.length);
   //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
   //console.log(uniquePlatforms)

src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts

@@ -1,7 +1,7 @@
 /* Imports */
 import fs from "fs";
 
-import { databaseReadWithReadCredentials } from "../../database/database-wrapper";
+import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
 
 /* Definitions */
 
@@ -26,7 +26,7 @@ let shuffleArray = (array) => {
 
 let main = async () => {
   let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim']
-  let json = await databaseReadWithReadCredentials({ group: "combined" });
+  let json = await pgReadWithReadCredentials({ tableName: "combined" });
   console.log(json.length);
   //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
   //console.log(uniquePlatforms)

src/backend/utils/misc/process-forecasts-into-elicit.ts

@@ -1,7 +1,7 @@
 /* Imports */
 import fs from "fs";
 
-import { databaseReadWithReadCredentials } from "../../database/database-wrapper";
+import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
 
 /* Definitions */
 let locationData = "./data/";
@@ -9,7 +9,7 @@ let locationData = "./data/";
 /* Body */
 // let rawdata =  fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src
 async function main() {
-  let data = await databaseReadWithReadCredentials({ group: "combined" }); //JSON.parse(rawdata)
+  let data = await pgReadWithReadCredentials({ tableName: "combined" }); //JSON.parse(rawdata)
   let processDescription = (description) => {
     if (description == null || description == undefined || description == "") {
       return "";

src/pages/api/create-dashboard-from-ids.ts

@@ -27,7 +27,6 @@ export default async function handler(
         creator: body.creator || "",
         extra: [],
       },
-      tableName: "dashboards",
     });
     res.status(200).send({
       dashboardId: id,

tsconfig.json

@@ -17,7 +17,8 @@
     "incremental": true,
     "moduleResolution": "node",
     "resolveJsonModule": true,
-    "isolatedModules": true
+    "isolatedModules": true,
+    "allowJs": true
   },
   "include": [
     "next-env.d.ts",