From 592309559d3a9f7508288c5d97acac0d70370bd2 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Mon, 28 Mar 2022 01:22:28 +0300 Subject: [PATCH 1/4] feat: prisma (empty config) --- package-lock.json | 36 ++++++++++++++++++++++++++++++++++++ package.json | 1 + prisma/schema.prisma | 11 +++++++++++ 3 files changed, 48 insertions(+) create mode 100644 prisma/schema.prisma diff --git a/package-lock.json b/package-lock.json index ee57773..a7c11e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ "postcss": "^8.2.1", "postcss-flexbugs-fixes": "^5.0.2", "postcss-preset-env": "^7.3.2", + "prisma": "^3.11.1", "query-string": "^7.1.1", "react": "^17.0.2", "react-component-export-image": "^1.0.6", @@ -1193,6 +1194,12 @@ "node": ">= 8" } }, + "node_modules/@prisma/engines": { + "version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9.tgz", + "integrity": "sha512-MILbsGnvmnhCbFGa2/iSnsyGyazU3afzD7ldjCIeLIGKkNBMSZgA2IvpYsAXl+6qFHKGrS3B2otKfV31dwMSQw==", + "hasInstallScript": true + }, "node_modules/@rescript/react": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@rescript%2freact/-/react-0.10.3.tgz", @@ -33505,6 +33512,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/prisma": { + "version": "3.11.1", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-3.11.1.tgz", + "integrity": "sha512-aYn8bQwt1xwR2oSsVNHT4PXU7EhsThIwmpNB/MNUaaMx5OPLTro6VdNJe/sJssXFLxhamfWeMjwmpXjljo6xkg==", + "hasInstallScript": true, + "dependencies": { + "@prisma/engines": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9" + }, + "bin": { + "prisma": "build/index.js", + "prisma2": "build/index.js" + }, + "engines": { + "node": ">=12.6" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -36219,6 +36242,11 @@ "fastq": "^1.6.0" } }, + "@prisma/engines": { + "version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9.tgz", + "integrity": "sha512-MILbsGnvmnhCbFGa2/iSnsyGyazU3afzD7ldjCIeLIGKkNBMSZgA2IvpYsAXl+6qFHKGrS3B2otKfV31dwMSQw==" + }, "@rescript/react": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@rescript%2freact/-/react-0.10.3.tgz", @@ -60202,6 +60230,14 @@ "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz", "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==" }, + "prisma": { + "version": "3.11.1", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-3.11.1.tgz", + "integrity": "sha512-aYn8bQwt1xwR2oSsVNHT4PXU7EhsThIwmpNB/MNUaaMx5OPLTro6VdNJe/sJssXFLxhamfWeMjwmpXjljo6xkg==", + "requires": { + "@prisma/engines": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", diff --git a/package.json b/package.json index 765c67a..65acc65 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "postcss": "^8.2.1", "postcss-flexbugs-fixes": "^5.0.2", "postcss-preset-env": "^7.3.2", + "prisma": "^3.11.1", "query-string": "^7.1.1", "react": "^17.0.2", "react-component-export-image": "^1.0.6", diff --git a/prisma/schema.prisma b/prisma/schema.prisma new file mode 100644 index 0000000..3d2f9fd --- /dev/null +++ b/prisma/schema.prisma @@ -0,0 +1,11 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DIGITALOCEAN_POSTGRES") +} From 6a6597c6571849ae46d5cdec9533d4ac0d8c44f7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Tue, 5 Apr 2022 23:38:40 +0300 Subject: [PATCH 2/4] WIP: more prisma migrate --- package-lock.json | 39 ++++++++ package.json | 1 + prisma/schema.prisma | 48 +++++++++- .../{manual => migrate}/noSchemaMigrate.ts | 0 src/backend/migrate/prepareForPrisma.ts | 92 +++++++++++++++++++ 5 files changed, 177 insertions(+), 3 deletions(-) rename src/backend/{manual => migrate}/noSchemaMigrate.ts (100%) create mode 100644 src/backend/migrate/prepareForPrisma.ts diff --git a/package-lock.json b/package-lock.json index a7c11e4..c71052d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "2.0.0", "license": "MIT", "dependencies": { + "@prisma/client": "^3.11.1", "@tailwindcss/forms": "^0.4.0", "@tailwindcss/typography": "^0.5.1", "@types/jsdom": "^16.2.14", @@ -1194,12 +1195,37 @@ "node": ">= 8" } }, + "node_modules/@prisma/client": { + "version": "3.11.1", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.11.1.tgz", + "integrity": "sha512-B3C7zQG4HbjJzUr2Zg9UVkBJutbqq9/uqkl1S138+keZCubJrwizx3RuIvGwI+s+pm3qbsyNqXiZgL3Ir0fSng==", + "hasInstallScript": true, + "dependencies": { + "@prisma/engines-version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9" + }, + "engines": { + "node": ">=12.6" + }, + "peerDependencies": { + "prisma": "*" + }, + "peerDependenciesMeta": { + "prisma": { + "optional": true + } + } + }, "node_modules/@prisma/engines": { "version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9", "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9.tgz", "integrity": "sha512-MILbsGnvmnhCbFGa2/iSnsyGyazU3afzD7ldjCIeLIGKkNBMSZgA2IvpYsAXl+6qFHKGrS3B2otKfV31dwMSQw==", "hasInstallScript": true }, + "node_modules/@prisma/engines-version": { + "version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9.tgz", + "integrity": "sha512-HkcsDniA4iNb/gi0iuyOJNAM7nD/LwQ0uJm15v360O5dee3TM4lWdSQiTYBMK6FF68ACUItmzSur7oYuUZ2zkQ==" + }, "node_modules/@rescript/react": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@rescript%2freact/-/react-0.10.3.tgz", @@ -36242,11 +36268,24 @@ "fastq": "^1.6.0" } }, + "@prisma/client": { + "version": "3.11.1", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-3.11.1.tgz", + "integrity": "sha512-B3C7zQG4HbjJzUr2Zg9UVkBJutbqq9/uqkl1S138+keZCubJrwizx3RuIvGwI+s+pm3qbsyNqXiZgL3Ir0fSng==", + "requires": { + "@prisma/engines-version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9" + } + }, "@prisma/engines": { "version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9", "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9.tgz", "integrity": "sha512-MILbsGnvmnhCbFGa2/iSnsyGyazU3afzD7ldjCIeLIGKkNBMSZgA2IvpYsAXl+6qFHKGrS3B2otKfV31dwMSQw==" }, + "@prisma/engines-version": { + "version": "3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-3.11.1-1.1a2506facaf1a4727b7c26850735e88ec779dee9.tgz", + "integrity": "sha512-HkcsDniA4iNb/gi0iuyOJNAM7nD/LwQ0uJm15v360O5dee3TM4lWdSQiTYBMK6FF68ACUItmzSur7oYuUZ2zkQ==" + }, "@rescript/react": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@rescript%2freact/-/react-0.10.3.tgz", diff --git a/package.json b/package.json index 65acc65..05ad0ef 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES" }, "dependencies": { + "@prisma/client": "^3.11.1", "@tailwindcss/forms": "^0.4.0", "@tailwindcss/typography": "^0.5.1", "@types/jsdom": "^16.2.14", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3d2f9fd..c69c78f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,6 +1,3 @@ -// This is your Prisma schema file, -// learn more about it in the docs: https://pris.ly/d/prisma-schema - generator client { provider = "prisma-client-js" } @@ -9,3 +6,48 @@ datasource db { provider = "postgresql" url = env("DIGITALOCEAN_POSTGRES") } + +model dashboards { + id String @id + title String + description String + contents Json + timestamp DateTime @db.Timestamp(6) + creator String + extra Json +} + +model frontpage { + id Int @id @default(autoincrement()) + frontpage_full Json + frontpage_sliced Json +} + +model history { + id String + title String + url String + platform String + description String + options Json + timestamp DateTime @db.Timestamp(6) + stars Int + qualityindicators Json + extra Json + pk Int @id @default(autoincrement()) + + @@index([id]) +} + +model questions { + id String @id + title String + url String + platform String + description String + options Json + timestamp DateTime @db.Timestamp(6) + stars Int + qualityindicators Json + extra Json +} diff --git a/src/backend/manual/noSchemaMigrate.ts b/src/backend/migrate/noSchemaMigrate.ts similarity index 100% rename from src/backend/manual/noSchemaMigrate.ts rename to src/backend/migrate/noSchemaMigrate.ts diff --git a/src/backend/migrate/prepareForPrisma.ts b/src/backend/migrate/prepareForPrisma.ts new file mode 100644 index 0000000..613f9bd --- /dev/null +++ b/src/backend/migrate/prepareForPrisma.ts @@ -0,0 +1,92 @@ +import "dotenv/config"; + +import { readWritePool } from "../database/pg-wrapper"; + +const migrate = async () => { + const client = await readWritePool.connect(); + + const execQuery = async (q: string) => { + console.log(q); + await client.query(q); + }; + + try { + await client.query("BEGIN"); + + const notNullColumn = async (table: string, column: string) => { + await execQuery( + `ALTER TABLE ${table} ALTER COLUMN ${column} SET NOT NULL` + ); + }; + + const jsonbColumn = async (table: string, column: string) => { + await execQuery( + `ALTER TABLE ${table} ALTER COLUMN ${column} SET DATA TYPE jsonb USING ${column}::jsonb` + ); + }; + + const t2c = { + dashboards: [ + "id", + "title", + "description", + "contents", + "timestamp", + "creator", + "extra", + ], + frontpage: ["frontpage_sliced", "frontpage_full"], + history: [ + "id", + "title", + "url", + "platform", + "description", + "options", + "timestamp", + "stars", + "qualityindicators", + "extra", + ], + questions: [ + "id", + "title", + "url", + "platform", + "description", + "options", + "timestamp", + "stars", + "qualityindicators", + "extra", + ], + }; + for (const [table, columns] of Object.entries(t2c)) { + for (const column of columns) { + await notNullColumn(table, column); + } + } + + await execQuery("ALTER TABLE history ADD COLUMN pk SERIAL PRIMARY KEY"); + await execQuery("ALTER TABLE dashboards ADD PRIMARY KEY (id)"); + await execQuery("ALTER TABLE questions ADD PRIMARY KEY (id)"); + + await jsonbColumn("dashboards", "contents"); + await jsonbColumn("dashboards", "extra"); + + for (const table of ["history", "questions"]) { + await jsonbColumn(table, "options"); + await jsonbColumn(table, "qualityindicators"); + await jsonbColumn(table, "extra"); + } + + await client.query("COMMIT"); + } catch (e) { + await client.query("ROLLBACK"); + throw e; + } finally { + client.release(); + } +}; + +migrate(); From 5e34ebf35e40e3cb37a47a7dad33dcdf1f8c2e79 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 7 Apr 2022 03:53:08 +0300 Subject: [PATCH 3/4] cleanup: unnecessary db-related code --- src/backend/database/pg-wrapper.ts | 256 +----------------- src/backend/flow/history/updateHistory.ts | 4 +- src/backend/flow/jobs.ts | 6 - src/backend/frontpage.ts | 12 +- src/backend/manual/manualDownload.ts | 4 +- src/backend/manual/manualInitialize.ts | 3 - src/backend/migrate/noSchemaMigrate.ts | 4 +- src/backend/migrate/prepareForPrisma.ts | 4 +- src/backend/utils/algolia.ts | 4 +- .../pullForecastsToCSVForRating.ts | 4 +- .../pullMetaculusForecastsToCSVForRating.ts | 4 +- .../misc/process-forecasts-into-elicit.ts | 4 +- 12 files changed, 28 insertions(+), 281 deletions(-) delete mode 100644 src/backend/manual/manualInitialize.ts diff --git a/src/backend/database/pg-wrapper.ts b/src/backend/database/pg-wrapper.ts index 5cb1990..aba9324 100644 --- a/src/backend/database/pg-wrapper.ts +++ b/src/backend/database/pg-wrapper.ts @@ -11,7 +11,7 @@ const allTableNames = [...forecastTableNames, "dashboards", "frontpage"]; /* Postgres database connection code */ const databaseURL = process.env.DIGITALOCEAN_POSTGRES; -export const readWritePool = new Pool({ +export const pool = new Pool({ connectionString: databaseURL, ssl: process.env.POSTGRES_NO_SSL ? false @@ -20,258 +20,15 @@ export const readWritePool = new Pool({ }, }); -const readOnlyDatabaseURL = - "postgresql://public_read_only_user:gOcihnLhqRIQUQYt@postgres-red-do-user-10290909-0.b.db.ondigitalocean.com:25060/metaforecastpg?sslmode=require" || - process.env.DIGITALOCEAN_POSTGRES_PUBLIC; -const readOnlyPool = new Pool({ - // never used - connectionString: readOnlyDatabaseURL, - ssl: process.env.POSTGRES_NO_SSL - ? false - : { - rejectUnauthorized: false, - }, -}); - -// Helpers -export const runPgCommand = async ({ - command, - pool, -}: { - command: string; - pool: Pool; -}) => { - console.log(command); - const client = await pool.connect(); - let result; - try { - let response = await client.query(command); - result = { results: response ? response.rows : null }; - } catch (error) { - console.log(error); - } finally { - client.release(); - } - return result; -}; - -// Initialize -let dropTable = (table: string) => `DROP TABLE IF EXISTS ${table}`; -let createIndex = (table: string) => - `CREATE INDEX ${table}_id_index ON ${table} (id);`; -let createUniqueIndex = (table: string) => - `CREATE UNIQUE INDEX ${table}_id_index ON ${table} (id);`; - -async function pgInitializeScaffolding() { - async function setPermissionsForPublicUser() { - let initCommands = [ - "REVOKE ALL ON DATABASE metaforecastpg FROM public_read_only_user;", - "GRANT CONNECT ON DATABASE metaforecastpg TO public_read_only_user;", - ]; - for (let command of initCommands) { - await runPgCommand({ command, pool: readWritePool }); - } - - await runPgCommand({ - command: - "GRANT SELECT ON ALL TABLES IN SCHEMA public TO public_read_only_user", - pool: readWritePool, - }); - - await runPgCommand({ - command: - "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO public_read_only_user", - pool: readWritePool, - }); - } - let YOLO = false; - if (YOLO) { - console.log("Set public user permissions"); - await setPermissionsForPublicUser(); - console.log(""); - } else { - console.log( - "pgInitializeScaffolding: This command is dangerous, set YOLO to true in the code to invoke it" - ); - } -} - -let buildMetaforecastTable = (table: string) => `CREATE TABLE ${table} ( - id text, - title text, - url text, - platform text, - description text, - options json, - timestamp timestamp, - stars int, - qualityindicators json, - extra json - );`; - -async function pgInitializeQuestions() { - let YOLO = false; - if (YOLO) { - console.log("Create tables & their indexes"); - const table = "questions"; - await runPgCommand({ - command: dropTable(table), - pool: readWritePool, - }); - await runPgCommand({ - command: buildMetaforecastTable(table), - pool: readWritePool, - }); - await runPgCommand({ - command: createUniqueIndex(table), - pool: readWritePool, - }); - console.log(""); - } else { - console.log( - "pgInitializeQuestions: This command is dangerous, set YOLO to true in the code to invoke it" - ); - } -} - -async function pgInitializeDashboards() { - let buildDashboard = () => - `CREATE TABLE dashboards ( - id text, - title text, - description text, - contents json, - timestamp timestamp, - creator text, - extra json - );`; - let YOLO = false; - if (YOLO) { - console.log("Create dashboard table and its index"); - - await runPgCommand({ - command: dropTable("dashboards"), - pool: readWritePool, - }); - - await runPgCommand({ - command: buildDashboard(), - pool: readWritePool, - }); - - await runPgCommand({ - command: createUniqueIndex("dashboards"), - pool: readWritePool, - }); - console.log(""); - } else { - console.log( - "pgInitializeDashboard: This command is dangerous, set YOLO to true in the code to invoke it" - ); - } -} - -let buildHistoryTable = (table: string) => `CREATE TABLE ${table} ( - id text, - title text, - url text, - platform text, - description text, - options json, - timestamp timestamp, - stars int, - qualityindicators json, - extra json - );`; -export async function pgInitializeHistories() { - let YOLO = false; - if (YOLO) { - console.log("Create history table & index"); - await runPgCommand({ - command: dropTable("history"), - pool: readWritePool, - }); - await runPgCommand({ - command: buildHistoryTable("history"), - pool: readWritePool, - }); - await runPgCommand({ - command: createIndex("history"), // Not unique!! - pool: readWritePool, - }); - console.log(""); - } else { - console.log( - "pgInitializeHistories: This command is dangerous, set YOLO to true in the code to invoke it" - ); - } -} - -async function pgInitializeFrontpage() { - let YOLO = false; - if (YOLO) { - await runPgCommand({ - command: dropTable("frontpage"), - pool: readWritePool, - }); - await runPgCommand({ - command: `CREATE TABLE frontpage ( - id serial primary key, - frontpage_full jsonb, - frontpage_sliced jsonb - );`, - pool: readWritePool, - }); - } else { - console.log( - "pgInitializeFrontpage: This command is dangerous, set YOLO to true in the code to invoke it" - ); - } -} - -export async function pgInitialize() { - await pgInitializeScaffolding(); - await pgInitializeQuestions(); - await pgInitializeHistories(); - await pgInitializeDashboards(); - await pgInitializeFrontpage(); -} - // Read -async function pgReadWithPool({ - tableName, - pool, -}: { - tableName: string; - pool: Pool; -}) { +export async function pgRead({ tableName }: { tableName: string }) { if (!allTableNames.includes(tableName)) { throw Error( `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections` ); } let command = `SELECT * from ${tableName}`; - let response = await runPgCommand({ command, pool }); - let results = response.results; - return results; -} - -export async function pgRead({ tableName }: { tableName: string }) { - return await pgReadWithPool({ tableName, pool: readWritePool }); -} - -export async function pgReadWithReadCredentials({ - tableName, -}: { - tableName: string; -}) { - // currently does not work. - /* return await pgReadWithPool({ - tableName, - pool: readOnlyPool, - }); - */ - return await pgReadWithPool({ tableName, pool: readWritePool }); + return (await pool.query(command)).rows; } export async function pgGetByIds({ @@ -284,8 +41,7 @@ export async function pgGetByIds({ let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3) let command = `SELECT * from ${table} where id in ${idstring}`; // see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn - let response = await runPgCommand({ command, pool: readWritePool }); - let results = response.results; + let results = (await pool.query(command)).rows; console.log(results); return results; } @@ -372,7 +128,7 @@ export async function pgInsertIntoDashboard({ datum }) { datum.creator || "", JSON.stringify(datum.extra || []), ]; - const client = await readWritePool.connect(); + const client = await pool.connect(); let result; try { result = await client.query(text, values); @@ -426,7 +182,7 @@ export async function pgUpsert({ } await measureTime(async () => { - const client = await readWritePool.connect(); + const client = await pool.connect(); try { await client.query("BEGIN"); if (replacePlatform) { diff --git a/src/backend/flow/history/updateHistory.ts b/src/backend/flow/history/updateHistory.ts index 8132390..d3e704e 100644 --- a/src/backend/flow/history/updateHistory.ts +++ b/src/backend/flow/history/updateHistory.ts @@ -1,7 +1,7 @@ -import { pgReadWithReadCredentials, pgUpsert } from "../../database/pg-wrapper"; +import { pgRead, pgUpsert } from "../../database/pg-wrapper"; export async function updateHistory() { - let latest = await pgReadWithReadCredentials({ tableName: "questions" }); + let latest = await pgRead({ tableName: "questions" }); await pgUpsert({ contents: latest, tableName: "history", diff --git a/src/backend/flow/jobs.ts b/src/backend/flow/jobs.ts index 7da315c..01a5a81 100644 --- a/src/backend/flow/jobs.ts +++ b/src/backend/flow/jobs.ts @@ -1,4 +1,3 @@ -import { pgInitialize } from "../database/pg-wrapper"; import { doEverything } from "../flow/doEverything"; import { updateHistory } from "../flow/history/updateHistory"; import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData"; @@ -45,11 +44,6 @@ export const jobs: Job[] = [ run: doEverything, separate: true, }, - { - name: "migrate", - message: "Initialize postgres database", - run: pgInitialize, - }, ]; function sleep(ms: number) { diff --git a/src/backend/frontpage.ts b/src/backend/frontpage.ts index 705da30..a010d00 100644 --- a/src/backend/frontpage.ts +++ b/src/backend/frontpage.ts @@ -1,8 +1,8 @@ -import { pgRead, readWritePool } from "./database/pg-wrapper"; +import { pgRead, pool } from "./database/pg-wrapper"; import { Forecast } from "./platforms"; export async function getFrontpage(): Promise { - const res = await readWritePool.query( + const res = await pool.query( "SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1" ); if (!res.rows.length) return []; @@ -10,7 +10,7 @@ export async function getFrontpage(): Promise { } export async function getFrontpageFull(): Promise { - const res = await readWritePool.query( + const res = await pool.query( "SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1" ); if (!res.rows.length) return []; @@ -23,18 +23,18 @@ export async function rebuildFrontpage() { }); const frontpageSliced = ( - await readWritePool.query(` + await pool.query(` SELECT * FROM questions WHERE (qualityindicators->>'stars')::int >= 3 AND description != '' - AND JSON_ARRAY_LENGTH(options) > 0 + AND JSONB_ARRAY_LENGTH(options) > 0 ORDER BY RANDOM() LIMIT 50 `) ).rows; const start = Date.now(); - await readWritePool.query( + await pool.query( "INSERT INTO frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)", [JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)] ); diff --git a/src/backend/manual/manualDownload.ts b/src/backend/manual/manualDownload.ts index 5b174cb..88472ef 100644 --- a/src/backend/manual/manualDownload.ts +++ b/src/backend/manual/manualDownload.ts @@ -2,10 +2,10 @@ import "dotenv/config"; import fs from "fs"; -import { pgReadWithReadCredentials } from "../database/pg-wrapper"; +import { pgRead } from "../database/pg-wrapper"; let main = async () => { - let json = await pgReadWithReadCredentials({ tableName: "questions" }); + let json = await pgRead({ tableName: "questions" }); let string = JSON.stringify(json, null, 2); let filename = "metaforecasts.json"; fs.writeFileSync(filename, string); diff --git a/src/backend/manual/manualInitialize.ts b/src/backend/manual/manualInitialize.ts deleted file mode 100644 index 1427013..0000000 --- a/src/backend/manual/manualInitialize.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { pgInitialize } from "../database/pg-wrapper"; - -pgInitialize(); diff --git a/src/backend/migrate/noSchemaMigrate.ts b/src/backend/migrate/noSchemaMigrate.ts index eca83b0..19a1f5a 100644 --- a/src/backend/migrate/noSchemaMigrate.ts +++ b/src/backend/migrate/noSchemaMigrate.ts @@ -1,9 +1,9 @@ import "dotenv/config"; -import { readWritePool } from "../database/pg-wrapper"; +import { pool } from "../database/pg-wrapper"; const migrate = async () => { - const client = await readWritePool.connect(); + const client = await pool.connect(); const execQuery = async (q: string) => { console.log(q); diff --git a/src/backend/migrate/prepareForPrisma.ts b/src/backend/migrate/prepareForPrisma.ts index 613f9bd..cee8905 100644 --- a/src/backend/migrate/prepareForPrisma.ts +++ b/src/backend/migrate/prepareForPrisma.ts @@ -1,9 +1,9 @@ import "dotenv/config"; -import { readWritePool } from "../database/pg-wrapper"; +import { pool } from "../database/pg-wrapper"; const migrate = async () => { - const client = await readWritePool.connect(); + const client = await pool.connect(); const execQuery = async (q: string) => { console.log(q); diff --git a/src/backend/utils/algolia.ts b/src/backend/utils/algolia.ts index 1cc3bb4..e6296e0 100644 --- a/src/backend/utils/algolia.ts +++ b/src/backend/utils/algolia.ts @@ -1,6 +1,6 @@ import algoliasearch from "algoliasearch"; -import { pgReadWithReadCredentials } from "../database/pg-wrapper"; +import { pgRead } from "../database/pg-wrapper"; import { platforms } from "../platforms"; let cookie = process.env.ALGOLIA_MASTER_API_KEY; @@ -20,7 +20,7 @@ let getoptionsstringforsearch = (record: any) => { }; export async function rebuildAlgoliaDatabaseTheEasyWay() { - let records: any[] = await pgReadWithReadCredentials({ + let records: any[] = await pgRead({ tableName: "questions", }); diff --git a/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts b/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts index 6f68afc..d6ac803 100644 --- a/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts +++ b/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts @@ -1,7 +1,7 @@ /* Imports */ import fs from "fs"; -import { pgReadWithReadCredentials } from "../../database/pg-wrapper"; +import { pgRead } from "../../database/pg-wrapper"; /* Definitions */ @@ -24,7 +24,7 @@ let main = async () => { "PredictIt", "Rootclaim", ]; - let json = await pgReadWithReadCredentials({ tableName: "questions" }); + let json = await pgRead({ tableName: "questions" }); console.log(json.length); //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))] //console.log(uniquePlatforms) diff --git a/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts b/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts index 7394638..ddd8c42 100644 --- a/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts +++ b/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts @@ -1,7 +1,7 @@ /* Imports */ import fs from "fs"; -import { pgReadWithReadCredentials } from "../../database/pg-wrapper"; +import { pgRead } from "../../database/pg-wrapper"; /* Definitions */ @@ -26,7 +26,7 @@ let shuffleArray = (array) => { let main = async () => { let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim'] - let json = await pgReadWithReadCredentials({ tableName: "questions" }); + let json = await pgRead({ tableName: "questions" }); console.log(json.length); //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))] //console.log(uniquePlatforms) diff --git a/src/backend/utils/misc/process-forecasts-into-elicit.ts b/src/backend/utils/misc/process-forecasts-into-elicit.ts index 7724bbb..c6ce4fb 100644 --- a/src/backend/utils/misc/process-forecasts-into-elicit.ts +++ b/src/backend/utils/misc/process-forecasts-into-elicit.ts @@ -1,7 +1,7 @@ /* Imports */ import fs from "fs"; -import { pgReadWithReadCredentials } from "../../database/pg-wrapper"; +import { pgRead } from "../../database/pg-wrapper"; /* Definitions */ let locationData = "./data/"; @@ -9,7 +9,7 @@ let locationData = "./data/"; /* Body */ // let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src async function main() { - let data = await pgReadWithReadCredentials({ tableName: "questions" }); //JSON.parse(rawdata) + let data = await pgRead({ tableName: "questions" }); //JSON.parse(rawdata) let processDescription = (description) => { if (description == null || description == undefined || description == "") { return ""; From 8b70568795d78714ff275d57bf9aa3df0f114c07 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 7 Apr 2022 23:19:59 +0300 Subject: [PATCH 4/4] feat: initial prisma migration --- .../20220407201706_init/migration.sql | 57 +++++++++++++++++++ prisma/migrations/migration_lock.toml | 3 + 2 files changed, 60 insertions(+) create mode 100644 prisma/migrations/20220407201706_init/migration.sql create mode 100644 prisma/migrations/migration_lock.toml diff --git a/prisma/migrations/20220407201706_init/migration.sql b/prisma/migrations/20220407201706_init/migration.sql new file mode 100644 index 0000000..f4ecbdd --- /dev/null +++ b/prisma/migrations/20220407201706_init/migration.sql @@ -0,0 +1,57 @@ +-- CreateTable +CREATE TABLE "dashboards" ( + "id" TEXT NOT NULL, + "title" TEXT NOT NULL, + "description" TEXT NOT NULL, + "contents" JSONB NOT NULL, + "timestamp" TIMESTAMP(6) NOT NULL, + "creator" TEXT NOT NULL, + "extra" JSONB NOT NULL, + + CONSTRAINT "dashboards_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "frontpage" ( + "id" SERIAL NOT NULL, + "frontpage_full" JSONB NOT NULL, + "frontpage_sliced" JSONB NOT NULL, + + CONSTRAINT "frontpage_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "history" ( + "id" TEXT NOT NULL, + "title" TEXT NOT NULL, + "url" TEXT NOT NULL, + "platform" TEXT NOT NULL, + "description" TEXT NOT NULL, + "options" JSONB NOT NULL, + "timestamp" TIMESTAMP(6) NOT NULL, + "stars" INTEGER NOT NULL, + "qualityindicators" JSONB NOT NULL, + "extra" JSONB NOT NULL, + "pk" SERIAL NOT NULL, + + CONSTRAINT "history_pkey" PRIMARY KEY ("pk") +); + +-- CreateTable +CREATE TABLE "questions" ( + "id" TEXT NOT NULL, + "title" TEXT NOT NULL, + "url" TEXT NOT NULL, + "platform" TEXT NOT NULL, + "description" TEXT NOT NULL, + "options" JSONB NOT NULL, + "timestamp" TIMESTAMP(6) NOT NULL, + "stars" INTEGER NOT NULL, + "qualityindicators" JSONB NOT NULL, + "extra" JSONB NOT NULL, + + CONSTRAINT "questions_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "history_id_idx" ON "history"("id"); diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..fbffa92 --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" \ No newline at end of file