From b258afe16d454ce6643f0974e8df316885a886f4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Fri, 22 Apr 2022 23:50:28 +0400 Subject: [PATCH] feat: store question ids only for frontpage (closes: #49) --- .../migration.sql | 10 +++ prisma/schema.prisma | 6 ++ src/backend/database/pg-wrapper.ts | 67 ------------------ src/backend/frontpage.ts | 68 ++++++++----------- src/graphql/schema/frontpage.ts | 19 +----- 5 files changed, 45 insertions(+), 125 deletions(-) create mode 100644 prisma/migrations/20220422193152_frontpage_ids/migration.sql diff --git a/prisma/migrations/20220422193152_frontpage_ids/migration.sql b/prisma/migrations/20220422193152_frontpage_ids/migration.sql new file mode 100644 index 0000000..d46f7d1 --- /dev/null +++ b/prisma/migrations/20220422193152_frontpage_ids/migration.sql @@ -0,0 +1,10 @@ +-- CreateTable +CREATE TABLE "FrontpageId" ( + "id" TEXT NOT NULL +); + +-- CreateIndex +CREATE UNIQUE INDEX "FrontpageId_id_key" ON "FrontpageId"("id"); + +-- AddForeignKey +ALTER TABLE "FrontpageId" ADD CONSTRAINT "FrontpageId_id_fkey" FOREIGN KEY ("id") REFERENCES "questions"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index faa6483..49baa57 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -60,5 +60,11 @@ model Question { qualityindicators Json extra Json + onFrontpage FrontpageId? @@map("questions") } + +model FrontpageId { + question Question @relation(fields: [id], references: [id]) + id String @unique +} diff --git a/src/backend/database/pg-wrapper.ts b/src/backend/database/pg-wrapper.ts index 3a57067..f7bc03f 100644 --- a/src/backend/database/pg-wrapper.ts +++ b/src/backend/database/pg-wrapper.ts @@ -1,7 +1,6 @@ import { Pool, PoolClient } from "pg"; import { Question } from "../platforms"; -import { hash } from "../utils/hash"; import { measureTime } from "../utils/measureTime"; import { roughSizeOfObject } from "../utils/roughSize"; @@ -31,21 +30,6 @@ export async function pgRead({ tableName }: { tableName: string }) { return (await pool.query(command)).rows; } -export async function pgGetByIds({ - ids, - table, -}: { - ids: string[]; - table: string; -}) { - let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3) - let command = `SELECT * from ${table} where id in ${idstring}`; - // see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn - let results = (await pool.query(command)).rows; - console.log(results); - return results; -} - export async function pgBulkInsert({ data, tableName, @@ -115,57 +99,6 @@ export async function pgBulkInsert({ } } -export async function pgInsertIntoDashboard({ datum }) { - let text = `INSERT INTO dashboards VALUES($1, $2, $3, $4, $5, $6, $7)`; - let timestamp = datum.timestamp || new Date().toISOString(); - timestamp = timestamp.slice(0, 19).replace("T", " "); - let values = [ - hash(JSON.stringify(datum.contents)), - datum.title || "", - datum.description || "", - JSON.stringify(datum.contents || []), - timestamp, // fixed - datum.creator || "", - JSON.stringify(datum.extra || []), - ]; - const client = await pool.connect(); - let result; - try { - result = await client.query(text, values); - } catch (error) { - console.log(error); - } finally { - client.release(); - } - // console.log(result) - return result; -} -/* For reference - id text, - title text, - description text, - contents json, - timestamp timestamp, - creator text, - extra json - */ -/* -pgInsertIntoDashboard({ - datum: { - title: "Test dashboard", - description: "A test dashboard", - contents: [ - "rootclaim-did-former-new-england-patriots-tight-end-aaron-hernandez-commit-suicide-19060", - "metaculus-3912", - "givewellopenphil-2021-133", - "rootclaim-what-happened-to-barry-and-honey-sherman-19972", - "rootclaim-what-caused-the-disappearance-of-malaysia-airlines-flight-370", - ], - creator: "Nuño Sempere", - }, - tableName: "dashboards", -}); -*/ export async function pgUpsert({ contents, tableName, diff --git a/src/backend/frontpage.ts b/src/backend/frontpage.ts index 36e37b4..3d41e93 100644 --- a/src/backend/frontpage.ts +++ b/src/backend/frontpage.ts @@ -1,47 +1,35 @@ -import { pgRead, pool } from "./database/pg-wrapper"; -import { Question } from "./platforms"; +import { Question } from "@prisma/client"; + +import { measureTime } from "./utils/measureTime"; export async function getFrontpage(): Promise { - const res = await pool.query( - "SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1" - ); - if (!res.rows.length) return []; - return res.rows[0].frontpage_sliced; -} - -export async function getFrontpageFull(): Promise { - const res = await pool.query( - "SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1" - ); - if (!res.rows.length) return []; - return res.rows[0].frontpage_full; + const questions = await prisma.question.findMany({ + where: { + onFrontpage: { + isNot: null, + }, + }, + }); + console.log(questions.length); + return questions; } export async function rebuildFrontpage() { - const frontpageFull = await pgRead({ - tableName: "questions", + await measureTime(async () => { + const rows = await prisma.$queryRaw<{ id: string }[]>` + SELECT id FROM questions + WHERE + (qualityindicators->>'stars')::int >= 3 + AND description != '' + AND JSONB_ARRAY_LENGTH(options) > 0 + ORDER BY RANDOM() LIMIT 50 + `; + + await prisma.$transaction([ + prisma.frontpageId.deleteMany({}), + prisma.frontpageId.createMany({ + data: rows, + }), + ]); }); - - const frontpageSliced = ( - await pool.query(` - SELECT * FROM questions - WHERE - (qualityindicators->>'stars')::int >= 3 - AND description != '' - AND JSONB_ARRAY_LENGTH(options) > 0 - ORDER BY RANDOM() LIMIT 50 - `) - ).rows; - - const start = Date.now(); - await pool.query( - "INSERT INTO frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)", - [JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)] - ); - - const end = Date.now(); - const difference = end - start; - console.log( - `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` - ); } diff --git a/src/graphql/schema/frontpage.ts b/src/graphql/schema/frontpage.ts index 46d8464..5b7693e 100644 --- a/src/graphql/schema/frontpage.ts +++ b/src/graphql/schema/frontpage.ts @@ -1,6 +1,3 @@ -import { Question } from "@prisma/client"; - -import { prisma } from "../../backend/database/prisma"; import { getFrontpage } from "../../backend/frontpage"; import { builder } from "../builder"; import { QuestionObj } from "./questions"; @@ -10,21 +7,7 @@ builder.queryField("frontpage", (t) => type: [QuestionObj], description: "Get a list of questions that are currently on the frontpage", resolve: async () => { - const legacyQuestions = await getFrontpage(); - const ids = legacyQuestions.map((q) => q.id); - const questions = await prisma.question.findMany({ - where: { - id: { - in: ids, - }, - }, - }); - const id2q: { [k: string]: Question } = {}; - for (const q of questions) { - id2q[q.id] = q; - } - - return ids.map((id) => id2q[id] || null).filter((q) => q !== null); + return await getFrontpage(); }, }) );