feat: store question ids only for frontpage (closes: #49)

prisma/migrations/20220422193152_frontpage_ids/migration.sql

@@ -0,0 +1,10 @@
+-- CreateTable
+CREATE TABLE "FrontpageId" (
+    "id" TEXT NOT NULL
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "FrontpageId_id_key" ON "FrontpageId"("id");
+
+-- AddForeignKey
+ALTER TABLE "FrontpageId" ADD CONSTRAINT "FrontpageId_id_fkey" FOREIGN KEY ("id") REFERENCES "questions"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/schema.prisma

@@ -60,5 +60,11 @@ model Question {
   qualityindicators Json
   extra             Json
 
+  onFrontpage FrontpageId?
   @@map("questions")
 }
+
+model FrontpageId {
+  question Question @relation(fields: [id], references: [id])
+  id       String   @unique
+}

src/backend/database/pg-wrapper.ts

@@ -1,7 +1,6 @@
 import { Pool, PoolClient } from "pg";
 
 import { Question } from "../platforms";
-import { hash } from "../utils/hash";
 import { measureTime } from "../utils/measureTime";
 import { roughSizeOfObject } from "../utils/roughSize";
 
@@ -31,21 +30,6 @@ export async function pgRead({ tableName }: { tableName: string }) {
   return (await pool.query(command)).rows;
 }
 
-export async function pgGetByIds({
-  ids,
-  table,
-}: {
-  ids: string[];
-  table: string;
-}) {
-  let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3)
-  let command = `SELECT * from ${table} where id in ${idstring}`;
-  // see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn
-  let results = (await pool.query(command)).rows;
-  console.log(results);
-  return results;
-}
-
 export async function pgBulkInsert({
   data,
   tableName,
@@ -115,57 +99,6 @@ export async function pgBulkInsert({
   }
 }
 
-export async function pgInsertIntoDashboard({ datum }) {
-  let text = `INSERT INTO dashboards VALUES($1, $2, $3, $4, $5, $6, $7)`;
-  let timestamp = datum.timestamp || new Date().toISOString();
-  timestamp = timestamp.slice(0, 19).replace("T", " ");
-  let values = [
-    hash(JSON.stringify(datum.contents)),
-    datum.title || "",
-    datum.description || "",
-    JSON.stringify(datum.contents || []),
-    timestamp, // fixed
-    datum.creator || "",
-    JSON.stringify(datum.extra || []),
-  ];
-  const client = await pool.connect();
-  let result;
-  try {
-    result = await client.query(text, values);
-  } catch (error) {
-    console.log(error);
-  } finally {
-    client.release();
-  }
-  // console.log(result)
-  return result;
-}
-/* For reference
-	  id text,
-		title text,
-		description text,
-		contents json,
-		timestamp timestamp,
-		creator text,
-		extra json
- */
-/*
-pgInsertIntoDashboard({
-  datum: {
-    title: "Test dashboard",
-    description: "A test dashboard",
-    contents: [
-      "rootclaim-did-former-new-england-patriots-tight-end-aaron-hernandez-commit-suicide-19060",
-      "metaculus-3912",
-      "givewellopenphil-2021-133",
-      "rootclaim-what-happened-to-barry-and-honey-sherman-19972",
-      "rootclaim-what-caused-the-disappearance-of-malaysia-airlines-flight-370",
-    ],
-    creator: "Nuño Sempere",
-  },
-  tableName: "dashboards",
-});
-*/
 export async function pgUpsert({
   contents,
   tableName,

src/backend/frontpage.ts

@@ -1,47 +1,35 @@
-import { pgRead, pool } from "./database/pg-wrapper";
+import { Question } from "@prisma/client";
-import { Question } from "./platforms";
+
+import { measureTime } from "./utils/measureTime";
 
 export async function getFrontpage(): Promise<Question[]> {
-  const res = await pool.query(
+  const questions = await prisma.question.findMany({
-    "SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
+    where: {
-  );
+      onFrontpage: {
-  if (!res.rows.length) return [];
+        isNot: null,
-  return res.rows[0].frontpage_sliced;
+      },
-}
+    },
-
+  });
-export async function getFrontpageFull(): Promise<Question[]> {
+  console.log(questions.length);
-  const res = await pool.query(
+  return questions;
-    "SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
-  );
-  if (!res.rows.length) return [];
-  return res.rows[0].frontpage_full;
 }
 
 export async function rebuildFrontpage() {
-  const frontpageFull = await pgRead({
+  await measureTime(async () => {
-    tableName: "questions",
+    const rows = await prisma.$queryRaw<{ id: string }[]>`
-  });
+      SELECT id FROM questions
-
-  const frontpageSliced = (
-    await pool.query(`
-    SELECT * FROM questions
       WHERE
         (qualityindicators->>'stars')::int >= 3
         AND description != ''
         AND JSONB_ARRAY_LENGTH(options) > 0
       ORDER BY RANDOM() LIMIT 50
-  `)
+    `;
-  ).rows;
 
-  const start = Date.now();
+    await prisma.$transaction([
-  await pool.query(
+      prisma.frontpageId.deleteMany({}),
-    "INSERT INTO frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
+      prisma.frontpageId.createMany({
-    [JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
+        data: rows,
-  );
+      }),
-
+    ]);
-  const end = Date.now();
+  });
-  const difference = end - start;
-  console.log(
-    `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
-  );
 }

src/graphql/schema/frontpage.ts

@@ -1,6 +1,3 @@
-import { Question } from "@prisma/client";
-
-import { prisma } from "../../backend/database/prisma";
 import { getFrontpage } from "../../backend/frontpage";
 import { builder } from "../builder";
 import { QuestionObj } from "./questions";
@@ -10,21 +7,7 @@ builder.queryField("frontpage", (t) =>
     type: [QuestionObj],
     description: "Get a list of questions that are currently on the frontpage",
     resolve: async () => {
-      const legacyQuestions = await getFrontpage();
+      return await getFrontpage();
-      const ids = legacyQuestions.map((q) => q.id);
-      const questions = await prisma.question.findMany({
-        where: {
-          id: {
-            in: ids,
-          },
-        },
-      });
-      const id2q: { [k: string]: Question } = {};
-      for (const q of questions) {
-        id2q[q.id] = q;
-      }
-
-      return ids.map((id) => id2q[id] || null).filter((q) => q !== null);
     },
   })
 );