quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: store question ids only for frontpage (closes: #49)

Browse Source
This commit is contained in:
Vyacheslav Matyukhin 2022-04-22 23:50:28 +04:00
parent 3ef786db4b
commit b258afe16d
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
5 changed files with 45 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
prisma/migrations/20220422193152_frontpage_ids/migration.sql Normal file
View File

@ -0,0 +1,10 @@
-- CreateTable
CREATE TABLE "FrontpageId" (
"id" TEXT NOT NULL
);
-- CreateIndex
CREATE UNIQUE INDEX "FrontpageId_id_key" ON "FrontpageId"("id");
-- AddForeignKey
ALTER TABLE "FrontpageId" ADD CONSTRAINT "FrontpageId_id_fkey" FOREIGN KEY ("id") REFERENCES "questions"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

6
prisma/schema.prisma
View File

@ -60,5 +60,11 @@ model Question {
qualityindicators Json
extra Json
onFrontpage FrontpageId?
@@map("questions")
}
model FrontpageId {
question Question @relation(fields: [id], references: [id])
id String @unique
}

67
src/backend/database/pg-wrapper.ts
View File

@ -1,7 +1,6 @@
import { Pool, PoolClient } from "pg";
import { Question } from "../platforms";
import { hash } from "../utils/hash";
import { measureTime } from "../utils/measureTime";
import { roughSizeOfObject } from "../utils/roughSize";
@ -31,21 +30,6 @@ export async function pgRead({ tableName }: { tableName: string }) {
return (await pool.query(command)).rows;
}
export async function pgGetByIds({
ids,
table,
}: {
ids: string[];
table: string;
}) {
let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3)
let command = `SELECT * from ${table} where id in ${idstring}`;
// see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn
let results = (await pool.query(command)).rows;
console.log(results);
return results;
}
export async function pgBulkInsert({
data,
tableName,
@ -115,57 +99,6 @@ export async function pgBulkInsert({
}
}
export async function pgInsertIntoDashboard({ datum }) {
let text = `INSERT INTO dashboards VALUES($1, $2, $3, $4, $5, $6, $7)`;
let timestamp = datum.timestamp || new Date().toISOString();
timestamp = timestamp.slice(0, 19).replace("T", " ");
let values = [
hash(JSON.stringify(datum.contents)),
datum.title || "",
datum.description || "",
JSON.stringify(datum.contents || []),
timestamp, // fixed
datum.creator || "",
JSON.stringify(datum.extra || []),
];
const client = await pool.connect();
let result;
try {
result = await client.query(text, values);
} catch (error) {
console.log(error);
} finally {
client.release();
}
// console.log(result)
return result;
}
/* For reference
id text,
title text,
description text,
contents json,
timestamp timestamp,
creator text,
extra json
*/
/*
pgInsertIntoDashboard({
datum: {
title: "Test dashboard",
description: "A test dashboard",
contents: [
"rootclaim-did-former-new-england-patriots-tight-end-aaron-hernandez-commit-suicide-19060",
"metaculus-3912",
"givewellopenphil-2021-133",
"rootclaim-what-happened-to-barry-and-honey-sherman-19972",
"rootclaim-what-caused-the-disappearance-of-malaysia-airlines-flight-370",
],
creator: "Nuño Sempere",
},
tableName: "dashboards",
});
*/
export async function pgUpsert({
contents,
tableName,

68
src/backend/frontpage.ts
View File

@ -1,47 +1,35 @@
import { pgRead, pool } from "./database/pg-wrapper";
import { Question } from "./platforms";
import { Question } from "@prisma/client";
import { measureTime } from "./utils/measureTime";
export async function getFrontpage(): Promise<Question[]> {
const res = await pool.query(
"SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
);
if (!res.rows.length) return [];
return res.rows[0].frontpage_sliced;
}
export async function getFrontpageFull(): Promise<Question[]> {
const res = await pool.query(
"SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
);
if (!res.rows.length) return [];
return res.rows[0].frontpage_full;
const questions = await prisma.question.findMany({
where: {
onFrontpage: {
isNot: null,
},
},
});
console.log(questions.length);
return questions;
}
export async function rebuildFrontpage() {
const frontpageFull = await pgRead({
tableName: "questions",
await measureTime(async () => {
const rows = await prisma.$queryRaw<{ id: string }[]>`
SELECT id FROM questions
WHERE
(qualityindicators->>'stars')::int >= 3
AND description != ''
AND JSONB_ARRAY_LENGTH(options) > 0
ORDER BY RANDOM() LIMIT 50
`;
await prisma.$transaction([
prisma.frontpageId.deleteMany({}),
prisma.frontpageId.createMany({
data: rows,
}),
]);
});
const frontpageSliced = (
await pool.query(`
SELECT * FROM questions
WHERE
(qualityindicators->>'stars')::int >= 3
AND description != ''
AND JSONB_ARRAY_LENGTH(options) > 0
ORDER BY RANDOM() LIMIT 50
`)
).rows;
const start = Date.now();
await pool.query(
"INSERT INTO frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
[JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
);
const end = Date.now();
const difference = end - start;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}

19
src/graphql/schema/frontpage.ts
View File

@ -1,6 +1,3 @@
import { Question } from "@prisma/client";
import { prisma } from "../../backend/database/prisma";
import { getFrontpage } from "../../backend/frontpage";
import { builder } from "../builder";
import { QuestionObj } from "./questions";
@ -10,21 +7,7 @@ builder.queryField("frontpage", (t) =>
type: [QuestionObj],
description: "Get a list of questions that are currently on the frontpage",
resolve: async () => {
const legacyQuestions = await getFrontpage();
const ids = legacyQuestions.map((q) => q.id);
const questions = await prisma.question.findMany({
where: {
id: {
in: ids,
},
},
});
const id2q: { [k: string]: Question } = {};
for (const q of questions) {
id2q[q.id] = q;
}
return ids.map((id) => id2q[id] || null).filter((q) => q !== null);
return await getFrontpage();
},
})
);