From 185464c0ae1a230f656e437232ff2cd665ef1e21 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Wed, 27 Apr 2022 22:07:00 +0400 Subject: [PATCH] feat: insert questions without removing (slower, but fixes fks) --- .../migration.sql | 5 ++ prisma/schema.prisma | 11 ++- src/backend/flow/history/updateHistory.ts | 5 +- src/backend/platforms/index.ts | 80 +++++++++++++----- src/backend/platforms/xrisk.ts | 14 ++-- src/graphql/schema/questions.ts | 84 ++++++++++++------- src/web/display/DisplayQuestion/index.tsx | 18 ++-- 7 files changed, 151 insertions(+), 66 deletions(-) create mode 100644 prisma/migrations/20220425220646_history_relation/migration.sql diff --git a/prisma/migrations/20220425220646_history_relation/migration.sql b/prisma/migrations/20220425220646_history_relation/migration.sql new file mode 100644 index 0000000..dd2a023 --- /dev/null +++ b/prisma/migrations/20220425220646_history_relation/migration.sql @@ -0,0 +1,5 @@ +ALTER TABLE "history" ADD COLUMN "idref" TEXT; + +ALTER TABLE "history" ADD CONSTRAINT "history_idref_fkey" FOREIGN KEY ("idref") REFERENCES "questions"("id") ON DELETE SET NULL ON UPDATE RESTRICT; + +UPDATE "history" SET idref = id WHERE id in (SELECT id FROM "questions"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9b5a035..0cc5a28 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,5 +1,6 @@ generator client { - provider = "prisma-client-js" + provider = "prisma-client-js" + previewFeatures = ["interactiveTransactions"] } generator pothos { @@ -25,15 +26,17 @@ model Dashboard { model History { id String + idref String? + question Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict) title String url String platform String description String options Json - timestamp DateTime @db.Timestamp(6) + timestamp DateTime @db.Timestamp(6) qualityindicators Json extra Json - pk Int @id @default(autoincrement()) + pk Int @id @default(autoincrement()) @@index([id]) @@map("history") @@ -75,6 +78,8 @@ model Question { extra Json onFrontpage FrontpageId? + history History[] + @@map("questions") } diff --git a/src/backend/flow/history/updateHistory.ts b/src/backend/flow/history/updateHistory.ts index 88aea20..8b56de3 100644 --- a/src/backend/flow/history/updateHistory.ts +++ b/src/backend/flow/history/updateHistory.ts @@ -3,6 +3,9 @@ import { prisma } from "../../database/prisma"; export async function updateHistory() { const questions = await prisma.question.findMany({}); await prisma.history.createMany({ - data: questions, + data: questions.map((q) => ({ + ...q, + idref: q.id, + })), }); } diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index dc26bd7..2f6e6f6 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -89,27 +89,69 @@ export const processPlatform = async (platform: Platform) => { console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`); return; } - const results = await platform.fetcher(); - if (results && results.length) { - await prisma.$transaction([ - prisma.question.deleteMany({ - where: { - platform: platform.name, - }, - }), - prisma.question.createMany({ - data: results.map((q) => ({ - extra: {}, - timestamp: new Date(), - ...q, - qualityindicators: q.qualityindicators as object, // fighting typescript - })), - }), - ]); - console.log("Done"); - } else { + const fetchedQuestions = await platform.fetcher(); + if (!fetchedQuestions || !fetchedQuestions.length) { console.log(`Platform ${platform.name} didn't return any results`); + return; } + + const prepareQuestion = (q: FetchedQuestion): Question => { + return { + extra: {}, + timestamp: new Date(), + ...q, + platform: platform.name, + qualityindicators: q.qualityindicators as object, // fighting typescript + }; + }; + + const oldQuestions = await prisma.question.findMany({ + where: { + platform: platform.name, + }, + }); + + const fetchedIds = fetchedQuestions.map((q) => q.id); + const oldIds = oldQuestions.map((q) => q.id); + + const fetchedIdsSet = new Set(fetchedIds); + const oldIdsSet = new Set(oldIds); + + const createdQuestions: Question[] = []; + const updatedQuestions: Question[] = []; + const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id)); + + for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) { + if (oldIdsSet.has(q.id)) { + updatedQuestions.push(q); + } else { + // TODO - check if question has changed for better performance + createdQuestions.push(q); + } + } + + await prisma.question.createMany({ + data: createdQuestions, + }); + + for (const q of updatedQuestions) { + await prisma.question.update({ + where: { id: q.id }, + data: q, + }); + } + + await prisma.question.deleteMany({ + where: { + id: { + in: deletedIds, + }, + }, + }); + + console.log( + `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created` + ); }; export interface PlatformConfig { diff --git a/src/backend/platforms/xrisk.ts b/src/backend/platforms/xrisk.ts index e4affaf..c769ae9 100644 --- a/src/backend/platforms/xrisk.ts +++ b/src/backend/platforms/xrisk.ts @@ -15,11 +15,15 @@ export const xrisk: Platform = { encoding: "utf-8", }); let results = JSON.parse(fileRaw); - results = results.map((item) => ({ - ...item, - id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique - platform: platformName, - })); + results = results.map((item) => { + item.extra = item.moreoriginsdata; + delete item.moreoriginsdata; + return { + ...item, + id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique + platform: platformName, + }; + }); return results; }, }; diff --git a/src/graphql/schema/questions.ts b/src/graphql/schema/questions.ts index 0d874c2..cb33bcd 100644 --- a/src/graphql/schema/questions.ts +++ b/src/graphql/schema/questions.ts @@ -1,3 +1,5 @@ +import { History, Question } from "@prisma/client"; + import { prisma } from "../../backend/database/prisma"; import { platforms, QualityIndicators } from "../../backend/platforms"; import { builder } from "../builder"; @@ -78,44 +80,67 @@ export const ProbabilityOptionObj = builder }), }); +const QuestionShapeInterface = builder + .interfaceRef("QuestionShape") + .implement({ + fields: (t) => ({ + title: t.exposeString("title"), + description: t.exposeString("description"), + url: t.exposeString("url", { + description: + "Non-unique, a very small number of platforms have a page for more than one prediction", + }), + platform: t.field({ + type: PlatformObj, + resolve: (parent) => parent.platform, + }), + timestamp: t.field({ + type: "Date", + description: "Timestamp at which metaforecast fetched the question", + resolve: (parent) => parent.timestamp, + }), + qualityIndicators: t.field({ + type: QualityIndicatorsObj, + resolve: (parent) => + parent.qualityindicators as any as QualityIndicators, + }), + options: t.field({ + type: [ProbabilityOptionObj], + resolve: ({ options }) => { + if (!Array.isArray(options)) { + return []; + } + return options as any[]; + }, + }), + }), + }); + +export const HistoryObj = builder.prismaObject("History", { + findUnique: (history) => ({ pk: history.pk }), + interfaces: [QuestionShapeInterface], + fields: (t) => ({ + id: t.exposeID("pk", { + description: "History items are identified by their integer ids", + }), + questionId: t.exposeID("id", { + description: "Unique string which identifies the question", + }), + }), +}); + export const QuestionObj = builder.prismaObject("Question", { findUnique: (question) => ({ id: question.id }), + interfaces: [QuestionShapeInterface], fields: (t) => ({ id: t.exposeID("id", { description: "Unique string which identifies the question", }), - title: t.exposeString("title"), - description: t.exposeString("description"), - url: t.exposeString("url", { - description: - "Non-unique, a very small number of platforms have a page for more than one prediction", - }), - timestamp: t.field({ - type: "Date", - description: "Timestamp at which metaforecast fetched the question", - resolve: (parent) => parent.timestamp, - }), - platform: t.field({ - type: PlatformObj, - resolve: (parent) => parent.platform, - }), - qualityIndicators: t.field({ - type: QualityIndicatorsObj, - resolve: (parent) => parent.qualityindicators as any as QualityIndicators, - }), - options: t.field({ - type: [ProbabilityOptionObj], - resolve: ({ options }) => { - if (!Array.isArray(options)) { - return []; - } - return options as any[]; - }, - }), visualization: t.string({ resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts nullable: true, }), + history: t.relation("history", {}), }), }); @@ -125,8 +150,7 @@ builder.queryField("questions", (t) => type: "Question", cursor: "id", maxSize: 1000, - resolve: (query, parent, args, context, info) => - prisma.question.findMany({ ...query }), + resolve: (query) => prisma.question.findMany({ ...query }), }, {}, {} diff --git a/src/web/display/DisplayQuestion/index.tsx b/src/web/display/DisplayQuestion/index.tsx index 3bb582f..b117207 100644 --- a/src/web/display/DisplayQuestion/index.tsx +++ b/src/web/display/DisplayQuestion/index.tsx @@ -146,14 +146,16 @@ export const DisplayQuestion: React.FC = ({ ) : null}
- - - - - + {process.env.NEXT_PUBLIC_ENABLE_QUESTION_PAGES ? ( + + + + + + ) : null}