feat: insert questions without removing (slower, but fixes fks)

prisma/migrations/20220425220646_history_relation/migration.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "history" ADD COLUMN     "idref" TEXT;
+
+ALTER TABLE "history" ADD CONSTRAINT "history_idref_fkey" FOREIGN KEY ("idref") REFERENCES "questions"("id") ON DELETE SET NULL ON UPDATE RESTRICT;
+
+UPDATE "history" SET idref = id WHERE id in (SELECT id FROM "questions");

prisma/schema.prisma

@@ -1,5 +1,6 @@
 generator client {
   provider        = "prisma-client-js"
+  previewFeatures = ["interactiveTransactions"]
 }
 
 generator pothos {
@@ -25,6 +26,8 @@ model Dashboard {
 
 model History {
   id                String
+  idref             String?
+  question          Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict)
   title             String
   url               String
   platform          String
@@ -75,6 +78,8 @@ model Question {
   extra             Json
 
   onFrontpage FrontpageId?
+  history     History[]
+
   @@map("questions")
 }
 

src/backend/flow/history/updateHistory.ts

@@ -3,6 +3,9 @@ import { prisma } from "../../database/prisma";
 export async function updateHistory() {
   const questions = await prisma.question.findMany({});
   await prisma.history.createMany({
-    data: questions,
+    data: questions.map((q) => ({
+      ...q,
+      idref: q.id,
+    })),
   });
 }

src/backend/platforms/index.ts

@@ -89,27 +89,69 @@ export const processPlatform = async (platform: Platform) => {
     console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`);
     return;
   }
-  const results = await platform.fetcher();
+  const fetchedQuestions = await platform.fetcher();
-  if (results && results.length) {
+  if (!fetchedQuestions || !fetchedQuestions.length) {
-    await prisma.$transaction([
+    console.log(`Platform ${platform.name} didn't return any results`);
-      prisma.question.deleteMany({
+    return;
-        where: {
+  }
-          platform: platform.name,
+
-        },
+  const prepareQuestion = (q: FetchedQuestion): Question => {
-      }),
+    return {
-      prisma.question.createMany({
-        data: results.map((q) => ({
       extra: {},
       timestamp: new Date(),
       ...q,
+      platform: platform.name,
       qualityindicators: q.qualityindicators as object, // fighting typescript
-        })),
+    };
-      }),
+  };
-    ]);
+
-    console.log("Done");
+  const oldQuestions = await prisma.question.findMany({
+    where: {
+      platform: platform.name,
+    },
+  });
+
+  const fetchedIds = fetchedQuestions.map((q) => q.id);
+  const oldIds = oldQuestions.map((q) => q.id);
+
+  const fetchedIdsSet = new Set(fetchedIds);
+  const oldIdsSet = new Set(oldIds);
+
+  const createdQuestions: Question[] = [];
+  const updatedQuestions: Question[] = [];
+  const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id));
+
+  for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) {
+    if (oldIdsSet.has(q.id)) {
+      updatedQuestions.push(q);
     } else {
-    console.log(`Platform ${platform.name} didn't return any results`);
+      // TODO - check if question has changed for better performance
+      createdQuestions.push(q);
     }
+  }
+
+  await prisma.question.createMany({
+    data: createdQuestions,
+  });
+
+  for (const q of updatedQuestions) {
+    await prisma.question.update({
+      where: { id: q.id },
+      data: q,
+    });
+  }
+
+  await prisma.question.deleteMany({
+    where: {
+      id: {
+        in: deletedIds,
+      },
+    },
+  });
+
+  console.log(
+    `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created`
+  );
 };
 
 export interface PlatformConfig {

src/backend/platforms/xrisk.ts

@@ -15,11 +15,15 @@ export const xrisk: Platform = {
       encoding: "utf-8",
     });
     let results = JSON.parse(fileRaw);
-    results = results.map((item) => ({
+    results = results.map((item) => {
+      item.extra = item.moreoriginsdata;
+      delete item.moreoriginsdata;
+      return {
         ...item,
         id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
         platform: platformName,
-    }));
+      };
+    });
     return results;
   },
 };

src/graphql/schema/questions.ts

@@ -1,3 +1,5 @@
+import { History, Question } from "@prisma/client";
+
 import { prisma } from "../../backend/database/prisma";
 import { platforms, QualityIndicators } from "../../backend/platforms";
 import { builder } from "../builder";
@@ -78,30 +80,29 @@ export const ProbabilityOptionObj = builder
     }),
   });
 
-export const QuestionObj = builder.prismaObject("Question", {
+const QuestionShapeInterface = builder
-  findUnique: (question) => ({ id: question.id }),
+  .interfaceRef<Question | History>("QuestionShape")
+  .implement({
     fields: (t) => ({
-    id: t.exposeID("id", {
-      description: "Unique string which identifies the question",
-    }),
       title: t.exposeString("title"),
       description: t.exposeString("description"),
       url: t.exposeString("url", {
         description:
           "Non-unique, a very small number of platforms have a page for more than one prediction",
       }),
+      platform: t.field({
+        type: PlatformObj,
+        resolve: (parent) => parent.platform,
+      }),
       timestamp: t.field({
         type: "Date",
         description: "Timestamp at which metaforecast fetched the question",
         resolve: (parent) => parent.timestamp,
       }),
-    platform: t.field({
-      type: PlatformObj,
-      resolve: (parent) => parent.platform,
-    }),
       qualityIndicators: t.field({
         type: QualityIndicatorsObj,
-      resolve: (parent) => parent.qualityindicators as any as QualityIndicators,
+        resolve: (parent) =>
+          parent.qualityindicators as any as QualityIndicators,
       }),
       options: t.field({
         type: [ProbabilityOptionObj],
@@ -112,10 +113,34 @@ export const QuestionObj = builder.prismaObject("Question", {
           return options as any[];
         },
       }),
+    }),
+  });
+
+export const HistoryObj = builder.prismaObject("History", {
+  findUnique: (history) => ({ pk: history.pk }),
+  interfaces: [QuestionShapeInterface],
+  fields: (t) => ({
+    id: t.exposeID("pk", {
+      description: "History items are identified by their integer ids",
+    }),
+    questionId: t.exposeID("id", {
+      description: "Unique string which identifies the question",
+    }),
+  }),
+});
+
+export const QuestionObj = builder.prismaObject("Question", {
+  findUnique: (question) => ({ id: question.id }),
+  interfaces: [QuestionShapeInterface],
+  fields: (t) => ({
+    id: t.exposeID("id", {
+      description: "Unique string which identifies the question",
+    }),
     visualization: t.string({
       resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts
       nullable: true,
     }),
+    history: t.relation("history", {}),
   }),
 });
 
@@ -125,8 +150,7 @@ builder.queryField("questions", (t) =>
       type: "Question",
       cursor: "id",
       maxSize: 1000,
-      resolve: (query, parent, args, context, info) =>
+      resolve: (query) => prisma.question.findMany({ ...query }),
-        prisma.question.findMany({ ...query }),
     },
     {},
     {}

src/web/display/DisplayQuestion/index.tsx

@@ -146,6 +146,7 @@ export const DisplayQuestion: React.FC<Props> = ({
             </div>
           ) : null}
           <div>
+            {process.env.NEXT_PUBLIC_ENABLE_QUESTION_PAGES ? (
               <Link href={`/questions/${question.id}`} passHref>
                 <a className="float-right block ml-2 mt-1.5">
                   <FaExpand
@@ -154,6 +155,7 @@ export const DisplayQuestion: React.FC<Props> = ({
                   />
                 </a>
               </Link>
+            ) : null}
             <Card.Title>
               <a
                 className="text-black no-underline"