feat: insert questions without removing (slower, but fixes fks)

prisma/migrations/20220425220646_history_relation/migration.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "history" ADD COLUMN     "idref" TEXT;
+
+ALTER TABLE "history" ADD CONSTRAINT "history_idref_fkey" FOREIGN KEY ("idref") REFERENCES "questions"("id") ON DELETE SET NULL ON UPDATE RESTRICT;
+
+UPDATE "history" SET idref = id WHERE id in (SELECT id FROM "questions");

prisma/schema.prisma

@@ -1,5 +1,6 @@
 generator client {
-  provider = "prisma-client-js"
+  provider        = "prisma-client-js"
+  previewFeatures = ["interactiveTransactions"]
 }
 
 generator pothos {
@@ -25,15 +26,17 @@ model Dashboard {
 
 model History {
   id                String
+  idref             String?
+  question          Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict)
   title             String
   url               String
   platform          String
   description       String
   options           Json
-  timestamp         DateTime @db.Timestamp(6)
+  timestamp         DateTime  @db.Timestamp(6)
   qualityindicators Json
   extra             Json
-  pk                Int      @id @default(autoincrement())
+  pk                Int       @id @default(autoincrement())
 
   @@index([id])
   @@map("history")
@@ -75,6 +78,8 @@ model Question {
   extra             Json
 
   onFrontpage FrontpageId?
+  history     History[]
+
   @@map("questions")
 }
 

src/backend/flow/history/updateHistory.ts

@@ -3,6 +3,9 @@ import { prisma } from "../../database/prisma";
 export async function updateHistory() {
   const questions = await prisma.question.findMany({});
   await prisma.history.createMany({
-    data: questions,
+    data: questions.map((q) => ({
+      ...q,
+      idref: q.id,
+    })),
   });
 }

src/backend/platforms/index.ts

@@ -89,27 +89,69 @@ export const processPlatform = async (platform: Platform) => {
     console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`);
     return;
   }
-  const results = await platform.fetcher();
+  const fetchedQuestions = await platform.fetcher();
-  if (results && results.length) {
+  if (!fetchedQuestions || !fetchedQuestions.length) {
-    await prisma.$transaction([
-      prisma.question.deleteMany({
-        where: {
-          platform: platform.name,
-        },
-      }),
-      prisma.question.createMany({
-        data: results.map((q) => ({
-          extra: {},
-          timestamp: new Date(),
-          ...q,
-          qualityindicators: q.qualityindicators as object, // fighting typescript
-        })),
-      }),
-    ]);
-    console.log("Done");
-  } else {
     console.log(`Platform ${platform.name} didn't return any results`);
+    return;
   }
+
+  const prepareQuestion = (q: FetchedQuestion): Question => {
+    return {
+      extra: {},
+      timestamp: new Date(),
+      ...q,
+      platform: platform.name,
+      qualityindicators: q.qualityindicators as object, // fighting typescript
+    };
+  };
+
+  const oldQuestions = await prisma.question.findMany({
+    where: {
+      platform: platform.name,
+    },
+  });
+
+  const fetchedIds = fetchedQuestions.map((q) => q.id);
+  const oldIds = oldQuestions.map((q) => q.id);
+
+  const fetchedIdsSet = new Set(fetchedIds);
+  const oldIdsSet = new Set(oldIds);
+
+  const createdQuestions: Question[] = [];
+  const updatedQuestions: Question[] = [];
+  const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id));
+
+  for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) {
+    if (oldIdsSet.has(q.id)) {
+      updatedQuestions.push(q);
+    } else {
+      // TODO - check if question has changed for better performance
+      createdQuestions.push(q);
+    }
+  }
+
+  await prisma.question.createMany({
+    data: createdQuestions,
+  });
+
+  for (const q of updatedQuestions) {
+    await prisma.question.update({
+      where: { id: q.id },
+      data: q,
+    });
+  }
+
+  await prisma.question.deleteMany({
+    where: {
+      id: {
+        in: deletedIds,
+      },
+    },
+  });
+
+  console.log(
+    `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created`
+  );
 };
 
 export interface PlatformConfig {

src/backend/platforms/xrisk.ts

@@ -15,11 +15,15 @@ export const xrisk: Platform = {
       encoding: "utf-8",
     });
     let results = JSON.parse(fileRaw);
-    results = results.map((item) => ({
+    results = results.map((item) => {
-      ...item,
+      item.extra = item.moreoriginsdata;
-      id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
+      delete item.moreoriginsdata;
-      platform: platformName,
+      return {
-    }));
+        ...item,
+        id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
+        platform: platformName,
+      };
+    });
     return results;
   },
 };

src/graphql/schema/questions.ts

@@ -1,3 +1,5 @@
+import { History, Question } from "@prisma/client";
+
 import { prisma } from "../../backend/database/prisma";
 import { platforms, QualityIndicators } from "../../backend/platforms";
 import { builder } from "../builder";
@@ -78,44 +80,67 @@ export const ProbabilityOptionObj = builder
     }),
   });
 
+const QuestionShapeInterface = builder
+  .interfaceRef<Question | History>("QuestionShape")
+  .implement({
+    fields: (t) => ({
+      title: t.exposeString("title"),
+      description: t.exposeString("description"),
+      url: t.exposeString("url", {
+        description:
+          "Non-unique, a very small number of platforms have a page for more than one prediction",
+      }),
+      platform: t.field({
+        type: PlatformObj,
+        resolve: (parent) => parent.platform,
+      }),
+      timestamp: t.field({
+        type: "Date",
+        description: "Timestamp at which metaforecast fetched the question",
+        resolve: (parent) => parent.timestamp,
+      }),
+      qualityIndicators: t.field({
+        type: QualityIndicatorsObj,
+        resolve: (parent) =>
+          parent.qualityindicators as any as QualityIndicators,
+      }),
+      options: t.field({
+        type: [ProbabilityOptionObj],
+        resolve: ({ options }) => {
+          if (!Array.isArray(options)) {
+            return [];
+          }
+          return options as any[];
+        },
+      }),
+    }),
+  });
+
+export const HistoryObj = builder.prismaObject("History", {
+  findUnique: (history) => ({ pk: history.pk }),
+  interfaces: [QuestionShapeInterface],
+  fields: (t) => ({
+    id: t.exposeID("pk", {
+      description: "History items are identified by their integer ids",
+    }),
+    questionId: t.exposeID("id", {
+      description: "Unique string which identifies the question",
+    }),
+  }),
+});
+
 export const QuestionObj = builder.prismaObject("Question", {
   findUnique: (question) => ({ id: question.id }),
+  interfaces: [QuestionShapeInterface],
   fields: (t) => ({
     id: t.exposeID("id", {
       description: "Unique string which identifies the question",
     }),
-    title: t.exposeString("title"),
-    description: t.exposeString("description"),
-    url: t.exposeString("url", {
-      description:
-        "Non-unique, a very small number of platforms have a page for more than one prediction",
-    }),
-    timestamp: t.field({
-      type: "Date",
-      description: "Timestamp at which metaforecast fetched the question",
-      resolve: (parent) => parent.timestamp,
-    }),
-    platform: t.field({
-      type: PlatformObj,
-      resolve: (parent) => parent.platform,
-    }),
-    qualityIndicators: t.field({
-      type: QualityIndicatorsObj,
-      resolve: (parent) => parent.qualityindicators as any as QualityIndicators,
-    }),
-    options: t.field({
-      type: [ProbabilityOptionObj],
-      resolve: ({ options }) => {
-        if (!Array.isArray(options)) {
-          return [];
-        }
-        return options as any[];
-      },
-    }),
     visualization: t.string({
       resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts
       nullable: true,
     }),
+    history: t.relation("history", {}),
   }),
 });
 
@@ -125,8 +150,7 @@ builder.queryField("questions", (t) =>
       type: "Question",
       cursor: "id",
       maxSize: 1000,
-      resolve: (query, parent, args, context, info) =>
+      resolve: (query) => prisma.question.findMany({ ...query }),
-        prisma.question.findMany({ ...query }),
     },
     {},
     {}

src/web/display/DisplayQuestion/index.tsx

@@ -146,14 +146,16 @@ export const DisplayQuestion: React.FC<Props> = ({
             </div>
           ) : null}
           <div>
-            <Link href={`/questions/${question.id}`} passHref>
+            {process.env.NEXT_PUBLIC_ENABLE_QUESTION_PAGES ? (
-              <a className="float-right block ml-2 mt-1.5">
+              <Link href={`/questions/${question.id}`} passHref>
-                <FaExpand
+                <a className="float-right block ml-2 mt-1.5">
-                  size="18"
+                  <FaExpand
-                  className="text-gray-400 hover:text-gray-700"
+                    size="18"
-                />
+                    className="text-gray-400 hover:text-gray-700"
-              </a>
+                  />
-            </Link>
+                </a>
+              </Link>
+            ) : null}
             <Card.Title>
               <a
                 className="text-black no-underline"