feat: insert questions without removing (slower, but fixes fks)

This commit is contained in:
Vyacheslav Matyukhin 2022-04-27 22:07:00 +04:00
parent 73a47d94c3
commit 185464c0ae
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
7 changed files with 151 additions and 66 deletions

View File

@ -0,0 +1,5 @@
ALTER TABLE "history" ADD COLUMN "idref" TEXT;
ALTER TABLE "history" ADD CONSTRAINT "history_idref_fkey" FOREIGN KEY ("idref") REFERENCES "questions"("id") ON DELETE SET NULL ON UPDATE RESTRICT;
UPDATE "history" SET idref = id WHERE id in (SELECT id FROM "questions");

View File

@ -1,5 +1,6 @@
generator client {
provider = "prisma-client-js"
provider = "prisma-client-js"
previewFeatures = ["interactiveTransactions"]
}
generator pothos {
@ -25,15 +26,17 @@ model Dashboard {
model History {
id String
idref String?
question Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict)
title String
url String
platform String
description String
options Json
timestamp DateTime @db.Timestamp(6)
timestamp DateTime @db.Timestamp(6)
qualityindicators Json
extra Json
pk Int @id @default(autoincrement())
pk Int @id @default(autoincrement())
@@index([id])
@@map("history")
@ -75,6 +78,8 @@ model Question {
extra Json
onFrontpage FrontpageId?
history History[]
@@map("questions")
}

View File

@ -3,6 +3,9 @@ import { prisma } from "../../database/prisma";
export async function updateHistory() {
const questions = await prisma.question.findMany({});
await prisma.history.createMany({
data: questions,
data: questions.map((q) => ({
...q,
idref: q.id,
})),
});
}

View File

@ -89,27 +89,69 @@ export const processPlatform = async (platform: Platform) => {
console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`);
return;
}
const results = await platform.fetcher();
if (results && results.length) {
await prisma.$transaction([
prisma.question.deleteMany({
where: {
platform: platform.name,
},
}),
prisma.question.createMany({
data: results.map((q) => ({
extra: {},
timestamp: new Date(),
...q,
qualityindicators: q.qualityindicators as object, // fighting typescript
})),
}),
]);
console.log("Done");
} else {
const fetchedQuestions = await platform.fetcher();
if (!fetchedQuestions || !fetchedQuestions.length) {
console.log(`Platform ${platform.name} didn't return any results`);
return;
}
const prepareQuestion = (q: FetchedQuestion): Question => {
return {
extra: {},
timestamp: new Date(),
...q,
platform: platform.name,
qualityindicators: q.qualityindicators as object, // fighting typescript
};
};
const oldQuestions = await prisma.question.findMany({
where: {
platform: platform.name,
},
});
const fetchedIds = fetchedQuestions.map((q) => q.id);
const oldIds = oldQuestions.map((q) => q.id);
const fetchedIdsSet = new Set(fetchedIds);
const oldIdsSet = new Set(oldIds);
const createdQuestions: Question[] = [];
const updatedQuestions: Question[] = [];
const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id));
for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) {
if (oldIdsSet.has(q.id)) {
updatedQuestions.push(q);
} else {
// TODO - check if question has changed for better performance
createdQuestions.push(q);
}
}
await prisma.question.createMany({
data: createdQuestions,
});
for (const q of updatedQuestions) {
await prisma.question.update({
where: { id: q.id },
data: q,
});
}
await prisma.question.deleteMany({
where: {
id: {
in: deletedIds,
},
},
});
console.log(
`Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created`
);
};
export interface PlatformConfig {

View File

@ -15,11 +15,15 @@ export const xrisk: Platform = {
encoding: "utf-8",
});
let results = JSON.parse(fileRaw);
results = results.map((item) => ({
...item,
id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
platform: platformName,
}));
results = results.map((item) => {
item.extra = item.moreoriginsdata;
delete item.moreoriginsdata;
return {
...item,
id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
platform: platformName,
};
});
return results;
},
};

View File

@ -1,3 +1,5 @@
import { History, Question } from "@prisma/client";
import { prisma } from "../../backend/database/prisma";
import { platforms, QualityIndicators } from "../../backend/platforms";
import { builder } from "../builder";
@ -78,44 +80,67 @@ export const ProbabilityOptionObj = builder
}),
});
const QuestionShapeInterface = builder
.interfaceRef<Question | History>("QuestionShape")
.implement({
fields: (t) => ({
title: t.exposeString("title"),
description: t.exposeString("description"),
url: t.exposeString("url", {
description:
"Non-unique, a very small number of platforms have a page for more than one prediction",
}),
platform: t.field({
type: PlatformObj,
resolve: (parent) => parent.platform,
}),
timestamp: t.field({
type: "Date",
description: "Timestamp at which metaforecast fetched the question",
resolve: (parent) => parent.timestamp,
}),
qualityIndicators: t.field({
type: QualityIndicatorsObj,
resolve: (parent) =>
parent.qualityindicators as any as QualityIndicators,
}),
options: t.field({
type: [ProbabilityOptionObj],
resolve: ({ options }) => {
if (!Array.isArray(options)) {
return [];
}
return options as any[];
},
}),
}),
});
export const HistoryObj = builder.prismaObject("History", {
findUnique: (history) => ({ pk: history.pk }),
interfaces: [QuestionShapeInterface],
fields: (t) => ({
id: t.exposeID("pk", {
description: "History items are identified by their integer ids",
}),
questionId: t.exposeID("id", {
description: "Unique string which identifies the question",
}),
}),
});
export const QuestionObj = builder.prismaObject("Question", {
findUnique: (question) => ({ id: question.id }),
interfaces: [QuestionShapeInterface],
fields: (t) => ({
id: t.exposeID("id", {
description: "Unique string which identifies the question",
}),
title: t.exposeString("title"),
description: t.exposeString("description"),
url: t.exposeString("url", {
description:
"Non-unique, a very small number of platforms have a page for more than one prediction",
}),
timestamp: t.field({
type: "Date",
description: "Timestamp at which metaforecast fetched the question",
resolve: (parent) => parent.timestamp,
}),
platform: t.field({
type: PlatformObj,
resolve: (parent) => parent.platform,
}),
qualityIndicators: t.field({
type: QualityIndicatorsObj,
resolve: (parent) => parent.qualityindicators as any as QualityIndicators,
}),
options: t.field({
type: [ProbabilityOptionObj],
resolve: ({ options }) => {
if (!Array.isArray(options)) {
return [];
}
return options as any[];
},
}),
visualization: t.string({
resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts
nullable: true,
}),
history: t.relation("history", {}),
}),
});
@ -125,8 +150,7 @@ builder.queryField("questions", (t) =>
type: "Question",
cursor: "id",
maxSize: 1000,
resolve: (query, parent, args, context, info) =>
prisma.question.findMany({ ...query }),
resolve: (query) => prisma.question.findMany({ ...query }),
},
{},
{}

View File

@ -146,14 +146,16 @@ export const DisplayQuestion: React.FC<Props> = ({
</div>
) : null}
<div>
<Link href={`/questions/${question.id}`} passHref>
<a className="float-right block ml-2 mt-1.5">
<FaExpand
size="18"
className="text-gray-400 hover:text-gray-700"
/>
</a>
</Link>
{process.env.NEXT_PUBLIC_ENABLE_QUESTION_PAGES ? (
<Link href={`/questions/${question.id}`} passHref>
<a className="float-right block ml-2 mt-1.5">
<FaExpand
size="18"
className="text-gray-400 hover:text-gray-700"
/>
</a>
</Link>
) : null}
<Card.Title>
<a
className="text-black no-underline"