From 5fdea80f8ba8b22a45c7d599831473d8f3406a0b Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Sat, 21 May 2022 00:59:33 +0400 Subject: [PATCH 1/3] feat: firstSeen, new questions in graphql --- .../migration.sql | 28 ++++++++++ .../20220520195517_indices/migration.sql | 14 +++++ prisma/schema.prisma | 46 +++++++++------- src/backend/flow/doEverything.ts | 7 ++- src/backend/flow/jobs.ts | 4 +- src/backend/frontpage.ts | 2 +- src/backend/platforms/givewellopenphil.ts | 2 +- src/backend/platforms/guesstimate.ts | 21 ++++---- src/backend/platforms/index.ts | 43 ++++++++++++--- src/backend/platforms/registry.ts | 53 ++++++++++++------- src/backend/platforms/smarkets.ts | 1 - src/backend/platforms/wildeford.ts | 3 +- src/backend/utils/algolia.ts | 44 ++++++++------- src/graphql/schema/platforms.ts | 10 ++-- src/graphql/schema/questions.ts | 35 ++++++++++-- src/graphql/schema/search.ts | 10 +++- src/pages/index.tsx | 4 +- src/pages/secretEmbed.tsx | 4 +- src/web/worker/searchWithAlgolia.ts | 9 ++-- 19 files changed, 240 insertions(+), 100 deletions(-) create mode 100644 prisma/migrations/20220519104640_new_timestamps/migration.sql create mode 100644 prisma/migrations/20220520195517_indices/migration.sql diff --git a/prisma/migrations/20220519104640_new_timestamps/migration.sql b/prisma/migrations/20220519104640_new_timestamps/migration.sql new file mode 100644 index 0000000..ab404ff --- /dev/null +++ b/prisma/migrations/20220519104640_new_timestamps/migration.sql @@ -0,0 +1,28 @@ +-- questions +ALTER TABLE "questions" + ADD COLUMN "fetched" TIMESTAMP(6), + ADD COLUMN "first_seen" TIMESTAMP(6); + +UPDATE "questions" + SET "fetched" = "timestamp", "first_seen" = "timestamp"; + +ALTER TABLE "questions" + ALTER COLUMN "fetched" SET NOT NULL, + ALTER COLUMN "first_seen" SET NOT NULL; + +-- history +ALTER TABLE "history" + ADD COLUMN "fetched" TIMESTAMP(6); + +UPDATE "history" SET "fetched" = "timestamp"; + +ALTER TABLE "history" + ALTER COLUMN "fetched" SET NOT NULL; + +-- populate first_seen +UPDATE questions + SET "first_seen" = h.fs + FROM ( + SELECT id, MIN(fetched) AS fs FROM history GROUP BY id + ) as h + WHERE questions.id = h.id; diff --git a/prisma/migrations/20220520195517_indices/migration.sql b/prisma/migrations/20220520195517_indices/migration.sql new file mode 100644 index 0000000..761f541 --- /dev/null +++ b/prisma/migrations/20220520195517_indices/migration.sql @@ -0,0 +1,14 @@ +-- CreateIndex +CREATE INDEX "history_platform_idx" ON "history"("platform"); + +-- CreateIndex +CREATE INDEX "history_fetched_idx" ON "history"("fetched"); + +-- CreateIndex +CREATE INDEX "questions_platform_idx" ON "questions"("platform"); + +-- CreateIndex +CREATE INDEX "questions_fetched_idx" ON "questions"("fetched"); + +-- CreateIndex +CREATE INDEX "questions_first_seen_idx" ON "questions"("first_seen"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0cc5a28..8020bbe 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -24,24 +24,6 @@ model Dashboard { @@map("dashboards") } -model History { - id String - idref String? - question Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict) - title String - url String - platform String - description String - options Json - timestamp DateTime @db.Timestamp(6) - qualityindicators Json - extra Json - pk Int @id @default(autoincrement()) - - @@index([id]) - @@map("history") -} - model Question { /// E.g. "fantasyscotus-580" id String @id @@ -68,7 +50,9 @@ model Question { // } // ] options Json - timestamp DateTime @db.Timestamp(6) + timestamp DateTime @db.Timestamp(6) // deprecated + fetched DateTime @db.Timestamp(6) + firstSeen DateTime @map("first_seen") @db.Timestamp(6) // { // "numforecasts": 120, @@ -80,9 +64,33 @@ model Question { onFrontpage FrontpageId? history History[] + @@index([platform]) + @@index([fetched]) + @@index([firstSeen]) @@map("questions") } +model History { + id String + idref String? + question Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict) + title String + url String + platform String + description String + options Json + timestamp DateTime @db.Timestamp(6) // deprecated + fetched DateTime @db.Timestamp(6) + qualityindicators Json + extra Json + pk Int @id @default(autoincrement()) + + @@index([id]) + @@index([platform]) + @@index([fetched]) + @@map("history") +} + model FrontpageId { question Question @relation(fields: [id], references: [id]) id String @unique diff --git a/src/backend/flow/doEverything.ts b/src/backend/flow/doEverything.ts index 8e2da5d..69e5097 100644 --- a/src/backend/flow/doEverything.ts +++ b/src/backend/flow/doEverything.ts @@ -1,9 +1,12 @@ -import { platforms } from "../platforms/registry"; +import { getPlatforms } from "../platforms/registry"; import { executeJobByName } from "./jobs"; /* Do everything */ export async function doEverything() { - let jobNames = [...platforms.map((platform) => platform.name), "algolia"]; + let jobNames = [ + ...getPlatforms().map((platform) => platform.name), + "algolia", + ]; console.log(""); console.log(""); diff --git a/src/backend/flow/jobs.ts b/src/backend/flow/jobs.ts index 61a3d2b..41ed575 100644 --- a/src/backend/flow/jobs.ts +++ b/src/backend/flow/jobs.ts @@ -1,7 +1,7 @@ import { doEverything } from "../flow/doEverything"; import { rebuildFrontpage } from "../frontpage"; import { processPlatform } from "../platforms"; -import { platforms } from "../platforms/registry"; +import { getPlatforms } from "../platforms/registry"; import { rebuildAlgoliaDatabase } from "../utils/algolia"; import { sleep } from "../utils/sleep"; @@ -14,7 +14,7 @@ interface Job { } export const jobs: Job[] = [ - ...platforms.map((platform) => ({ + ...getPlatforms().map((platform) => ({ name: platform.name, message: `Download predictions from ${platform.name}`, ...(platform.version === "v2" ? { args: platform.fetcherArgs } : {}), diff --git a/src/backend/frontpage.ts b/src/backend/frontpage.ts index 4965cd9..3220dfa 100644 --- a/src/backend/frontpage.ts +++ b/src/backend/frontpage.ts @@ -26,7 +26,7 @@ export async function rebuildFrontpage() { AND questions.description != '' AND JSONB_ARRAY_LENGTH(questions.options) > 0 GROUP BY questions.id - HAVING COUNT(DISTINCT history.timestamp) >= 7 + HAVING COUNT(DISTINCT history.fetched) >= 7 ORDER BY RANDOM() LIMIT 50 `; diff --git a/src/backend/platforms/givewellopenphil.ts b/src/backend/platforms/givewellopenphil.ts index b3e667e..7be7dd2 100644 --- a/src/backend/platforms/givewellopenphil.ts +++ b/src/backend/platforms/givewellopenphil.ts @@ -79,7 +79,7 @@ export const givewellopenphil: Platform = { const dataWithDate = data.map((datum: any) => ({ ...datum, platform: platformName, - timestamp: new Date("2021-02-23"), + // timestamp: new Date("2021-02-23"), })); return dataWithDate; }, diff --git a/src/backend/platforms/guesstimate.ts b/src/backend/platforms/guesstimate.ts index becb351..6deda58 100644 --- a/src/backend/platforms/guesstimate.ts +++ b/src/backend/platforms/guesstimate.ts @@ -2,9 +2,8 @@ import axios from "axios"; import { Question } from "@prisma/client"; -import { prisma } from "../database/prisma"; -import { AlgoliaQuestion } from "../utils/algolia"; -import { FetchedQuestion, Platform, prepareQuestion } from "./"; +import { AlgoliaQuestion, questionToAlgoliaQuestion } from "../utils/algolia"; +import { FetchedQuestion, Platform, prepareQuestion, upsertSingleQuestion } from "./"; /* Definitions */ const searchEndpoint = @@ -55,10 +54,12 @@ async function search(query: string): Promise { const models: any[] = response.data.hits; const mappedModels: AlgoliaQuestion[] = models.map((model) => { const q = modelToQuestion(model); - return { + return questionToAlgoliaQuestion({ ...q, - timestamp: String(q.timestamp), - }; + fetched: new Date(), + timestamp: new Date(), + firstSeen: new Date(), + }); }); // filter for duplicates. Surprisingly common. @@ -76,12 +77,8 @@ async function search(query: string): Promise { const fetchQuestion = async (id: number): Promise => { const response = await axios({ url: `${apiEndpoint}/spaces/${id}` }); - let q = modelToQuestion(response.data); - return await prisma.question.upsert({ - where: { id: q.id }, - create: q, - update: q, - }); + const q = modelToQuestion(response.data); + return await upsertSingleQuestion(q); }; export const guesstimate: Platform & { diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index 384ab92..5473c34 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -28,9 +28,14 @@ export interface QualityIndicators { export type FetchedQuestion = Omit< Question, - "extra" | "qualityindicators" | "timestamp" | "platform" | "options" + | "extra" + | "qualityindicators" + | "fetched" + | "firstSeen" + | "timestamp" + | "platform" + | "options" > & { - timestamp?: Date; extra?: object; // required in DB but annoying to return empty; also this is slightly stricter than Prisma's JsonValue options: QuestionOption[]; // stronger type than Prisma's JsonValue qualityindicators: Omit; // slightly stronger type than Prisma's JsonValue @@ -78,8 +83,14 @@ export type Platform = { // So here we build a new type which should be ok to use both in place of prisma's Question type and as an input to its update or create methods. type PreparedQuestion = Omit< Question, - "extra" | "qualityindicators" | "options" + | "extra" + | "qualityindicators" + | "options" + | "fetched" + | "firstSeen" + | "timestamp" > & { + fetched: Date; extra: NonNullable; qualityindicators: NonNullable; options: NonNullable; @@ -91,8 +102,8 @@ export const prepareQuestion = ( ): PreparedQuestion => { return { extra: {}, - timestamp: new Date(), ...q, + fetched: new Date(), platform: platform.name, qualityindicators: { ...q.qualityindicators, @@ -101,6 +112,21 @@ export const prepareQuestion = ( }; }; +export const upsertSingleQuestion = async ( + q: PreparedQuestion +): Promise => { + return await prisma.question.upsert({ + where: { id: q.id }, + create: { + ...q, + firstSeen: new Date(), + timestamp: q.fetched, // deprecated + }, + update: q, + }); + // TODO - update history? +}; + export const processPlatform = async ( platform: Platform, args?: { [k in T]: string } @@ -144,9 +170,9 @@ export const processPlatform = async ( for (const q of fetchedQuestions.map((q) => prepareQuestion(q, platform))) { if (oldIdsSet.has(q.id)) { + // TODO - check if question has changed for better performance updatedQuestions.push(q); } else { - // TODO - check if question has changed for better performance createdQuestions.push(q); } } @@ -154,7 +180,11 @@ export const processPlatform = async ( const stats: { created?: number; updated?: number; deleted?: number } = {}; await prisma.question.createMany({ - data: createdQuestions, + data: createdQuestions.map((q) => ({ + ...q, + firstSeen: new Date(), + timestamp: q.fetched, // deprecated + })), }); stats.created = createdQuestions.length; @@ -181,6 +211,7 @@ export const processPlatform = async ( await prisma.history.createMany({ data: [...createdQuestions, ...updatedQuestions].map((q) => ({ ...q, + timestamp: q.fetched, // deprecated idref: q.id, })), }); diff --git a/src/backend/platforms/registry.ts b/src/backend/platforms/registry.ts index ed6d902..ce022a6 100644 --- a/src/backend/platforms/registry.ts +++ b/src/backend/platforms/registry.ts @@ -17,30 +17,43 @@ import { smarkets } from "./smarkets"; import { wildeford } from "./wildeford"; import { xrisk } from "./xrisk"; -export const platforms: Platform[] = [ - betfair, - fantasyscotus, - foretold, - givewellopenphil, - goodjudgment, - goodjudgmentopen, - guesstimate, - infer, - kalshi, - manifold, - metaculus, - polymarket, - predictit, - rootclaim, - smarkets, - wildeford, - xrisk, -]; +// function instead of const array, this helps to fight circular dependencies +export const getPlatforms = (): Platform[] => { + return [ + betfair, + fantasyscotus, + foretold, + givewellopenphil, + goodjudgment, + goodjudgmentopen, + guesstimate, + infer, + kalshi, + manifold, + metaculus, + polymarket, + predictit, + rootclaim, + smarkets, + wildeford, + xrisk, + ]; +}; + +let _nameToLabelCache: { [k: string]: string } | undefined; +export function platformNameToLabel(name: string): string { + if (!_nameToLabelCache) { + _nameToLabelCache = Object.fromEntries( + getPlatforms().map((platform) => [platform.name, platform.label]) + ); + } + return _nameToLabelCache[name] || name; +} // get frontend-safe version of platforms data export const getPlatformsConfig = (): PlatformConfig[] => { - const platformsConfig = platforms.map((platform) => ({ + const platformsConfig = getPlatforms().map((platform) => ({ name: platform.name, label: platform.label, color: platform.color, diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts index 6da9777..4442b14 100644 --- a/src/backend/platforms/smarkets.ts +++ b/src/backend/platforms/smarkets.ts @@ -162,7 +162,6 @@ async function processEventMarkets(event: any, ctx: Context) { url: "https://smarkets.com/event/" + market.event_id + market.slug, description: market.description, options, - timestamp: new Date(), qualityindicators: {}, extra: { contracts, diff --git a/src/backend/platforms/wildeford.ts b/src/backend/platforms/wildeford.ts index 30be453..95a9e10 100644 --- a/src/backend/platforms/wildeford.ts +++ b/src/backend/platforms/wildeford.ts @@ -96,7 +96,8 @@ async function processPredictions( url: prediction["url"], description: prediction["Notes"] || "", options, - timestamp: new Date(Date.parse(prediction["Prediction Date"] + "Z")), + //// TODO - use `created` field for this + // timestamp: new Date(Date.parse(prediction["Prediction Date"] + "Z")), qualityindicators: {}, }; return result; diff --git a/src/backend/utils/algolia.ts b/src/backend/utils/algolia.ts index f2cc38d..98f4c45 100644 --- a/src/backend/utils/algolia.ts +++ b/src/backend/utils/algolia.ts @@ -3,16 +3,23 @@ import algoliasearch from "algoliasearch"; import { Question } from "@prisma/client"; import { prisma } from "../database/prisma"; -import { platforms } from "../platforms/registry"; +import { platformNameToLabel } from "../platforms/registry"; -let cookie = process.env.ALGOLIA_MASTER_API_KEY || ""; +const cookie = process.env.ALGOLIA_MASTER_API_KEY || ""; const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || ""; const client = algoliasearch(algoliaAppId, cookie); const index = client.initIndex("metaforecast"); -export type AlgoliaQuestion = Omit & { - timestamp: string; +export type AlgoliaQuestion = Omit< + Question, + "fetched" | "firstSeen" | "timestamp" +> & { + timestamp?: string; // deprecated + fetched?: string; + firstSeen?: string; optionsstringforsearch?: string; + platformLabel: string; + objectID: string; }; const getoptionsstringforsearch = (record: Question): string => { @@ -26,23 +33,24 @@ const getoptionsstringforsearch = (record: Question): string => { return result; }; +export const questionToAlgoliaQuestion = ( + question: Question +): AlgoliaQuestion => { + return { + ...question, + fetched: question.fetched.toISOString(), + timestamp: question.timestamp.toISOString(), // deprecated + firstSeen: question.firstSeen.toISOString(), + platformLabel: platformNameToLabel(question.platform), + objectID: question.id, + optionsstringforsearch: getoptionsstringforsearch(question), + }; +}; + export async function rebuildAlgoliaDatabase() { const questions = await prisma.question.findMany(); - const platformNameToLabel = Object.fromEntries( - platforms.map((platform) => [platform.name, platform.label]) - ); - - const records: AlgoliaQuestion[] = questions.map( - (question, index: number) => ({ - ...question, - timestamp: `${question.timestamp}`, - platformLabel: - platformNameToLabel[question.platform] || question.platform, - objectID: index, - optionsstringforsearch: getoptionsstringforsearch(question), - }) - ); + const records: AlgoliaQuestion[] = questions.map(questionToAlgoliaQuestion); if (await index.exists()) { console.log("Index exists"); diff --git a/src/graphql/schema/platforms.ts b/src/graphql/schema/platforms.ts index 10dc2b4..8e339f3 100644 --- a/src/graphql/schema/platforms.ts +++ b/src/graphql/schema/platforms.ts @@ -1,5 +1,5 @@ import { prisma } from "../../backend/database/prisma"; -import { platforms } from "../../backend/platforms/registry"; +import { getPlatforms } from "../../backend/platforms/registry"; import { builder } from "../builder"; export const PlatformObj = builder.objectRef("Platform").implement({ @@ -20,7 +20,7 @@ export const PlatformObj = builder.objectRef("Platform").implement({ return "Guesstimate"; } // kinda slow and repetitive, TODO - store a map {name => platform} somewhere and `getPlatform` util function? - const platform = platforms.find((p) => p.name === platformName); + const platform = getPlatforms().find((p) => p.name === platformName); if (!platform) { throw new Error(`Unknown platform ${platformName}`); } @@ -36,10 +36,10 @@ export const PlatformObj = builder.objectRef("Platform").implement({ platform: platformName, }, _max: { - timestamp: true, + fetched: true, }, }); - return res._max.timestamp; + return res._max.fetched; }, }), }), @@ -49,7 +49,7 @@ builder.queryField("platforms", (t) => t.field({ type: [PlatformObj], resolve: async (parent, args) => { - return platforms.map((platform) => platform.name); + return getPlatforms().map((platform) => platform.name); }, }) ); diff --git a/src/graphql/schema/questions.ts b/src/graphql/schema/questions.ts index fb28bed..a0b676e 100644 --- a/src/graphql/schema/questions.ts +++ b/src/graphql/schema/questions.ts @@ -70,8 +70,16 @@ const QuestionShapeInterface = builder }), timestamp: t.field({ type: "Date", - description: "Timestamp at which metaforecast fetched the question", - resolve: (parent) => parent.timestamp, + description: + "Last timestamp at which metaforecast fetched the question", + deprecationReason: "Renamed to `fetched`", + resolve: (parent) => parent.fetched, + }), + fetched: t.field({ + type: "Date", + description: + "Last timestamp at which metaforecast fetched the question", + resolve: (parent) => parent.fetched, }), qualityIndicators: t.field({ type: QualityIndicatorsObj, @@ -114,10 +122,15 @@ export const QuestionObj = builder.prismaObject("Question", { resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts nullable: true, }), + firstSeen: t.field({ + type: "Date", + description: "First timestamp at which metaforecast fetched the question", + resolve: (parent) => parent.firstSeen, + }), history: t.relation("history", { query: () => ({ orderBy: { - timestamp: "asc", + fetched: "asc", }, }), }), @@ -130,7 +143,21 @@ builder.queryField("questions", (t) => type: "Question", cursor: "id", maxSize: 1000, - resolve: (query) => prisma.question.findMany({ ...query }), + args: { + orderBy: t.arg({ + type: builder.enumType("QuestionsOrderBy", { + values: ["FIRST_SEEN_DESC"] as const, + }), + }), + }, + resolve: (query, parent, args) => { + return prisma.question.findMany({ + ...query, + ...(args.orderBy === "FIRST_SEEN_DESC" + ? { orderBy: [{ firstSeen: "desc" }, { id: "asc" }] } + : {}), // TODO - explicit default order? + }); + }, }, {}, {} diff --git a/src/graphql/schema/search.ts b/src/graphql/schema/search.ts index 6039e21..6d31357 100644 --- a/src/graphql/schema/search.ts +++ b/src/graphql/schema/search.ts @@ -63,7 +63,15 @@ builder.queryField("searchQuestions", (t) => return results.map((q) => ({ ...q, - timestamp: new Date(q.timestamp), + fetched: new Date( + q.fetched || q.timestamp || new Date().toISOString() // q.timestamp is deprecated, TODO - just use `q.fetched` + ), + timestamp: new Date( + q.fetched || q.timestamp || new Date().toISOString() + ), + firstSeen: new Date( + q.firstSeen || new Date().toISOString() // TODO - q.firstSeen is not yet populated in algolia + ), })); }, }) diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 5814c91..85e6036 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -1,7 +1,7 @@ import { GetServerSideProps, NextPage } from "next"; import React from "react"; -import { getPlatformsConfig, platforms } from "../backend/platforms/registry"; +import { getPlatforms, getPlatformsConfig } from "../backend/platforms/registry"; import { Layout } from "../web/common/Layout"; import { Props, QueryParameters, SearchScreen } from "../web/search/components/SearchScreen"; import { FrontpageDocument, SearchDocument } from "../web/search/queries.generated"; @@ -19,7 +19,7 @@ export const getServerSideProps: GetServerSideProps = async ( query: "", starsThreshold: 2, forecastsThreshold: 0, - forecastingPlatforms: platforms.map((platform) => platform.name), + forecastingPlatforms: getPlatforms().map((platform) => platform.name), }; const initialQueryParameters: QueryParameters = { diff --git a/src/pages/secretEmbed.tsx b/src/pages/secretEmbed.tsx index 039af0d..d6477d5 100644 --- a/src/pages/secretEmbed.tsx +++ b/src/pages/secretEmbed.tsx @@ -3,7 +3,7 @@ import { GetServerSideProps, NextPage } from "next"; import React from "react"; -import { platforms } from "../backend/platforms/registry"; +import { getPlatforms } from "../backend/platforms/registry"; import { QuestionFragment } from "../web/fragments.generated"; import { QuestionCard } from "../web/questions/components/QuestionCard"; import { SearchDocument } from "../web/search/queries.generated"; @@ -23,7 +23,7 @@ export const getServerSideProps: GetServerSideProps = async ( query: "", starsThreshold: 2, forecastsThreshold: 0, - forecastingPlatforms: platforms.map((platform) => platform.name), + forecastingPlatforms: getPlatforms().map((platform) => platform.name), ...urlQuery, }; diff --git a/src/web/worker/searchWithAlgolia.ts b/src/web/worker/searchWithAlgolia.ts index 8f0d4ca..5f273f0 100644 --- a/src/web/worker/searchWithAlgolia.ts +++ b/src/web/worker/searchWithAlgolia.ts @@ -96,6 +96,7 @@ export default async function searchWithAlgolia({ title: "No search results match your query", url: "https://metaforecast.org", platform: "metaforecast", + platformLabel: "metaforecast", description: "Maybe try a broader query?", options: [ { @@ -109,7 +110,7 @@ export default async function searchWithAlgolia({ type: "PROBABILITY", }, ], - timestamp: `${new Date().toISOString().slice(0, 10)}`, + fetched: new Date().toISOString(), qualityindicators: { numforecasts: 1, numforecasters: 1, @@ -126,8 +127,10 @@ export default async function searchWithAlgolia({ title: `Did you mean: ${queryString}?`, url: "https://metaforecast.org/recursion?bypassEasterEgg=true", platform: "metaforecast", + platformLabel: "metaforecast", description: "Fatal error: Too much recursion. Click to proceed anyways", + fetched: new Date().toISOString(), options: [ { name: "Yes", @@ -140,7 +143,6 @@ export default async function searchWithAlgolia({ type: "PROBABILITY", }, ], - timestamp: `${new Date().toISOString().slice(0, 10)}`, qualityindicators: { numforecasts: 1, numforecasters: 1, @@ -161,6 +163,7 @@ export default async function searchWithAlgolia({ title: "No search results appear to match your query", url: "https://metaforecast.org", platform: "metaforecast", + platformLabel: "metaforecast", description: "Maybe try a broader query? That said, we could be wrong.", options: [ { @@ -174,7 +177,7 @@ export default async function searchWithAlgolia({ type: "PROBABILITY", }, ], - timestamp: `${new Date().toISOString().slice(0, 10)}`, + fetched: new Date().toISOString(), qualityindicators: { numforecasts: 1, numforecasters: 1, From 9c162fab9db181eeaee416f15d10c5067132021c Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 26 May 2022 13:47:54 +0400 Subject: [PATCH 2/3] feat: expose fetchedStr and firstSeenStr --- src/graphql/schema/questions.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/graphql/schema/questions.ts b/src/graphql/schema/questions.ts index a0b676e..5e4b095 100644 --- a/src/graphql/schema/questions.ts +++ b/src/graphql/schema/questions.ts @@ -81,6 +81,11 @@ const QuestionShapeInterface = builder "Last timestamp at which metaforecast fetched the question", resolve: (parent) => parent.fetched, }), + fetchedStr: t.string({ + description: + "Last timestamp at which metaforecast fetched the question, in ISO 8601 format", + resolve: (parent) => parent.fetched.toISOString(), + }), qualityIndicators: t.field({ type: QualityIndicatorsObj, resolve: (parent) => @@ -127,6 +132,11 @@ export const QuestionObj = builder.prismaObject("Question", { description: "First timestamp at which metaforecast fetched the question", resolve: (parent) => parent.firstSeen, }), + firstSeenStr: t.string({ + description: + "First timestamp at which metaforecast fetched the question, in ISO 8601 format", + resolve: (parent) => parent.firstSeen.toISOString(), + }), history: t.relation("history", { query: () => ({ orderBy: { From ab2efa8a88bd7d8aa131e4005470c37c6cc3100e Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 26 May 2022 14:14:49 +0400 Subject: [PATCH 3/3] feat: remove old timestamp field --- .../20220526101259_no_timestamp_field/migration.sql | 12 ++++++++++++ prisma/schema.prisma | 2 -- src/backend/platforms/guesstimate.ts | 1 - src/backend/platforms/index.ts | 11 +---------- src/backend/utils/algolia.ts | 11 +++-------- src/graphql/schema/search.ts | 11 ++--------- src/web/worker/searchWithAlgolia.ts | 3 +++ 7 files changed, 21 insertions(+), 30 deletions(-) create mode 100644 prisma/migrations/20220526101259_no_timestamp_field/migration.sql diff --git a/prisma/migrations/20220526101259_no_timestamp_field/migration.sql b/prisma/migrations/20220526101259_no_timestamp_field/migration.sql new file mode 100644 index 0000000..c80c710 --- /dev/null +++ b/prisma/migrations/20220526101259_no_timestamp_field/migration.sql @@ -0,0 +1,12 @@ +/* + Warnings: + + - You are about to drop the column `timestamp` on the `history` table. All the data in the column will be lost. + - You are about to drop the column `timestamp` on the `questions` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "history" DROP COLUMN "timestamp"; + +-- AlterTable +ALTER TABLE "questions" DROP COLUMN "timestamp"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8020bbe..bc983a7 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -50,7 +50,6 @@ model Question { // } // ] options Json - timestamp DateTime @db.Timestamp(6) // deprecated fetched DateTime @db.Timestamp(6) firstSeen DateTime @map("first_seen") @db.Timestamp(6) @@ -79,7 +78,6 @@ model History { platform String description String options Json - timestamp DateTime @db.Timestamp(6) // deprecated fetched DateTime @db.Timestamp(6) qualityindicators Json extra Json diff --git a/src/backend/platforms/guesstimate.ts b/src/backend/platforms/guesstimate.ts index 6deda58..c10a3dc 100644 --- a/src/backend/platforms/guesstimate.ts +++ b/src/backend/platforms/guesstimate.ts @@ -57,7 +57,6 @@ async function search(query: string): Promise { return questionToAlgoliaQuestion({ ...q, fetched: new Date(), - timestamp: new Date(), firstSeen: new Date(), }); }); diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index 5473c34..bc46403 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -32,7 +32,6 @@ export type FetchedQuestion = Omit< | "qualityindicators" | "fetched" | "firstSeen" - | "timestamp" | "platform" | "options" > & { @@ -83,12 +82,7 @@ export type Platform = { // So here we build a new type which should be ok to use both in place of prisma's Question type and as an input to its update or create methods. type PreparedQuestion = Omit< Question, - | "extra" - | "qualityindicators" - | "options" - | "fetched" - | "firstSeen" - | "timestamp" + "extra" | "qualityindicators" | "options" | "fetched" | "firstSeen" > & { fetched: Date; extra: NonNullable; @@ -120,7 +114,6 @@ export const upsertSingleQuestion = async ( create: { ...q, firstSeen: new Date(), - timestamp: q.fetched, // deprecated }, update: q, }); @@ -183,7 +176,6 @@ export const processPlatform = async ( data: createdQuestions.map((q) => ({ ...q, firstSeen: new Date(), - timestamp: q.fetched, // deprecated })), }); stats.created = createdQuestions.length; @@ -211,7 +203,6 @@ export const processPlatform = async ( await prisma.history.createMany({ data: [...createdQuestions, ...updatedQuestions].map((q) => ({ ...q, - timestamp: q.fetched, // deprecated idref: q.id, })), }); diff --git a/src/backend/utils/algolia.ts b/src/backend/utils/algolia.ts index 98f4c45..2c01454 100644 --- a/src/backend/utils/algolia.ts +++ b/src/backend/utils/algolia.ts @@ -10,13 +10,9 @@ const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || ""; const client = algoliasearch(algoliaAppId, cookie); const index = client.initIndex("metaforecast"); -export type AlgoliaQuestion = Omit< - Question, - "fetched" | "firstSeen" | "timestamp" -> & { - timestamp?: string; // deprecated - fetched?: string; - firstSeen?: string; +export type AlgoliaQuestion = Omit & { + fetched: string; + firstSeen: string; optionsstringforsearch?: string; platformLabel: string; objectID: string; @@ -39,7 +35,6 @@ export const questionToAlgoliaQuestion = ( return { ...question, fetched: question.fetched.toISOString(), - timestamp: question.timestamp.toISOString(), // deprecated firstSeen: question.firstSeen.toISOString(), platformLabel: platformNameToLabel(question.platform), objectID: question.id, diff --git a/src/graphql/schema/search.ts b/src/graphql/schema/search.ts index 6d31357..b587118 100644 --- a/src/graphql/schema/search.ts +++ b/src/graphql/schema/search.ts @@ -63,15 +63,8 @@ builder.queryField("searchQuestions", (t) => return results.map((q) => ({ ...q, - fetched: new Date( - q.fetched || q.timestamp || new Date().toISOString() // q.timestamp is deprecated, TODO - just use `q.fetched` - ), - timestamp: new Date( - q.fetched || q.timestamp || new Date().toISOString() - ), - firstSeen: new Date( - q.firstSeen || new Date().toISOString() // TODO - q.firstSeen is not yet populated in algolia - ), + fetched: new Date(q.fetched), + firstSeen: new Date(q.firstSeen), })); }, }) diff --git a/src/web/worker/searchWithAlgolia.ts b/src/web/worker/searchWithAlgolia.ts index 5f273f0..6798f5a 100644 --- a/src/web/worker/searchWithAlgolia.ts +++ b/src/web/worker/searchWithAlgolia.ts @@ -111,6 +111,7 @@ export default async function searchWithAlgolia({ }, ], fetched: new Date().toISOString(), + firstSeen: new Date().toISOString(), qualityindicators: { numforecasts: 1, numforecasters: 1, @@ -131,6 +132,7 @@ export default async function searchWithAlgolia({ description: "Fatal error: Too much recursion. Click to proceed anyways", fetched: new Date().toISOString(), + firstSeen: new Date().toISOString(), options: [ { name: "Yes", @@ -178,6 +180,7 @@ export default async function searchWithAlgolia({ }, ], fetched: new Date().toISOString(), + firstSeen: new Date().toISOString(), qualityindicators: { numforecasts: 1, numforecasters: 1,