import { Question } from "@prisma/client"; import { prisma } from "../database/prisma"; import { betfair } from "./betfair"; import { fantasyscotus } from "./fantasyscotus"; import { foretold } from "./foretold"; import { givewellopenphil } from "./givewellopenphil"; import { goodjudgment } from "./goodjudgment"; import { goodjudgmentopen } from "./goodjudgmentopen"; import { infer } from "./infer"; import { kalshi } from "./kalshi"; import { manifold } from "./manifold"; import { metaculus } from "./metaculus"; import { polymarket } from "./polymarket"; import { predictit } from "./predictit"; import { rootclaim } from "./rootclaim"; import { smarkets } from "./smarkets"; import { wildeford } from "./wildeford"; import { xrisk } from "./xrisk"; export interface QualityIndicators { stars: number; numforecasts?: number | string; numforecasters?: number; liquidity?: number | string; volume?: number; volume7Days?: number; volume24Hours?: number; address?: number; tradevolume?: string; pool?: any; createdTime?: any; shares_volume?: any; yes_bid?: any; yes_ask?: any; spread?: any; open_interest?: any; trade_volume?: any; } export type FetchedQuestion = Omit< Question, "extra" | "qualityindicators" | "timestamp" > & { timestamp?: Date; extra?: object; // required in DB but annoying to return empty; also this is slightly stricter than Prisma's JsonValue qualityindicators: QualityIndicators; // slightly stronger type than Prisma's JsonValue }; // fetcher should return null if platform failed to fetch questions for some reason export type PlatformFetcher = () => Promise; export interface Platform { name: string; // short name for ids and `platform` db column, e.g. "xrisk" label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates" color: string; // used on frontend fetcher?: PlatformFetcher; } // draft for the future callback-based streaming/chunking API: // interface FetchOptions { // since?: string; // some kind of cursor, Date object or opaque string? // save: (questions: Question[]) => Promise; // } // export type PlatformFetcher = (options: FetchOptions) => Promise; export const platforms: Platform[] = [ betfair, fantasyscotus, foretold, givewellopenphil, goodjudgment, goodjudgmentopen, infer, kalshi, manifold, metaculus, polymarket, predictit, rootclaim, smarkets, wildeford, xrisk, ]; export const processPlatform = async (platform: Platform) => { if (!platform.fetcher) { console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`); return; } const fetchedQuestions = await platform.fetcher(); if (!fetchedQuestions || !fetchedQuestions.length) { console.log(`Platform ${platform.name} didn't return any results`); return; } const prepareQuestion = (q: FetchedQuestion): Question => { return { extra: {}, timestamp: new Date(), ...q, platform: platform.name, qualityindicators: q.qualityindicators as object, // fighting typescript }; }; const oldQuestions = await prisma.question.findMany({ where: { platform: platform.name, }, }); const fetchedIds = fetchedQuestions.map((q) => q.id); const oldIds = oldQuestions.map((q) => q.id); const fetchedIdsSet = new Set(fetchedIds); const oldIdsSet = new Set(oldIds); const createdQuestions: Question[] = []; const updatedQuestions: Question[] = []; const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id)); for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) { if (oldIdsSet.has(q.id)) { updatedQuestions.push(q); } else { // TODO - check if question has changed for better performance createdQuestions.push(q); } } await prisma.question.createMany({ data: createdQuestions, }); for (const q of updatedQuestions) { await prisma.question.update({ where: { id: q.id }, data: q, }); } await prisma.question.deleteMany({ where: { id: { in: deletedIds, }, }, }); console.log( `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created` ); }; export interface PlatformConfig { name: string; label: string; color: string; } // get frontend-safe version of platforms data export const getPlatformsConfig = (options: { withGuesstimate: boolean; }): PlatformConfig[] => { const platformsConfig = platforms.map((platform) => ({ name: platform.name, label: platform.label, color: platform.color, })); if (options.withGuesstimate) { platformsConfig.push({ name: "guesstimate", label: "Guesstimate", color: "223900", }); } return platformsConfig; };