Merge pull request #60 from quantified-uncertainty/graphql

GraphQL

codegen.yml

@@ -0,0 +1,32 @@
+schema: src/graphql/build-schema.js
+documents:
+  - "src/**/*.graphql"
+
+# This should be updated to match your client files
+# documents: 'client/**/!(*.d).{ts,tsx}'
+generates:
+  # This will take your schema and print an SDL schema.
+  schema.graphql:
+    plugins:
+      - schema-ast
+
+  src/graphql/types.generated.ts:
+    plugins:
+      - typescript
+
+  src/graphql/introspection.json:
+    plugins:
+      - introspection:
+          minify: true
+
+  src/:
+    preset: near-operation-file
+    presetConfig:
+      extension: .generated.tsx
+      baseTypesPath: graphql/types.generated.ts
+    plugins:
+      - typescript-operations:
+          strictScalars: true
+          scalars:
+            Date: number
+      - typed-document-node

docs/coding-style.md

@@ -13,7 +13,7 @@
 # React
 
 - create one file per one component (tiny helper components in the same file are fine)
-- name file identically to the component it describes (e.g. `const DisplayForecasts: React.FC<Props> = ...` in `DisplayForecasts.ts`)
+- name file identically to the component it describes (e.g. `const DisplayQuestions: React.FC<Props> = ...` in `DisplayQuestions.ts`)
 - use named export instead of default export for all React components
   - it's better for refactoring
   - and it plays well with `React.FC` typing

docs/graphql.md

@@ -0,0 +1,45 @@
+Metaforecast website is implemented on top of GraphQL API.
+
+Tech stack:
+
+- [Pothos](https://pothos-graphql.dev/) on the backend for implementing our graphql server
+- [urql](https://formidable.com/open-source/urql/) on the frontend
+- [GraphQL Code Generator](https://www.graphql-code-generator.com/) for schema generation, queries generation and schema introspection
+
+Rationale for this stack can be found on [#32](https://github.com/quantified-uncertainty/metaforecast/issues/32) and [#21](https://github.com/quantified-uncertainty/metaforecast/issues/32) in comments.
+
+# Code layout
+
+List of all files used for graphql:
+
+- [schema.graphql](../schema.graphql), GraphQL schema generated by graphql-code-generator.
+- [codegen.yml](../codegen.yml), graphql-code-generator [config](https://www.graphql-code-generator.com/docs/config-reference/codegen-config)
+- [graphql.config.yaml](../graphql.config.yaml), [GraphQL Config](https://www.graphql-config.com/) for better VS Code integration ([VS Code GraphQL extension](https://marketplace.visualstudio.com/items?itemName=GraphQL.vscode-graphql) type-checks graphql files thanks to it)
+- [src/pages/api/graphql.ts](../src/pages/api/graphql.ts) implements the GraphQL HTTP endpoint and GraphQL playground
+- [src/web/urql.ts](../src/web/urql.ts) contains some helper functions.
+
+[src/graphql/](../src/graphql) dir contains GraphQL server code.
+
+`*.graphql` files in `src/web/**` contain GraphQL fragments, queries and mutations which are used on the frontend.
+
+`graphql-code-generator` converts those into `*.generated.ts` files which can be imported from the React components.
+
+# Recipes
+
+**I want to check out what Metaforecast's GraphQL API is capable of**
+
+Go to [/api/graphql](https://metaforecast.org/api/graphql) and do some queries by hand. Note the "Docs" link in the top right corner.
+
+**I want to add a new query/mutation to our schema**
+
+Read the [Pothos](https://pothos-graphql.dev/) docs to learn how to implement new objects and fields. Add the new code somewhere in `src/graphql/schema/`.
+
+**I want to display a new piece of data on the metaforecast website**
+
+- add a query in a nearest `queries.graphql` file (if there isn't one, create it)
+- run `gql-gen` (or keep it running with `gql-gen -w`) to get a `graphql.generated.ts` file
+- use [useQuery](https://formidable.com/open-source/urql/docs/basics/react-preact/#queries) with your new `MyQueryDocument` which you can import from the `graphql.generated.ts` file
+
+If you need SSR, you'll also need to do **the same** query with **the same variables** from your page's `getServerSideProps` function. Check out any existing page which calls `ssrUrql`.
+
+(You might not need to use `useQuery` in your components if you're doing SSR and the page content is not dynamic, just pass the data from `getServerSideProps` in `props` and avoid GraphQL on the client side).

graphql.config.yaml

@@ -0,0 +1 @@
+schema: http://localhost:3000/api/graphql

package.json

@@ -21,12 +21,16 @@
     "reload": "heroku run:detached ts-node -T src/backend/flow/doEverythingForScheduler.js",
     "setCookies": "./src/backend/manual/setCookies.sh",
     "next-dev": "next dev",
-    "next-build": "next build",
+    "build": "prisma generate && next build",
     "next-start": "next start",
     "next-export": "next export",
     "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES"
   },
   "dependencies": {
+    "@graphql-yoga/node": "^2.1.0",
+    "@pothos/core": "^3.5.1",
+    "@pothos/plugin-prisma": "^3.4.0",
+    "@pothos/plugin-relay": "^3.10.0",
     "@prisma/client": "^3.11.1",
     "@tailwindcss/forms": "^0.4.0",
     "@tailwindcss/typography": "^0.5.1",
@@ -55,6 +59,7 @@
     "multiselect-react-dropdown": "^2.0.17",
     "next": "12",
     "next-plausible": "^3.1.6",
+    "next-urql": "^3.3.2",
     "nprogress": "^0.2.0",
     "open": "^7.3.1",
     "papaparse": "^5.3.0",
@@ -72,6 +77,7 @@
     "react-dropdown": "^1.9.2",
     "react-hook-form": "^7.27.0",
     "react-icons": "^4.2.0",
+    "react-is": "^18.0.0",
     "react-markdown": "^8.0.0",
     "react-safe": "^1.3.0",
     "react-select": "^5.2.2",
@@ -81,9 +87,18 @@
     "tailwindcss": "^3.0.22",
     "textversionjs": "^1.1.3",
     "ts-node": "^10.7.0",
-    "tunnel": "^0.0.6"
+    "tunnel": "^0.0.6",
+    "urql": "^2.2.0",
+    "urql-custom-scalars-exchange": "^0.1.5"
   },
   "devDependencies": {
+    "@graphql-codegen/cli": "^2.6.2",
+    "@graphql-codegen/introspection": "^2.1.1",
+    "@graphql-codegen/near-operation-file-preset": "^2.2.9",
+    "@graphql-codegen/schema-ast": "^2.4.1",
+    "@graphql-codegen/typed-document-node": "^2.2.8",
+    "@graphql-codegen/typescript": "^2.4.8",
+    "@graphql-codegen/typescript-operations": "^2.3.5",
     "@netlify/plugin-nextjs": "^4.2.4",
     "@svgr/cli": "^6.2.1",
     "@types/pg": "^8.6.5",

prisma/schema.prisma

@@ -2,12 +2,16 @@ generator client {
   provider = "prisma-client-js"
 }
 
+generator pothos {
+  provider = "prisma-pothos-types"
+}
+
 datasource db {
   provider = "postgresql"
   url      = env("DIGITALOCEAN_POSTGRES")
 }
 
-model dashboards {
+model Dashboard {
   id          String   @id
   title       String
   description String
@@ -15,15 +19,19 @@ model dashboards {
   timestamp   DateTime @db.Timestamp(6)
   creator     String
   extra       Json
+
+  @@map("dashboards")
 }
 
-model frontpage {
+model Frontpage {
   id               Int  @id @default(autoincrement())
   frontpage_full   Json
   frontpage_sliced Json
+
+  @@map("frontpage")
 }
 
-model history {
+model History {
   id                String
   title             String
   url               String
@@ -37,9 +45,10 @@ model history {
   pk                Int      @id @default(autoincrement())
 
   @@index([id])
+  @@map("history")
 }
 
-model questions {
+model Question {
   id                String   @id
   title             String
   url               String
@@ -50,4 +59,6 @@ model questions {
   stars             Int
   qualityindicators Json
   extra             Json
+
+  @@map("questions")
 }

schema.graphql

@@ -0,0 +1,86 @@
+input CreateDashboardInput {
+  creator: String
+  description: String
+  ids: [ID!]!
+  title: String!
+}
+
+type CreateDashboardResult {
+  dashboard: Dashboard!
+}
+
+type Dashboard {
+  creator: String!
+  description: String!
+  id: ID!
+  questions: [Question!]!
+  title: String!
+}
+
+"""Date serialized as the Unix timestamp."""
+scalar Date
+
+type Mutation {
+  createDashboard(input: CreateDashboardInput!): CreateDashboardResult!
+}
+
+type PageInfo {
+  endCursor: String
+  hasNextPage: Boolean!
+  hasPreviousPage: Boolean!
+  startCursor: String
+}
+
+"""Platform supported by metaforecast"""
+type Platform {
+  id: ID!
+  label: String!
+}
+
+type ProbabilityOption {
+  name: String
+  probability: Float
+}
+
+"""Various indicators of the question's quality"""
+type QualityIndicators {
+  numForecasts: Int
+  stars: Int!
+}
+
+type Query {
+  dashboard(id: ID!): Dashboard!
+  frontpage: [Question!]!
+  questions(after: String, before: String, first: Int, last: Int): QueryQuestionsConnection!
+  searchQuestions(input: SearchInput!): [Question!]!
+}
+
+type QueryQuestionsConnection {
+  edges: [QueryQuestionsConnectionEdge]!
+  pageInfo: PageInfo!
+}
+
+type QueryQuestionsConnectionEdge {
+  cursor: String!
+  node: Question!
+}
+
+type Question {
+  description: String!
+  id: ID!
+  options: [ProbabilityOption!]!
+  platform: Platform!
+  qualityIndicators: QualityIndicators!
+  timestamp: Date!
+  title: String!
+  url: String!
+  visualization: String
+}
+
+input SearchInput {
+  forecastingPlatforms: [String!]
+  forecastsThreshold: Int
+  limit: Int
+  query: String!
+  starsThreshold: Int
+}

src/backend/dashboards.ts

@@ -1,9 +0,0 @@
-export interface DashboardItem {
-  id: string;
-  title: string;
-  description: string;
-  contents: any;
-  timestamp: string;
-  creator: string;
-  extra: any;
-}

src/backend/database/pg-wrapper.ts

@@ -1,13 +1,13 @@
 import { Pool, PoolClient } from "pg";
 
-import { Forecast } from "../platforms";
+import { Question } from "../platforms";
 import { hash } from "../utils/hash";
 import { measureTime } from "../utils/measureTime";
 import { roughSizeOfObject } from "../utils/roughSize";
 
-const forecastTableNames = ["questions", "history"];
+const questionTableNames = ["questions", "history"];
 
-const allTableNames = [...forecastTableNames, "dashboards", "frontpage"];
+const allTableNames = [...questionTableNames, "dashboards", "frontpage"];
 
 /* Postgres database connection code */
 const databaseURL = process.env.DIGITALOCEAN_POSTGRES;
@@ -51,11 +51,11 @@ export async function pgBulkInsert({
   tableName,
   client,
 }: {
-  data: Forecast[];
+  data: Question[];
   tableName: string;
   client: PoolClient;
 }) {
-  if (!forecastTableNames.includes(tableName)) {
+  if (!questionTableNames.includes(tableName)) {
     throw Error(
       `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
     );
@@ -171,11 +171,11 @@ export async function pgUpsert({
   tableName,
   replacePlatform,
 }: {
-  contents: Forecast[];
+  contents: Question[];
   tableName: string;
   replacePlatform?: string;
 }) {
-  if (!forecastTableNames.includes(tableName)) {
+  if (!questionTableNames.includes(tableName)) {
     throw Error(
       `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
     );

src/backend/database/prisma.ts

@@ -0,0 +1,15 @@
+import { PrismaClient } from "@prisma/client";
+
+declare global {
+  // allow global `var` declarations
+  // eslint-disable-next-line no-var
+  var prisma: PrismaClient | undefined;
+}
+
+export const prisma =
+  global.prisma ||
+  new PrismaClient({
+    log: ["query"],
+  });
+
+if (process.env.NODE_ENV !== "production") global.prisma = prisma;

src/backend/frontpage.ts

@@ -1,7 +1,7 @@
 import { pgRead, pool } from "./database/pg-wrapper";
-import { Forecast } from "./platforms";
+import { Question } from "./platforms";
 
-export async function getFrontpage(): Promise<Forecast[]> {
+export async function getFrontpage(): Promise<Question[]> {
   const res = await pool.query(
     "SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
   );
@@ -9,7 +9,7 @@ export async function getFrontpage(): Promise<Forecast[]> {
   return res.rows[0].frontpage_sliced;
 }
 
-export async function getFrontpageFull(): Promise<Forecast[]> {
+export async function getFrontpageFull(): Promise<Question[]> {
   const res = await pool.query(
     "SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
   );

src/backend/migrate/noSchemaMigrate.ts

@@ -15,7 +15,7 @@ const migrate = async () => {
     FantasySCOTUS: "fantasyscotus",
     Foretold: "foretold",
     "GiveWell/OpenPhilanthropy": "givewellopenphil",
-    "Good Judgment": "goodjudgement",
+    "Good Judgment": "goodjudgment",
     "Good Judgment Open": "goodjudgmentopen",
     Infer: "infer",
     Kalshi: "kalshi",

src/backend/platforms/betfair.ts

@@ -3,7 +3,7 @@ import axios from "axios";
 import https from "https";
 
 import { calculateStars } from "../utils/stars";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 const platformName = "betfair";
 
@@ -80,7 +80,7 @@ async function whipIntoShape(data) {
 async function processPredictions(data) {
   let predictions = await whipIntoShape(data);
   // console.log(JSON.stringify(predictions, null, 4))
-  let results: Forecast[] = predictions.map((prediction) => {
+  let results: Question[] = predictions.map((prediction) => {
     /* if(Math.floor(Math.random() * 10) % 20 ==0){
        console.log(JSON.stringify(prediction, null, 4))
     } */

src/backend/platforms/fantasyscotus.ts

@@ -2,7 +2,7 @@
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 const platformName = "fantasyscotus";
 
@@ -67,7 +67,7 @@ async function processData(data) {
   let historicalPercentageCorrect = data.stats.pcnt_correct;
   let historicalProbabilityCorrect =
     Number(historicalPercentageCorrect.replace("%", "")) / 100;
-  let results: Forecast[] = [];
+  let results: Question[] = [];
   for (let event of events) {
     if (event.accuracy == "") {
       let id = `${platformName}-${event.id}`;
@@ -75,7 +75,7 @@ async function processData(data) {
       let predictionData = await getPredictionsData(event.docket_url);
       let pAffirm = predictionData.proportionAffirm;
       //let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
-      let eventObject: Forecast = {
+      let eventObject: Question = {
         id: id,
         title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
         url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,

src/backend/platforms/index.ts

@@ -16,7 +16,21 @@ import { smarkets } from "./smarkets";
 import { wildeford } from "./wildeford";
 import { xrisk } from "./xrisk";
 
-export interface Forecast {
+export interface QualityIndicators {
+  stars: number;
+  numforecasts?: number | string;
+  numforecasters?: number;
+  liquidity?: number | string;
+  volume?: number;
+  volume7Days?: number;
+  volume24Hours?: number;
+  address?: number;
+  tradevolume?: string;
+  pool?: any;
+  createdTime?: any;
+}
+
+export interface Question {
   id: string;
   // "fantasyscotus-580"
 
@@ -53,7 +67,7 @@ export interface Forecast {
   stars?: number;
   // 2
 
-  qualityindicators: any;
+  qualityindicators: QualityIndicators;
   /*
   {
     "numforecasts": 120,
@@ -63,8 +77,8 @@ export interface Forecast {
   extra?: any;
 }
 
-// fetcher should return null if platform failed to fetch forecasts for some reason
+// fetcher should return null if platform failed to fetch questions for some reason
-export type PlatformFetcher = () => Promise<Forecast[] | null>;
+export type PlatformFetcher = () => Promise<Question[] | null>;
 
 export interface Platform {
   name: string; // short name for ids and `platform` db column, e.g. "xrisk"
@@ -76,7 +90,7 @@ export interface Platform {
 // draft for the future callback-based streaming/chunking API:
 // interface FetchOptions {
 //   since?: string; // some kind of cursor, Date object or opaque string?
-//   save: (forecasts: Forecast[]) => Promise<void>;
+//   save: (questions: Question[]) => Promise<void>;
 // }
 
 // export type PlatformFetcher = (options: FetchOptions) => Promise<void>;

src/backend/platforms/infer.ts

@@ -1,12 +1,11 @@
 /* Imports */
 import axios from "axios";
-import { Tabletojson } from "tabletojson";
 
 import { applyIfSecretExists } from "../utils/getSecrets";
 import { measureTime } from "../utils/measureTime";
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 /* Definitions */
 const platformName = "infer";
@@ -78,12 +77,12 @@ async function fetchStats(questionUrl, cookie) {
   let comments_count = firstEmbeddedJson.question.comments_count;
   let numforecasters = firstEmbeddedJson.question.predictors_count;
   let numforecasts = firstEmbeddedJson.question.prediction_sets_count;
-  let forecastType = firstEmbeddedJson.question.type;
+  let questionType = firstEmbeddedJson.question.type;
   if (
-    forecastType.includes("Binary") ||
+    questionType.includes("Binary") ||
-    forecastType.includes("NonExclusiveOpinionPoolQuestion") ||
+    questionType.includes("NonExclusiveOpinionPoolQuestion") ||
-    forecastType.includes("Forecast::Question") ||
+    questionType.includes("Forecast::Question") ||
-    !forecastType.includes("Forecast::MultiTimePeriodQuestion")
+    !questionType.includes("Forecast::MultiTimePeriodQuestion")
   ) {
     options = firstEmbeddedJson.question.answers.map((answer) => ({
       name: answer.name,
@@ -148,7 +147,7 @@ function sleep(ms) {
 async function infer_inner(cookie: string) {
   let i = 1;
   let response = await fetchPage(i, cookie);
-  let results: Forecast[] = [];
+  let results: Question[] = [];
 
   await measureTime(async () => {
     // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
@@ -179,7 +178,7 @@ async function infer_inner(cookie: string) {
           let questionNumRegex = new RegExp("questions/([0-9]+)");
           let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
           let id = `${platformName}-${questionNum}`;
-          let question: Forecast = {
+          let question: Question = {
             id: id,
             title: title,
             description: moreinfo.description,

src/backend/platforms/manifold.ts

@@ -2,7 +2,7 @@
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 /* Definitions */
 const platformName = "manifold";
@@ -23,7 +23,7 @@ async function fetchData() {
   return response;
 }
 
-function showStatistics(results: Forecast[]) {
+function showStatistics(results: Question[]) {
   console.log(`Num unresolved markets: ${results.length}`);
   let sum = (arr) => arr.reduce((tally, a) => tally + a, 0);
   let num2StarsOrMore = results.filter(
@@ -44,7 +44,7 @@ function showStatistics(results: Forecast[]) {
 }
 
 async function processPredictions(predictions) {
-  let results: Forecast[] = await predictions.map((prediction) => {
+  let results: Question[] = await predictions.map((prediction) => {
     let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name
     let probability = prediction.probability;
     let options = [
@@ -59,7 +59,7 @@ async function processPredictions(predictions) {
         type: "PROBABILITY",
       },
     ];
-    const result: Forecast = {
+    const result: Question = {
       id: id,
       title: prediction.question,
       url: prediction.url,

src/backend/platforms/polymarket.ts

@@ -2,7 +2,7 @@
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 /* Definitions */
 const platformName = "polymarket";
@@ -68,7 +68,7 @@ export const polymarket: Platform = {
   label: "PolyMarket",
   color: "#00314e",
   async fetcher() {
-    let results: Forecast[] = [];
+    let results: Question[] = [];
     let webpageEndpointData = await fetchAllContractInfo();
     for (let marketInfo of webpageEndpointData) {
       let address = marketInfo.marketMakerAddress;
@@ -102,7 +102,7 @@ export const polymarket: Platform = {
             });
           }
 
-          let result: Forecast = {
+          let result: Question = {
             id: id,
             title: marketInfo.question,
             url: "https://polymarket.com/market/" + marketInfo.slug,

src/backend/platforms/rootclaim.ts

@@ -3,7 +3,7 @@ import { JSDOM } from "jsdom";
 
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 const platformName = "rootclaim";
 const jsonEndpoint =
@@ -50,7 +50,7 @@ export const rootclaim: Platform = {
   color: "#0d1624",
   async fetcher() {
     const claims = await fetchAllRootclaims();
-    const results: Forecast[] = [];
+    const results: Question[] = [];
 
     for (const claim of claims) {
       const id = `${platformName}-${claim.slug.toLowerCase()}`;
@@ -71,7 +71,7 @@ export const rootclaim: Platform = {
 
       const description = await fetchDescription(url, claim.isclaim);
 
-      let obj: Forecast = {
+      let obj: Question = {
         id,
         title: toMarkdown(claim.question).replace("\n", ""),
         url,

src/backend/platforms/smarkets.ts

@@ -1,7 +1,7 @@
 import axios from "axios";
 
 import { calculateStars } from "../utils/stars";
-import { Forecast, Platform } from "./";
+import { Platform, Question } from "./";
 
 /* Definitions */
 const platformName = "smarkets";
@@ -159,7 +159,7 @@ export const smarkets: Platform = {
       name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`)
     }
     */
-      let result: Forecast = {
+      let result: Question = {
         id: id,
         title: name,
         url: "https://smarkets.com/event/" + market.event_id + market.slug,

src/backend/utils/algolia.ts

@@ -1,6 +1,8 @@
 import algoliasearch from "algoliasearch";
 
-import { pgRead } from "../database/pg-wrapper";
+import { Question } from "@prisma/client";
+
+import { prisma } from "../database/prisma";
 import { platforms } from "../platforms";
 
 let cookie = process.env.ALGOLIA_MASTER_API_KEY;
@@ -8,10 +10,14 @@ const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
 const client = algoliasearch(algoliaAppId, cookie);
 const index = client.initIndex("metaforecast");
 
-let getoptionsstringforsearch = (record: any) => {
+export type AlgoliaQuestion = Omit<Question, "timestamp"> & {
+  timestamp: string;
+};
+
+const getoptionsstringforsearch = (record: Question): string => {
   let result = "";
-  if (!!record.options && record.options.length > 0) {
+  if (!!record.options && (record.options as any[]).length > 0) {
-    result = record.options
+    result = (record.options as any[])
       .map((option: any) => option.name || null)
       .filter((x: any) => x != null)
       .join(", ");
@@ -19,23 +25,23 @@ let getoptionsstringforsearch = (record: any) => {
   return result;
 };
 
-export async function rebuildAlgoliaDatabaseTheEasyWay() {
+export async function rebuildAlgoliaDatabase() {
-  let records: any[] = await pgRead({
+  const questions = await prisma.question.findMany();
-    tableName: "questions",
-  });
 
   const platformNameToLabel = Object.fromEntries(
     platforms.map((platform) => [platform.name, platform.label])
   );
 
-  records = records.map((record, index: number) => ({
+  const records: AlgoliaQuestion[] = questions.map(
-    ...record,
+    (question, index: number) => ({
-    platformLabel: platformNameToLabel[record.platform] || record.platform,
+      ...question,
-    has_numforecasts: record.numforecasts ? true : false,
+      timestamp: `${question.timestamp}`,
+      platformLabel:
+        platformNameToLabel[question.platform] || question.platform,
       objectID: index,
-    optionsstringforsearch: getoptionsstringforsearch(record),
+      optionsstringforsearch: getoptionsstringforsearch(question),
-  }));
+    })
-  // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
+  );
 
   if (index.exists()) {
     console.log("Index exists");
@@ -45,5 +51,3 @@ export async function rebuildAlgoliaDatabaseTheEasyWay() {
     );
   }
 }
-
-export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay;

src/backend/utils/evaluations/pullForecastsToCSVForRating.ts

@@ -8,14 +8,14 @@ import { pgRead } from "../../database/pg-wrapper";
 /* Utilities */
 
 /* Support functions */
-let getQualityIndicators = (forecast) =>
+const getQualityIndicators = (question) =>
-  Object.entries(forecast.qualityindicators)
+  Object.entries(question.qualityindicators)
     .map((entry) => `${entry[0]}: ${entry[1]}`)
     .join("; ");
 
 /* Body */
 
-let main = async () => {
+const main = async () => {
   let highQualityPlatforms = [
     "CSET-foretell",
     "Foretold",
@@ -24,21 +24,21 @@ let main = async () => {
     "PredictIt",
     "Rootclaim",
   ];
-  let json = await pgRead({ tableName: "questions" });
+  const json = await pgRead({ tableName: "questions" });
   console.log(json.length);
   //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
   //console.log(uniquePlatforms)
 
-  let forecastsFromGoodPlatforms = json.filter((forecast) =>
+  const questionsFromGoodPlatforms = json.filter((question) =>
-    highQualityPlatforms.includes(forecast.platform)
+    highQualityPlatforms.includes(question.platform)
   );
-  let tsv =
+  const tsv =
     "index\ttitle\turl\tqualityindicators\n" +
-    forecastsFromGoodPlatforms
+    questionsFromGoodPlatforms
-      .map((forecast, index) => {
+      .map((question, index) => {
-        let row = `${index}\t${forecast.title}\t${
+        let row = `${index}\t${question.title}\t${
-          forecast.url
+          question.url
-        }\t${getQualityIndicators(forecast)}`;
+        }\t${getQualityIndicators(question)}`;
         console.log(row);
         return row;
       })

src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts

@@ -1,6 +1,7 @@
 /* Imports */
 import fs from "fs";
 
+import { shuffleArray } from "../../../utils";
 import { pgRead } from "../../database/pg-wrapper";
 
 /* Definitions */
@@ -8,42 +9,33 @@ import { pgRead } from "../../database/pg-wrapper";
 /* Utilities */
 
 /* Support functions */
-let getQualityIndicators = (forecast) =>
+let getQualityIndicators = (question) =>
-  Object.entries(forecast.qualityindicators)
+  Object.entries(question.qualityindicators)
     .map((entry) => `${entry[0]}: ${entry[1]}`)
     .join("; ");
 
-let shuffleArray = (array) => {
-  // See: https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array
-  for (let i = array.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1));
-    [array[i], array[j]] = [array[j], array[i]];
-  }
-  return array;
-};
-
 /* Body */
 
 let main = async () => {
   let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim']
   let json = await pgRead({ tableName: "questions" });
   console.log(json.length);
-  //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
+  //let uniquePlatforms = [...new Set(json.map(question => question.platform))]
   //console.log(uniquePlatforms)
 
-  let forecastsFromGoodPlatforms = json.filter((forecast) =>
+  let questionsFromGoodPlatforms = json.filter((question) =>
-    highQualityPlatforms.includes(forecast.platform)
+    highQualityPlatforms.includes(question.platform)
   );
-  let forecastsFromGoodPlatformsShuffled = shuffleArray(
+  let questionsFromGoodPlatformsShuffled = shuffleArray(
-    forecastsFromGoodPlatforms
+    questionsFromGoodPlatforms
   );
   let tsv =
     "index\ttitle\turl\tqualityindicators\n" +
-    forecastsFromGoodPlatforms
+    questionsFromGoodPlatforms
-      .map((forecast, index) => {
+      .map((question, index) => {
-        let row = `${index}\t${forecast.title}\t${
+        let row = `${index}\t${question.title}\t${
-          forecast.url
+          question.url
-        }\t${getQualityIndicators(forecast)}`;
+        }\t${getQualityIndicators(question)}`;
         console.log(row);
         return row;
       })

src/graphql/build-schema.js

@@ -0,0 +1,5 @@
+// see https://pothos-graphql.dev/docs/guide/generating-client-types#export-schema-in-a-js-file,
+// but we use ts-node instead of @boost/module
+require("ts-node").register({});
+
+module.exports = require("./schema/index.ts");

src/graphql/builder.ts

@@ -0,0 +1,40 @@
+import SchemaBuilder from "@pothos/core";
+import PrismaPlugin from "@pothos/plugin-prisma";
+import RelayPlugin from "@pothos/plugin-relay";
+
+import { prisma } from "../backend/database/prisma";
+
+import type PrismaTypes from "@pothos/plugin-prisma/generated";
+export const builder = new SchemaBuilder<{
+  PrismaTypes: PrismaTypes;
+  Scalars: {
+    Date: {
+      Input: Date;
+      Output: Date;
+    };
+  };
+}>({
+  plugins: [PrismaPlugin, RelayPlugin],
+  prisma: {
+    client: prisma,
+  },
+  relayOptions: {
+    clientMutationId: "omit",
+    cursorType: "String",
+    // these are required for some reason, though it's not documented and probably a bug
+    brandLoadedObjects: undefined,
+    encodeGlobalID: undefined,
+    decodeGlobalID: undefined,
+  },
+});
+
+builder.scalarType("Date", {
+  description: "Date serialized as the Unix timestamp.",
+  serialize: (d) => d.getTime() / 1000,
+  parseValue: (d) => {
+    return new Date(d as string); // not sure if this is correct, need to check
+  },
+});
+
+builder.queryType({});
+builder.mutationType({});

src/graphql/schema/dashboards.ts

@@ -0,0 +1,110 @@
+import { Dashboard } from "@prisma/client";
+
+import { prisma } from "../../backend/database/prisma";
+import { hash } from "../../backend/utils/hash";
+import { builder } from "../builder";
+import { QuestionObj } from "./questions";
+
+const DashboardObj = builder.objectRef<Dashboard>("Dashboard").implement({
+  fields: (t) => ({
+    id: t.exposeID("id"),
+    title: t.exposeString("title", {
+      description: "The title of the dashboard",
+    }),
+    description: t.exposeString("description", {
+      description: "The longer description of the dashboard",
+    }),
+    creator: t.exposeString("creator", {
+      description: 'The creator of the dashboard, e.g. "Peter Parker"',
+    }),
+    questions: t.field({
+      type: [QuestionObj],
+      description: "The list of questions on the dashboard",
+      resolve: async (parent) => {
+        return await prisma.question.findMany({
+          where: {
+            id: {
+              in: parent.contents as string[],
+            },
+          },
+        });
+      },
+    }),
+  }),
+});
+
+builder.queryField("dashboard", (t) =>
+  t.field({
+    type: DashboardObj,
+    description: "Look up a single dashboard by its id",
+    args: {
+      id: t.arg({ type: "ID", required: true }),
+    },
+    resolve: async (parent, args) => {
+      return await prisma.dashboard.findUnique({
+        where: {
+          id: String(args.id),
+        },
+      });
+    },
+  })
+);
+
+const CreateDashboardResult = builder
+  .objectRef<{ dashboard: Dashboard }>("CreateDashboardResult")
+  .implement({
+    fields: (t) => ({
+      dashboard: t.field({
+        type: DashboardObj,
+        resolve: (parent) => parent.dashboard,
+      }),
+    }),
+  });
+
+const CreateDashboardInput = builder.inputType("CreateDashboardInput", {
+  fields: (t) => ({
+    title: t.string({
+      required: true,
+      description: "The title of the dashboard",
+    }),
+    description: t.string({
+      description: "The longer description of the dashboard",
+    }),
+    creator: t.string({
+      description: 'The creator of the dashboard, e.g. "Peter Parker"',
+    }),
+    ids: t.idList({ required: true, description: "List of question ids" }),
+  }),
+});
+
+builder.mutationField("createDashboard", (t) =>
+  t.field({
+    type: CreateDashboardResult,
+    description:
+      "Create a new dashboard; if the dashboard with given ids already exists then it will be returned instead.",
+    args: {
+      input: t.arg({ type: CreateDashboardInput, required: true }),
+    },
+    resolve: async (parent, args) => {
+      const id = hash(JSON.stringify(args.input.ids));
+      const dashboard = await prisma.dashboard.upsert({
+        where: {
+          id,
+        },
+        update: {},
+        create: {
+          id,
+          title: args.input.title,
+          description: args.input.description || "",
+          creator: args.input.creator || "",
+          contents: args.input.ids,
+          extra: [],
+          timestamp: new Date(),
+        },
+      });
+      return {
+        dashboard,
+      };
+    },
+  })
+);

src/graphql/schema/frontpage.ts

@@ -0,0 +1,30 @@
+import { Question } from "@prisma/client";
+
+import { prisma } from "../../backend/database/prisma";
+import { getFrontpage } from "../../backend/frontpage";
+import { builder } from "../builder";
+import { QuestionObj } from "./questions";
+
+builder.queryField("frontpage", (t) =>
+  t.field({
+    type: [QuestionObj],
+    description: "Get a list of questions that are currently on the frontpage",
+    resolve: async () => {
+      const legacyQuestions = await getFrontpage();
+      const ids = legacyQuestions.map((q) => q.id);
+      const questions = await prisma.question.findMany({
+        where: {
+          id: {
+            in: ids,
+          },
+        },
+      });
+      const id2q: { [k: string]: Question } = {};
+      for (const q of questions) {
+        id2q[q.id] = q;
+      }
+
+      return ids.map((id) => id2q[id] || null).filter((q) => q !== null);
+    },
+  })
+);

src/graphql/schema/index.ts

@@ -0,0 +1,8 @@
+import "./dashboards";
+import "./frontpage";
+import "./questions";
+import "./search";
+
+import { builder } from "../builder";
+
+export const schema = builder.toSchema({});

src/graphql/schema/questions.ts

@@ -0,0 +1,116 @@
+import { prisma } from "../../backend/database/prisma";
+import { platforms, QualityIndicators } from "../../backend/platforms";
+import { builder } from "../builder";
+
+const PlatformObj = builder.objectRef<string>("Platform").implement({
+  description: "Forecasting platform supported by Metaforecast",
+  fields: (t) => ({
+    id: t.id({
+      description: 'Short unique platform name, e.g. "xrisk"',
+      resolve: (x) => x,
+    }),
+    label: t.string({
+      description:
+        'Platform name for displaying on frontend etc., e.g. "X-risk estimates"',
+      resolve: (platformName) => {
+        if (platformName === "metaforecast") {
+          return "Metaforecast";
+        }
+        if (platformName === "guesstimate") {
+          return "Guesstimate";
+        }
+        // kinda slow and repetitive, TODO - store a map {name => platform} somewhere and `getPlatform` util function?
+        const platform = platforms.find((p) => p.name === platformName);
+        if (!platform) {
+          throw new Error(`Unknown platform ${platformName}`);
+        }
+        return platform.label;
+      },
+    }),
+  }),
+});
+
+export const QualityIndicatorsObj = builder
+  .objectRef<QualityIndicators>("QualityIndicators")
+  .implement({
+    description: "Various indicators of the question's quality",
+    fields: (t) => ({
+      stars: t.exposeInt("stars", {
+        description: "0 to 5",
+      }),
+      numForecasts: t.int({
+        nullable: true,
+        resolve: (parent) =>
+          parent.numforecasts === undefined
+            ? undefined
+            : Number(parent.numforecasts),
+      }),
+    }),
+  });
+
+export const ProbabilityOptionObj = builder
+  .objectRef<{ name: string; probability: number }>("ProbabilityOption")
+  .implement({
+    fields: (t) => ({
+      name: t.exposeString("name", { nullable: true }),
+      probability: t.exposeFloat("probability", {
+        description: "0 to 1",
+        nullable: true,
+      }),
+    }),
+  });
+
+export const QuestionObj = builder.prismaObject("Question", {
+  findUnique: (question) => ({ id: question.id }),
+  fields: (t) => ({
+    id: t.exposeID("id", {
+      description: "Unique string which identifies the question",
+    }),
+    title: t.exposeString("title"),
+    description: t.exposeString("description"),
+    url: t.exposeString("url", {
+      description:
+        "Non-unique, a very small number of platforms have a page for more than one prediction",
+    }),
+    timestamp: t.field({
+      type: "Date",
+      description: "Timestamp at which metaforecast fetched the question",
+      resolve: (parent) => parent.timestamp,
+    }),
+    platform: t.field({
+      type: PlatformObj,
+      resolve: (parent) => parent.platform,
+    }),
+    qualityIndicators: t.field({
+      type: QualityIndicatorsObj,
+      resolve: (parent) => parent.qualityindicators as any as QualityIndicators,
+    }),
+    options: t.field({
+      type: [ProbabilityOptionObj],
+      resolve: ({ options }) => {
+        if (!Array.isArray(options)) {
+          return [];
+        }
+        return options as any[];
+      },
+    }),
+    visualization: t.string({
+      resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts
+      nullable: true,
+    }),
+  }),
+});
+
+builder.queryField("questions", (t) =>
+  t.prismaConnection(
+    {
+      type: "Question",
+      cursor: "id",
+      maxSize: 1000,
+      resolve: (query, parent, args, context, info) =>
+        prisma.question.findMany({ ...query }),
+    },
+    {},
+    {}
+  )
+);

src/graphql/schema/search.ts

@@ -0,0 +1,70 @@
+import { AlgoliaQuestion } from "../../backend/utils/algolia";
+import searchGuesstimate from "../../web/worker/searchGuesstimate";
+import searchWithAlgolia from "../../web/worker/searchWithAlgolia";
+import { builder } from "../builder";
+import { QuestionObj } from "./questions";
+
+const SearchInput = builder.inputType("SearchInput", {
+  fields: (t) => ({
+    query: t.string({ required: true }),
+    starsThreshold: t.int({
+      description: "Minimum number of stars on a question",
+    }),
+    forecastsThreshold: t.int({
+      description: "Minimum number of forecasts on a question",
+    }),
+    forecastingPlatforms: t.stringList({
+      description: "List of platform ids to filter by",
+    }),
+    limit: t.int(),
+  }),
+});
+
+builder.queryField("searchQuestions", (t) =>
+  t.field({
+    type: [QuestionObj],
+    description:
+      "Search for questions; uses Algolia instead of the primary metaforecast database",
+    args: {
+      input: t.arg({ type: SearchInput, required: true }),
+    },
+    resolve: async (parent, { input }) => {
+      // defs
+      const query = input.query === undefined ? "" : input.query;
+      if (query === "") return [];
+      const forecastsThreshold = input.forecastsThreshold;
+      const starsThreshold = input.starsThreshold;
+      const platformsIncludeGuesstimate =
+        input.forecastingPlatforms?.includes("guesstimate") &&
+        starsThreshold <= 1;
+
+      // preparation
+      const unawaitedAlgoliaResponse = searchWithAlgolia({
+        queryString: query,
+        hitsPerPage: input.limit + 50,
+        starsThreshold,
+        filterByPlatforms: input.forecastingPlatforms,
+        forecastsThreshold,
+      });
+
+      let results: AlgoliaQuestion[] = [];
+
+      // consider the guesstimate and the non-guesstimate cases separately.
+      if (platformsIncludeGuesstimate) {
+        const [responsesNotGuesstimate, responsesGuesstimate] =
+          await Promise.all([
+            unawaitedAlgoliaResponse,
+            searchGuesstimate(query),
+          ]); // faster than two separate requests
+        results = [...responsesNotGuesstimate, ...responsesGuesstimate];
+      } else {
+        results = await unawaitedAlgoliaResponse;
+      }
+
+      return results.map((q) => ({
+        ...q,
+        timestamp: new Date(q.timestamp),
+      }));
+    },
+  })
+);

src/graphql/types.generated.ts

@@ -0,0 +1,133 @@
+export type Maybe<T> = T | null;
+export type InputMaybe<T> = Maybe<T>;
+export type Exact<T extends { [key: string]: unknown }> = { [K in keyof T]: T[K] };
+export type MakeOptional<T, K extends keyof T> = Omit<T, K> & { [SubKey in K]?: Maybe<T[SubKey]> };
+export type MakeMaybe<T, K extends keyof T> = Omit<T, K> & { [SubKey in K]: Maybe<T[SubKey]> };
+/** All built-in and custom scalars, mapped to their actual values */
+export type Scalars = {
+  ID: string;
+  String: string;
+  Boolean: boolean;
+  Int: number;
+  Float: number;
+  /** Date serialized as the Unix timestamp. */
+  Date: any;
+};
+
+export type CreateDashboardInput = {
+  creator?: InputMaybe<Scalars['String']>;
+  description?: InputMaybe<Scalars['String']>;
+  ids: Array<Scalars['ID']>;
+  title: Scalars['String'];
+};
+
+export type CreateDashboardResult = {
+  __typename?: 'CreateDashboardResult';
+  dashboard: Dashboard;
+};
+
+export type Dashboard = {
+  __typename?: 'Dashboard';
+  creator: Scalars['String'];
+  description: Scalars['String'];
+  id: Scalars['ID'];
+  questions: Array<Question>;
+  title: Scalars['String'];
+};
+
+export type Mutation = {
+  __typename?: 'Mutation';
+  createDashboard: CreateDashboardResult;
+};
+
+
+export type MutationCreateDashboardArgs = {
+  input: CreateDashboardInput;
+};
+
+export type PageInfo = {
+  __typename?: 'PageInfo';
+  endCursor?: Maybe<Scalars['String']>;
+  hasNextPage: Scalars['Boolean'];
+  hasPreviousPage: Scalars['Boolean'];
+  startCursor?: Maybe<Scalars['String']>;
+};
+
+/** Platform supported by metaforecast */
+export type Platform = {
+  __typename?: 'Platform';
+  id: Scalars['ID'];
+  label: Scalars['String'];
+};
+
+export type ProbabilityOption = {
+  __typename?: 'ProbabilityOption';
+  name?: Maybe<Scalars['String']>;
+  probability?: Maybe<Scalars['Float']>;
+};
+
+/** Various indicators of the question's quality */
+export type QualityIndicators = {
+  __typename?: 'QualityIndicators';
+  numForecasts?: Maybe<Scalars['Int']>;
+  stars: Scalars['Int'];
+};
+
+export type Query = {
+  __typename?: 'Query';
+  dashboard: Dashboard;
+  frontpage: Array<Question>;
+  questions: QueryQuestionsConnection;
+  searchQuestions: Array<Question>;
+};
+
+
+export type QueryDashboardArgs = {
+  id: Scalars['ID'];
+};
+
+
+export type QueryQuestionsArgs = {
+  after?: InputMaybe<Scalars['String']>;
+  before?: InputMaybe<Scalars['String']>;
+  first?: InputMaybe<Scalars['Int']>;
+  last?: InputMaybe<Scalars['Int']>;
+};
+
+
+export type QuerySearchQuestionsArgs = {
+  input: SearchInput;
+};
+
+export type QueryQuestionsConnection = {
+  __typename?: 'QueryQuestionsConnection';
+  edges: Array<Maybe<QueryQuestionsConnectionEdge>>;
+  pageInfo: PageInfo;
+};
+
+export type QueryQuestionsConnectionEdge = {
+  __typename?: 'QueryQuestionsConnectionEdge';
+  cursor: Scalars['String'];
+  node: Question;
+};
+
+export type Question = {
+  __typename?: 'Question';
+  description: Scalars['String'];
+  id: Scalars['ID'];
+  options: Array<ProbabilityOption>;
+  platform: Platform;
+  qualityIndicators: QualityIndicators;
+  timestamp: Scalars['Date'];
+  title: Scalars['String'];
+  url: Scalars['String'];
+  visualization?: Maybe<Scalars['String']>;
+};
+
+export type SearchInput = {
+  forecastingPlatforms?: InputMaybe<Array<Scalars['String']>>;
+  forecastsThreshold?: InputMaybe<Scalars['Int']>;
+  limit?: InputMaybe<Scalars['Int']>;
+  query: Scalars['String'];
+  starsThreshold?: InputMaybe<Scalars['Int']>;
+};

src/pages/_app.tsx

@@ -1,13 +1,15 @@
 import "nprogress/nprogress.css";
 import "../styles/main.css";
 
+import PlausibleProvider from "next-plausible";
+import { withUrqlClient } from "next-urql";
+import { AppProps } from "next/app";
 import Router from "next/router";
 import NProgress from "nprogress";
 
-import PlausibleProvider from "next-plausible";
+import { getUrqlClientOptions } from "../web/urql";
 
 Router.events.on("routeChangeStart", (as, { shallow }) => {
-  console.log(shallow);
   if (!shallow) {
     NProgress.start();
   }
@@ -15,7 +17,7 @@ Router.events.on("routeChangeStart", (as, { shallow }) => {
 Router.events.on("routeChangeComplete", () => NProgress.done());
 Router.events.on("routeChangeError", () => NProgress.done());
 
-function MyApp({ Component, pageProps }) {
+function MyApp({ Component, pageProps }: AppProps) {
   return (
     <PlausibleProvider domain="metaforecast.org">
       <Component {...pageProps} />
@@ -23,4 +25,6 @@ function MyApp({ Component, pageProps }) {
   );
 }
 
-export default MyApp;
+export default withUrqlClient((ssr) => getUrqlClientOptions(ssr), {
+  ssr: false,
+})(MyApp);

src/pages/api/all-forecasts.ts

@@ -1,13 +0,0 @@
-import { NextApiRequest, NextApiResponse } from "next/types";
-
-import { getFrontpageFull } from "../../backend/frontpage";
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse
-) {
-  let frontpageFull = await getFrontpageFull();
-  console.log(frontpageFull.map((element) => element.title).slice(0, 5));
-  console.log("...");
-  res.status(200).json(frontpageFull);
-}

src/pages/api/create-dashboard-from-ids.ts

@@ -1,39 +0,0 @@
-import { NextApiRequest, NextApiResponse } from "next/types";
-
-import { pgInsertIntoDashboard } from "../../backend/database/pg-wrapper";
-import { hash } from "../../backend/utils/hash";
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse
-) {
-  if (req.method !== "POST") {
-    res.status(400).send("Expected POST request");
-    return;
-  }
-
-  let body = req.body;
-  console.log(body);
-  try {
-    let id = hash(JSON.stringify(body.ids));
-    let pgResponse = await pgInsertIntoDashboard({
-      datum: {
-        id: id,
-        title: body.title || "",
-        description: body.description || "",
-        contents: body.ids,
-        creator: body.creator || "",
-        extra: [],
-      },
-    });
-    res.status(200).send({
-      dashboardId: id,
-      pgResponse: pgResponse,
-    });
-  } catch (error) {
-    res.status(400).send({
-      id: null,
-      pgResponse: JSON.stringify(error),
-    });
-  }
-}

src/pages/api/dashboard-by-id.ts

@@ -1,35 +0,0 @@
-import { NextApiRequest, NextApiResponse } from "next/types";
-
-import { pgGetByIds } from "../../backend/database/pg-wrapper";
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse
-) {
-  if (req.method !== "POST") {
-    res.status(400).send("Expected POST request");
-    return;
-  }
-
-  console.log(req.body);
-  let id = req.body.id;
-  console.log(id);
-  let dashboardItemArray = await pgGetByIds({
-    ids: [id],
-    table: "dashboards",
-  });
-  if (!!dashboardItemArray && dashboardItemArray.length > 0) {
-    let dashboardItem = dashboardItemArray[0];
-    console.log(dashboardItem);
-    let dashboardContents = await pgGetByIds({
-      ids: dashboardItem.contents,
-      table: "questions",
-    });
-    res.status(200).send({
-      dashboardContents,
-      dashboardItem,
-    });
-  } else {
-    res.status(404).send({ error: `Dashboard not found with id ${id}` });
-  }
-}

src/pages/api/frontpage.ts

@@ -1,11 +0,0 @@
-import { NextApiRequest, NextApiResponse } from "next/types";
-
-import { getFrontpage } from "../../backend/frontpage";
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse
-) {
-  let frontpageElements = await getFrontpage();
-  res.status(200).json(frontpageElements);
-}

src/pages/api/graphql.ts

@@ -0,0 +1,13 @@
+import { NextApiRequest, NextApiResponse } from "next";
+
+// apollo-server-micro is problematic since v3, see https://github.com/apollographql/apollo-server/issues/5547, so we use graphql-yoga instead
+import { createServer } from "@graphql-yoga/node";
+
+import { schema } from "../../graphql/schema";
+
+const server = createServer<{
+  req: NextApiRequest;
+  res: NextApiResponse;
+}>({ schema });
+
+export default server;

src/pages/api/notes.md

@@ -1,15 +0,0 @@
-# Metaforecast API
-
-Modelled roughly after the [Manifold Markets API.](https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5). Much as theirs, the metaforecast API is also in alpha. It has at various points been
-
-## List out all markets
-
-## Get markets for one particular platform
-
-## Get history
-
-Not yet implemented
-
-//
-
-https://nextjs.org/docs/messages/api-routes-response-size-limit

src/pages/api/questions.ts

@@ -1,13 +0,0 @@
-import { NextApiRequest, NextApiResponse } from "next/types";
-
-import { pgRead } from "../../backend/database/pg-wrapper";
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse
-) {
-  let allQuestions = await pgRead({ tableName: "questions" });
-  console.log(allQuestions.map((element) => element.title).slice(0, 5));
-  console.log("...");
-  res.status(200).json(allQuestions);
-}

src/pages/capture.tsx

@@ -1,7 +1,7 @@
 import { NextPage } from "next";
 import React from "react";
 
-import { displayForecastsWrapperForCapture } from "../web/display/displayForecastsWrappers";
+import { displayQuestionsWrapperForCapture } from "../web/display/displayQuestionsWrappers";
 import { Layout } from "../web/display/Layout";
 import { Props } from "../web/search/anySearchPage";
 import CommonDisplay from "../web/search/CommonDisplay";
@@ -10,15 +10,15 @@ export { getServerSideProps } from "../web/search/anySearchPage";
 
 const CapturePage: NextPage<Props> = (props) => {
   return (
-    <Layout page={"capture"}>
+    <Layout page="capture">
       <CommonDisplay
         {...props}
         hasSearchbar={true}
         hasCapture={true}
         hasAdvancedOptions={false}
-        placeholder={"Get best title match..."}
+        placeholder="Get best title match..."
         displaySeeMoreHint={false}
-        displayForecastsWrapper={displayForecastsWrapperForCapture}
+        displayQuestionsWrapper={displayQuestionsWrapperForCapture}
       />
     </Layout>
   );

src/pages/dashboards/embed/[id].tsx

@@ -1,49 +1,44 @@
 import { GetServerSideProps, NextPage } from "next";
 import Error from "next/error";
 
-import { DashboardItem } from "../../../backend/dashboards";
+import {
-import { DisplayForecasts } from "../../../web/display/DisplayForecasts";
+    DashboardByIdDocument, DashboardFragment
-import { FrontendForecast } from "../../../web/platforms";
+} from "../../../web/dashboards/queries.generated";
-import { getDashboardForecastsByDashboardId } from "../../../web/worker/getDashboardForecasts";
+import { DisplayQuestions } from "../../../web/display/DisplayQuestions";
-import { reqToBasePath } from "../../../web/utils";
+import { ssrUrql } from "../../../web/urql";
 
 interface Props {
-  dashboardForecasts: FrontendForecast[];
+  dashboard?: DashboardFragment;
-  dashboardItem: DashboardItem;
   numCols?: number;
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
+  const [ssrCache, client] = ssrUrql();
   const dashboardId = context.query.id as string;
   const numCols = Number(context.query.numCols);
 
-  const { dashboardItem, dashboardForecasts } =
+  const dashboard = (
-    await getDashboardForecastsByDashboardId({
+    await client.query(DashboardByIdDocument, { id: dashboardId }).toPromise()
-      dashboardId,
+  ).data?.result;
-      basePath: reqToBasePath(context.req), // required on server side to find the API endpoint
-    });
 
-  if (!dashboardItem) {
+  if (!dashboard) {
     context.res.statusCode = 404;
   }
 
   return {
     props: {
-      dashboardForecasts,
+      // reduntant: page component doesn't do graphql requests, but it's still nice/more consistent to have data in cache
-      dashboardItem,
+      urqlState: ssrCache.extractData(),
+      dashboard,
       numCols: !numCols ? null : numCols < 5 ? numCols : 4,
     },
   };
 };
 
-const EmbedDashboardPage: NextPage<Props> = ({
+const EmbedDashboardPage: NextPage<Props> = ({ dashboard, numCols }) => {
-  dashboardForecasts,
+  if (!dashboard) {
-  dashboardItem,
-  numCols,
-}) => {
-  if (!dashboardItem) {
     return <Error statusCode={404} />;
   }
 
@@ -57,9 +52,9 @@ const EmbedDashboardPage: NextPage<Props> = ({
             numCols || 3
           } gap-4 mb-6`}
         >
-          <DisplayForecasts
+          <DisplayQuestions
-            results={dashboardForecasts}
+            results={dashboard.questions}
-            numDisplay={dashboardForecasts.length}
+            numDisplay={dashboard.questions.length}
             showIdToggle={false}
           />
         </div>

src/pages/dashboards/index.tsx

@@ -1,24 +1,32 @@
-import axios from "axios";
 import { NextPage } from "next";
 import { useRouter } from "next/router";
+import { useMutation } from "urql";
 
+import { CreateDashboardDocument } from "../../web/dashboards/queries.generated";
 import { DashboardCreator } from "../../web/display/DashboardCreator";
 import { Layout } from "../../web/display/Layout";
 import { LineHeader } from "../../web/display/LineHeader";
 
 const DashboardsPage: NextPage = () => {
   const router = useRouter();
+  const [createDashboardResult, createDashboard] = useMutation(
+    CreateDashboardDocument
+  );
 
-  const handleSubmit = async (data) => {
+  const handleSubmit = async (data: any) => {
-    // Send to server to create
+    const result = await createDashboard({
-    // Get back the id
+      input: {
-    let response = await axios({
+        title: data.title,
-      url: "/api/create-dashboard-from-ids",
+        description: data.description,
-      method: "POST",
+        creator: data.creator,
-      headers: { "Content-Type": "application/json" },
+        ids: data.ids,
-      data: JSON.stringify(data),
+      },
-    }).then((res) => res.data);
+    });
-    await router.push(`/dashboards/view/${response.dashboardId}`);
+    const dashboardId = result?.data?.result?.dashboard?.id;
+    if (!dashboardId) {
+      throw new Error("Couldn't create a dashboard"); // TODO - toaster
+    }
+    await router.push(`/dashboards/view/${dashboardId}`);
   };
 
   return (

src/pages/dashboards/view/[id].tsx

@@ -2,57 +2,56 @@ import { GetServerSideProps, NextPage } from "next";
 import Error from "next/error";
 import Link from "next/link";
 
-import { DashboardItem } from "../../../backend/dashboards";
+import {
-import { DisplayForecasts } from "../../../web/display/DisplayForecasts";
+    DashboardByIdDocument, DashboardFragment
+} from "../../../web/dashboards/queries.generated";
+import { DisplayQuestions } from "../../../web/display/DisplayQuestions";
 import { InfoBox } from "../../../web/display/InfoBox";
 import { Layout } from "../../../web/display/Layout";
 import { LineHeader } from "../../../web/display/LineHeader";
-import { FrontendForecast } from "../../../web/platforms";
+import { ssrUrql } from "../../../web/urql";
-import { reqToBasePath } from "../../../web/utils";
-import { getDashboardForecastsByDashboardId } from "../../../web/worker/getDashboardForecasts";
 
 interface Props {
-  dashboardForecasts: FrontendForecast[];
+  dashboard?: DashboardFragment;
-  dashboardItem: DashboardItem;
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
+  const [ssrCache, client] = ssrUrql();
   const dashboardId = context.query.id as string;
 
-  const { dashboardForecasts, dashboardItem } =
+  const dashboard = (
-    await getDashboardForecastsByDashboardId({
+    await client.query(DashboardByIdDocument, { id: dashboardId }).toPromise()
-      dashboardId,
+  ).data?.result;
-      basePath: reqToBasePath(context.req), // required on server side to find the API endpoint
-    });
 
-  if (!dashboardItem) {
+  if (!dashboard) {
     context.res.statusCode = 404;
   }
 
   return {
     props: {
-      dashboardForecasts,
+      // reduntant: page component doesn't do graphql requests, but it's still nice/more consistent to have data in cache
-      dashboardItem,
+      urqlState: ssrCache.extractData(),
+      dashboard,
     },
   };
 };
 
-const DashboardMetadata: React.FC<{ dashboardItem: DashboardItem }> = ({
+const DashboardMetadata: React.FC<{ dashboard: DashboardFragment }> = ({
-  dashboardItem,
+  dashboard,
 }) => (
   <div>
-    {dashboardItem?.title ? (
+    {dashboard.title ? (
       <h1 className="text-4xl text-center text-gray-600 mt-2 mb-2">
-        {dashboardItem.title}
+        {dashboard.title}
       </h1>
     ) : null}
 
-    {dashboardItem && dashboardItem.creator ? (
+    {dashboard.creator ? (
       <p className="text-lg text-center text-gray-600 mt-2 mb-2">
         Created by:{" "}
-        {dashboardItem.creator === "Clay Graubard" ? (
+        {dashboard.creator === "Clay Graubard" ? (
           <>
             @
             <a
@@ -63,40 +62,38 @@ const DashboardMetadata: React.FC<{ dashboardItem: DashboardItem }> = ({
             </a>
           </>
         ) : (
-          dashboardItem.creator
+          dashboard.creator
         )}
       </p>
     ) : null}
 
-    {dashboardItem?.description ? (
+    {dashboard.description ? (
       <p className="text-lg text-center text-gray-600 mt-2 mb-2">
-        {dashboardItem.description}
+        {dashboard.description}
       </p>
     ) : null}
   </div>
 );
 
 /* Body */
-const ViewDashboardPage: NextPage<Props> = ({
+const ViewDashboardPage: NextPage<Props> = ({ dashboard }) => {
-  dashboardForecasts,
-  dashboardItem,
-}) => {
   return (
     <Layout page="view-dashboard">
       <div className="flex flex-col my-8 space-y-8">
-        {dashboardItem ? (
+        {dashboard ? (
-          <DashboardMetadata dashboardItem={dashboardItem} />
+          <>
-        ) : (
+            <DashboardMetadata dashboard={dashboard} />
-          <Error statusCode={404} />
-        )}
-
             <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
-          <DisplayForecasts
+              <DisplayQuestions
-            results={dashboardForecasts}
+                results={dashboard.questions}
-            numDisplay={dashboardForecasts.length}
+                numDisplay={dashboard.questions.length}
                 showIdToggle={false}
               />
             </div>
+          </>
+        ) : (
+          <Error statusCode={404} />
+        )}
 
         <div className="max-w-xl self-center">
           <InfoBox>

src/pages/index.tsx

@@ -1,7 +1,7 @@
 import { NextPage } from "next";
 import React from "react";
 
-import { displayForecastsWrapperForSearch } from "../web/display/displayForecastsWrappers";
+import { displayQuestionsWrapperForSearch } from "../web/display/displayQuestionsWrappers";
 import { Layout } from "../web/display/Layout";
 import { Props } from "../web/search/anySearchPage";
 import CommonDisplay from "../web/search/CommonDisplay";
@@ -10,7 +10,7 @@ export { getServerSideProps } from "../web/search/anySearchPage";
 
 const IndexPage: NextPage<Props> = (props) => {
   return (
-    <Layout page={"search"}>
+    <Layout page="search">
       <CommonDisplay
         {...props}
         hasSearchbar={true}
@@ -18,7 +18,7 @@ const IndexPage: NextPage<Props> = (props) => {
         hasAdvancedOptions={true}
         placeholder={"Find forecasts about..."}
         displaySeeMoreHint={true}
-        displayForecastsWrapper={displayForecastsWrapperForSearch}
+        displayQuestionsWrapper={displayQuestionsWrapperForSearch}
       />
     </Layout>
   );

src/pages/secretEmbed.tsx

@@ -4,18 +4,19 @@ import { GetServerSideProps, NextPage } from "next";
 import React from "react";
 
 import { platforms } from "../backend/platforms";
-import { DisplayForecast } from "../web/display/DisplayForecast";
+import { DisplayQuestion } from "../web/display/DisplayQuestion";
-import { FrontendForecast } from "../web/platforms";
+import { QuestionFragment, SearchDocument } from "../web/search/queries.generated";
-import searchAccordingToQueryData from "../web/worker/searchAccordingToQueryData";
+import { ssrUrql } from "../web/urql";
 
 interface Props {
-  results: FrontendForecast[];
+  results: QuestionFragment[];
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
-  let urlQuery = context.query; // this is an object, not a string which I have to parse!!
+  const [ssrCache, client] = ssrUrql();
+  let urlQuery = context.query;
 
   let initialQueryParameters = {
     query: "",
@@ -25,14 +26,24 @@ export const getServerSideProps: GetServerSideProps<Props> = async (
     ...urlQuery,
   };
 
-  let results: FrontendForecast[] = [];
+  let results: QuestionFragment[] = [];
-  if (initialQueryParameters.query != "") {
+  if (initialQueryParameters.query !== "") {
-    results = await searchAccordingToQueryData(initialQueryParameters, 1);
+    results = (
+      await client
+        .query(SearchDocument, {
+          input: {
+            ...initialQueryParameters,
+            limit: 1,
+          },
+        })
+        .toPromise()
+    ).data.result;
   }
 
   return {
     props: {
-      results: results,
+      urqlState: ssrCache.extractData(),
+      results,
     },
   };
 };
@@ -46,8 +57,8 @@ const SecretEmbedPage: NextPage<Props> = ({ results }) => {
         <div>
           <div id="secretEmbed">
             {result ? (
-              <DisplayForecast
+              <DisplayQuestion
-                forecast={result}
+                question={result}
                 showTimeStamp={true}
                 expandFooterToFullWidth={true}
               />

src/utils.ts

@@ -0,0 +1,8 @@
+export const shuffleArray = <T>(array: T[]): T[] => {
+  // See: https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array
+  for (let i = array.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+};

src/web/dashboards/queries.generated.tsx

@@ -0,0 +1,23 @@
+import * as Types from '../../graphql/types.generated';
+
+import { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core';
+import { QuestionFragmentDoc } from '../search/queries.generated';
+export type DashboardFragment = { __typename?: 'Dashboard', id: string, title: string, description: string, creator: string, questions: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: number, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> };
+
+export type DashboardByIdQueryVariables = Types.Exact<{
+  id: Types.Scalars['ID'];
+}>;
+
+
+export type DashboardByIdQuery = { __typename?: 'Query', result: { __typename?: 'Dashboard', id: string, title: string, description: string, creator: string, questions: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: number, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> } };
+
+export type CreateDashboardMutationVariables = Types.Exact<{
+  input: Types.CreateDashboardInput;
+}>;
+
+
+export type CreateDashboardMutation = { __typename?: 'Mutation', result: { __typename?: 'CreateDashboardResult', dashboard: { __typename?: 'Dashboard', id: string, title: string, description: string, creator: string, questions: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: number, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> } } };
+
+export const DashboardFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"Dashboard"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Dashboard"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"title"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"creator"}},{"kind":"Field","name":{"kind":"Name","value":"questions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Question"}}]}}]}},...QuestionFragmentDoc.definitions]} as unknown as DocumentNode<DashboardFragment, unknown>;
+export const DashboardByIdDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"DashboardById"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"dashboard"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Dashboard"}}]}}]}},...DashboardFragmentDoc.definitions]} as unknown as DocumentNode<DashboardByIdQuery, DashboardByIdQueryVariables>;
+export const CreateDashboardDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateDashboard"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"CreateDashboardInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"createDashboard"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"dashboard"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Dashboard"}}]}}]}}]}},...DashboardFragmentDoc.definitions]} as unknown as DocumentNode<CreateDashboardMutation, CreateDashboardMutationVariables>;

src/web/dashboards/queries.graphql

@@ -0,0 +1,23 @@
+fragment Dashboard on Dashboard {
+  id
+  title
+  description
+  creator
+  questions {
+    ...Question
+  }
+}
+
+query DashboardById($id: ID!) {
+  result: dashboard(id: $id) {
+    ...Dashboard
+  }
+}
+
+mutation CreateDashboard($input: CreateDashboardInput!) {
+  result: createDashboard(input: $input) {
+    dashboard {
+      ...Dashboard
+    }
+  }
+}

src/web/display/DashboardCreator.tsx

@@ -60,8 +60,10 @@ Your old input was: ${value}`;
 
         <InfoBox>
           You can find the necessary ids by toggling the advanced options in the
-          search, or by visiting{" "}
+          search, or by using{" "}
-          <a href="/api/all-forecasts">/api/all-forecasts</a>
+          <a href="/api/graphql?query=%7B%0A++questions%28first%3A+100%29+%7B%0A++++pageInfo+%7B%0A++++++hasNextPage%0A++++++endCursor%0A++++%7D%0A++++edges+%7B%0A++++++node+%7B%0A++++++++id%0A++++++++title%0A++++++++url%0A++++++%7D%0A++++%7D%0A++%7D%0A%7D">
+            GraphQL API
+          </a>
         </InfoBox>
       </div>
     </form>

src/web/display/DisplayForecasts.tsx

@@ -1,38 +0,0 @@
-import React from "react";
-
-import { FrontendForecast } from "../platforms";
-import { DisplayForecast } from "./DisplayForecast";
-
-interface Props {
-  results: FrontendForecast[];
-  numDisplay: number;
-  showIdToggle: boolean;
-}
-
-export const DisplayForecasts: React.FC<Props> = ({
-  results,
-  numDisplay,
-  showIdToggle,
-}) => {
-  if (!results) {
-    return <></>;
-  }
-  return (
-    <>
-      {results.slice(0, numDisplay).map((result) => (
-        /*let displayWithMetaculusCapture =
-          fuseSearchResult.item.platform == "Metaculus"
-            ? metaculusEmbed(fuseSearchResult.item)
-            : displayForecast({ ...fuseSearchResult.item });
-        */
-        <DisplayForecast
-          key={result.id}
-          forecast={result}
-          showTimeStamp={false}
-          expandFooterToFullWidth={false}
-          showIdToggle={showIdToggle}
-        />
-      ))}
-    </>
-  );
-};

src/web/display/DisplayOneQuestionForCapture.tsx

@@ -2,16 +2,16 @@ import domtoimage from "dom-to-image"; // https://github.com/tsayen/dom-to-image
 import { useEffect, useRef, useState } from "react";
 import { CopyToClipboard } from "react-copy-to-clipboard";
 
-import { FrontendForecast } from "../platforms";
+import { QuestionFragment } from "../search/queries.generated";
 import { uploadToImgur } from "../worker/uploadToImgur";
-import { DisplayForecast } from "./DisplayForecast";
+import { DisplayQuestion } from "./DisplayQuestion";
 
-function displayOneForecastInner(result: FrontendForecast, containerRef) {
+function displayOneQuestionInner(result: QuestionFragment, containerRef) {
   return (
     <div ref={containerRef}>
       {result ? (
-        <DisplayForecast
+        <DisplayQuestion
-          forecast={result}
+          question={result}
           showTimeStamp={true}
           expandFooterToFullWidth={true}
         />
@@ -168,10 +168,10 @@ let generateMetaculusSource = (result, hasDisplayBeenCaptured) => {
 };
 
 interface Props {
-  result: FrontendForecast;
+  result: QuestionFragment;
 }
 
-export const DisplayOneForecastForCapture: React.FC<Props> = ({ result }) => {
+export const DisplayOneQuestionForCapture: React.FC<Props> = ({ result }) => {
   const [hasDisplayBeenCaptured, setHasDisplayBeenCaptured] = useState(false);
 
   useEffect(() => {
@@ -226,7 +226,7 @@ export const DisplayOneForecastForCapture: React.FC<Props> = ({ result }) => {
   return (
     <div className="grid grid-cols-1 md:grid-cols-2 gap-4 w-full justify-center">
       <div className="flex col-span-1 items-center justify-center">
-        {displayOneForecastInner(result, containerRef)}
+        {displayOneQuestionInner(result, containerRef)}
       </div>
       <div className="flex col-span-1 items-center justify-center">
         {generateCaptureButton(result, onCaptureButtonClick)}

src/web/display/DisplayQuestion/QuestionFooter.tsx

@@ -105,7 +105,7 @@ const getCurrencySymbolIfNeeded = ({
 
 const showFirstQualityIndicator = ({
   numforecasts,
-  timestamp,
+  lastUpdated,
   showTimeStamp,
   qualityindicators,
 }) => {
@@ -124,7 +124,7 @@ const showFirstQualityIndicator = ({
           <circle cx="4" cy="4" r="4" fill="rgb(29, 78, 216)" />
         </svg>
         {`Last updated: ${
-          timestamp && !!timestamp.slice ? timestamp.slice(0, 10) : "unknown"
+          lastUpdated ? lastUpdated.toISOString().slice(0, 10) : "unknown"
         }`}
       </span>
     );
@@ -135,13 +135,13 @@ const showFirstQualityIndicator = ({
 
 const displayQualityIndicators: React.FC<{
   numforecasts: number;
-  timestamp: number;
+  lastUpdated: Date;
   showTimeStamp: boolean;
   qualityindicators: any;
   platform: string; // id string - e.g. "goodjudgment", not "Good Judgment"
 }> = ({
   numforecasts,
-  timestamp,
+  lastUpdated,
   showTimeStamp,
   qualityindicators,
   platform,
@@ -151,7 +151,7 @@ const displayQualityIndicators: React.FC<{
     <div className="text-sm">
       {showFirstQualityIndicator({
         numforecasts,
-        timestamp,
+        lastUpdated,
         showTimeStamp,
         qualityindicators,
       })}
@@ -238,18 +238,18 @@ interface Props {
   platformLabel: string;
   numforecasts: any;
   qualityindicators: any;
-  timestamp: any;
+  lastUpdated: Date;
   showTimeStamp: boolean;
   expandFooterToFullWidth: boolean;
 }
 
-export const ForecastFooter: React.FC<Props> = ({
+export const QuestionFooter: React.FC<Props> = ({
   stars,
   platform,
   platformLabel,
   numforecasts,
   qualityindicators,
-  timestamp,
+  lastUpdated,
   showTimeStamp,
   expandFooterToFullWidth,
 }) => {
@@ -288,7 +288,7 @@ export const ForecastFooter: React.FC<Props> = ({
       >
         {displayQualityIndicators({
           numforecasts,
-          timestamp,
+          lastUpdated,
           showTimeStamp,
           qualityindicators,
           platform,

src/web/display/DisplayQuestion/index.tsx

@@ -1,9 +1,9 @@
 import { FaRegClipboard } from "react-icons/fa";
 import ReactMarkdown from "react-markdown";
 
-import { FrontendForecast } from "../../platforms";
+import { QuestionFragment } from "../../search/queries.generated";
 import { Card } from "../Card";
-import { ForecastFooter } from "./ForecastFooter";
+import { QuestionFooter } from "./QuestionFooter";
 
 const truncateText = (length: number, text: string): string => {
   if (!text) {
@@ -254,13 +254,14 @@ const CopyText: React.FC<{ text: string; displayText: string }> = ({
   </div>
 );
 
-const LastUpdated: React.FC<{ timestamp: string }> = ({ timestamp }) => (
+const LastUpdated: React.FC<{ timestamp: Date }> = ({ timestamp }) => (
   <div className="flex items-center">
     <svg className="mt-1" height="10" width="16">
       <circle cx="4" cy="4" r="4" fill="rgb(29, 78, 216)" />
     </svg>
     <span className="text-gray-600">
-      Last updated: {timestamp ? timestamp.slice(0, 10) : "unknown"}
+      Last updated:{" "}
+      {timestamp ? timestamp.toISOString().slice(0, 10) : "unknown"}
     </span>
   </div>
 );
@@ -268,22 +269,21 @@ const LastUpdated: React.FC<{ timestamp: string }> = ({ timestamp }) => (
 // Main component
 
 interface Props {
-  forecast: FrontendForecast;
+  question: QuestionFragment;
   showTimeStamp: boolean;
   expandFooterToFullWidth: boolean;
   showIdToggle?: boolean;
 }
 
-export const DisplayForecast: React.FC<Props> = ({
+export const DisplayQuestion: React.FC<Props> = ({
-  forecast: {
+  question: {
     id,
     title,
     url,
     platform,
-    platformLabel,
     description,
     options,
-    qualityindicators,
+    qualityIndicators,
     timestamp,
     visualization,
   },
@@ -291,8 +291,9 @@ export const DisplayForecast: React.FC<Props> = ({
   expandFooterToFullWidth,
   showIdToggle,
 }) => {
+  const lastUpdated = new Date(timestamp * 1000);
   const displayTimestampAtBottom =
-    checkIfDisplayTimeStampAtBottom(qualityindicators);
+    checkIfDisplayTimeStampAtBottom(qualityIndicators);
 
   const yesNoOptions =
     options.length === 2 &&
@@ -332,7 +333,7 @@ export const DisplayForecast: React.FC<Props> = ({
                     showTimeStamp && !displayTimestampAtBottom ? "sm:block" : ""
                   }`}
                 >
-                  <LastUpdated timestamp={timestamp} />
+                  <LastUpdated timestamp={lastUpdated} />
                 </div>
               </div>
             )}
@@ -344,18 +345,18 @@ export const DisplayForecast: React.FC<Props> = ({
                     showTimeStamp && !displayTimestampAtBottom ? "sm:block" : ""
                   } ml-6`}
                 >
-                  <LastUpdated timestamp={timestamp} />
+                  <LastUpdated timestamp={lastUpdated} />
                 </div>
               </div>
             )}
 
-            {platform !== "guesstimate" && options.length < 3 && (
+            {platform.id !== "guesstimate" && options.length < 3 && (
               <div className="text-gray-500">
                 <DisplayMarkdown description={description} />
               </div>
             )}
 
-            {platform === "guesstimate" && (
+            {platform.id === "guesstimate" && (
               <img
                 className="rounded-sm"
                 src={visualization}
@@ -369,16 +370,16 @@ export const DisplayForecast: React.FC<Props> = ({
             } self-center`}
           >
             {/* This one is exclusively for mobile*/}
-            <LastUpdated timestamp={timestamp} />
+            <LastUpdated timestamp={lastUpdated} />
           </div>
           <div className="w-full">
-            <ForecastFooter
+            <QuestionFooter
-              stars={qualityindicators.stars}
+              stars={qualityIndicators.stars}
-              platform={platform}
+              platform={platform.id}
-              platformLabel={platformLabel || platform} // author || platformLabel,
+              platformLabel={platform.label}
-              numforecasts={qualityindicators.numforecasts}
+              numforecasts={qualityIndicators.numForecasts}
-              qualityindicators={qualityindicators}
+              qualityindicators={qualityIndicators}
-              timestamp={timestamp}
+              lastUpdated={lastUpdated}
               showTimeStamp={showTimeStamp && displayTimestampAtBottom}
               expandFooterToFullWidth={expandFooterToFullWidth}
             />

src/web/display/DisplayQuestions.tsx

@@ -0,0 +1,33 @@
+import React from "react";
+
+import { QuestionFragment } from "../search/queries.generated";
+import { DisplayQuestion } from "./DisplayQuestion";
+
+interface Props {
+  results: QuestionFragment[];
+  numDisplay: number;
+  showIdToggle: boolean;
+}
+
+export const DisplayQuestions: React.FC<Props> = ({
+  results,
+  numDisplay,
+  showIdToggle,
+}) => {
+  if (!results) {
+    return <></>;
+  }
+  return (
+    <>
+      {results.slice(0, numDisplay).map((result) => (
+        <DisplayQuestion
+          key={result.id}
+          question={result}
+          showTimeStamp={false}
+          expandFooterToFullWidth={false}
+          showIdToggle={showIdToggle}
+        />
+      ))}
+    </>
+  );
+};

src/web/display/displayQuestionsWrappers.tsx

@@ -1,14 +1,14 @@
-import { DisplayForecasts } from "./DisplayForecasts";
+import { DisplayOneQuestionForCapture } from "./DisplayOneQuestionForCapture";
-import { DisplayOneForecastForCapture } from "./DisplayOneForecastForCapture";
+import { DisplayQuestions } from "./DisplayQuestions";
 
-export function displayForecastsWrapperForSearch({
+export function displayQuestionsWrapperForSearch({
   results,
   numDisplay,
   showIdToggle,
 }) {
   return (
     <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
-      <DisplayForecasts
+      <DisplayQuestions
         results={results || []}
         numDisplay={numDisplay}
         showIdToggle={showIdToggle}
@@ -17,13 +17,13 @@ export function displayForecastsWrapperForSearch({
   );
 }
 
-export function displayForecastsWrapperForCapture({
+export function displayQuestionsWrapperForCapture({
   results,
   whichResultToDisplayAndCapture,
 }) {
   return (
     <div className="grid grid-cols-1 w-full justify-center">
-      <DisplayOneForecastForCapture
+      <DisplayOneQuestionForCapture
         result={results[whichResultToDisplayAndCapture]}
       />
     </div>

src/web/hooks.ts

@@ -13,3 +13,15 @@ export const useNoInitialEffect = (
     return effect();
   }, deps);
 };
+
+export const useIsFirstRender = (): boolean => {
+  const isFirst = React.useRef(true);
+
+  if (isFirst.current) {
+    isFirst.current = false;
+
+    return true;
+  }
+
+  return isFirst.current;
+};

src/web/platforms.ts

@@ -1,21 +0,0 @@
-import { Forecast, PlatformConfig } from "../backend/platforms";
-
-export type FrontendForecast = Forecast & {
-  platformLabel: string;
-  visualization?: any;
-};
-
-// ok on client side
-export const addLabelsToForecasts = (
-  forecasts: Forecast[],
-  platformsConfig: PlatformConfig[]
-): FrontendForecast[] => {
-  const platformNameToLabel = Object.fromEntries(
-    platformsConfig.map((platform) => [platform.name, platform.label])
-  );
-
-  return forecasts.map((result) => ({
-    ...result,
-    platformLabel: platformNameToLabel[result.platform] || result.platform,
-  }));
-};

src/web/search/CommonDisplay.tsx

@@ -1,14 +1,14 @@
 import { useRouter } from "next/router";
-import React, { Fragment, useState } from "react";
+import React, { Fragment, useMemo, useState } from "react";
+import { useQuery } from "urql";
 
 import { ButtonsForStars } from "../display/ButtonsForStars";
 import { MultiSelectPlatform } from "../display/MultiSelectPlatform";
 import { QueryForm } from "../display/QueryForm";
 import { SliderElement } from "../display/SliderElement";
-import { useNoInitialEffect } from "../hooks";
+import { useIsFirstRender, useNoInitialEffect } from "../hooks";
-import { FrontendForecast } from "../platforms";
-import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
 import { Props as AnySearchPageProps, QueryParameters } from "./anySearchPage";
+import { QuestionFragment, SearchDocument } from "./queries.generated";
 
 interface Props extends AnySearchPageProps {
   hasSearchbar: boolean;
@@ -16,8 +16,8 @@ interface Props extends AnySearchPageProps {
   hasAdvancedOptions: boolean;
   placeholder: string;
   displaySeeMoreHint: boolean;
-  displayForecastsWrapper: (opts: {
+  displayQuestionsWrapper: (opts: {
-    results: FrontendForecast[];
+    results: QuestionFragment[];
     numDisplay: number;
     whichResultToDisplayAndCapture: number;
     showIdToggle: boolean;
@@ -27,7 +27,6 @@ interface Props extends AnySearchPageProps {
 /* Body */
 const CommonDisplay: React.FC<Props> = ({
   defaultResults,
-  initialResults,
   initialQueryParameters,
   defaultQueryParameters,
   initialNumDisplay,
@@ -38,10 +37,11 @@ const CommonDisplay: React.FC<Props> = ({
   hasAdvancedOptions,
   placeholder,
   displaySeeMoreHint,
-  displayForecastsWrapper,
+  displayQuestionsWrapper,
 }) => {
-  const router = useRouter();
   /* States */
+  const router = useRouter();
+  const isFirstRender = useIsFirstRender();
 
   const [queryParameters, setQueryParameters] = useState<QueryParameters>(
     initialQueryParameters
@@ -52,60 +52,67 @@ const CommonDisplay: React.FC<Props> = ({
   // used to distinguish numDisplay updates which force search and don't force search, see effects below
   const [forceSearch, setForceSearch] = useState(0);
 
-  const [results, setResults] = useState(initialResults);
   const [advancedOptions, showAdvancedOptions] = useState(false);
   const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] =
     useState(0);
   const [showIdToggle, setShowIdToggle] = useState(false);
 
-  /* Functions which I want to have access to the Home namespace */
+  const [typing, setTyping] = useState(false);
-  // I don't want to create an "defaultResults" object for each search.
-  async function executeSearchOrAnswerWithDefaultResults() {
-    const queryData = {
-      ...queryParameters,
-      numDisplay,
-    };
 
-    const filterManually = (
+  // must match the query from anySearchPage.ts getServerSideProps
-      queryData: QueryParameters,
+  const [queryResults, reexecuteQuery] = useQuery({
-      results: FrontendForecast[]
+    query: SearchDocument,
-    ) => {
+    variables: {
-      if (
+      input: {
-        queryData.forecastingPlatforms &&
+        ...queryParameters,
-        queryData.forecastingPlatforms.length > 0
+        limit: numDisplay,
-      ) {
+      },
-        results = results.filter((result) =>
+    },
-          queryData.forecastingPlatforms.includes(result.platform)
+    pause: !isFirstRender,
-        );
+    // note that if we don't force cache-only on SSR then queryResults.fetching is true which leads to an empty page
-      }
+    requestPolicy: isFirstRender ? "cache-only" : "network-only",
-      if (queryData.starsThreshold === 4) {
+  });
-        results = results.filter(
-          (result) => result.qualityindicators.stars >= 4
-        );
-      }
-      if (queryData.forecastsThreshold) {
-        // results = results.filter(result => (result.qualityindicators && result.qualityindicators.numforecasts > forecastsThreshold))
-      }
-      return results;
-    };
 
   const queryIsEmpty =
-      !queryData || queryData.query == "" || queryData.query == undefined;
+    queryParameters.query === undefined || queryParameters.query === "";
 
-    const results = queryIsEmpty
+  const results: QuestionFragment[] = useMemo(() => {
-      ? filterManually(queryData, defaultResults)
+    if (typing || queryResults.fetching) return []; // TODO - return results but show spinner or darken out all cards?
-      : await searchAccordingToQueryData(queryData, numDisplay);
 
-    setResults(results);
+    if (queryIsEmpty) {
+      const filterManually = (results: QuestionFragment[]) => {
+        let filteredResults = [...results];
+        if (
+          queryParameters.forecastingPlatforms &&
+          queryParameters.forecastingPlatforms.length > 0
+        ) {
+          filteredResults = filteredResults.filter((result) =>
+            queryParameters.forecastingPlatforms.includes(result.platform.id)
+          );
         }
+        if (queryParameters.starsThreshold === 4) {
+          filteredResults = filteredResults.filter(
+            (result) => result.qualityIndicators.stars >= 4
+          );
+        }
+        if (queryParameters.forecastsThreshold) {
+          // TODO / FIXME / remove?
+        }
+        return filteredResults;
+      };
+      return filterManually(defaultResults);
+    } else {
+      return queryResults.data?.result || [];
+    }
+  }, [queryResults.data, queryParameters]);
 
-  // I don't want the function which display forecasts (displayForecasts) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home.
+  // I don't want the component which display questions (DisplayQuestions) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home.
-  const getInfoToDisplayForecastsFunction = () => {
+  const getInfoToDisplayQuestionsFunction = () => {
     const numDisplayRounded =
       numDisplay % 3 != 0
         ? numDisplay + (3 - (Math.round(numDisplay) % 3))
         : numDisplay;
-    return displayForecastsWrapper({
+    return displayQuestionsWrapper({
       results,
       numDisplay: numDisplayRounded,
       whichResultToDisplayAndCapture,
@@ -145,10 +152,13 @@ const CommonDisplay: React.FC<Props> = ({
   useNoInitialEffect(updateRoute, [numDisplay]);
 
   useNoInitialEffect(() => {
-    setResults([]);
+    setTyping(true);
-    const newTimeoutId = setTimeout(() => {
+    const newTimeoutId = setTimeout(async () => {
       updateRoute();
-      executeSearchOrAnswerWithDefaultResults();
+      if (!queryIsEmpty) {
+        reexecuteQuery();
+      }
+      setTyping(false);
     }, 500);
 
     // avoid sending results if user has not stopped typing.
@@ -307,10 +317,10 @@ const CommonDisplay: React.FC<Props> = ({
         </div>
       ) : null}
 
-      <div>{getInfoToDisplayForecastsFunction()}</div>
+      <div>{getInfoToDisplayQuestionsFunction()}</div>
 
       {displaySeeMoreHint &&
-      (!results || (results.length != 0 && numDisplay < results.length)) ? (
+      (!results || (results.length !== 0 && numDisplay < results.length)) ? (
         <div>
           <p className="mt-4 mb-4">
             {"Can't find what you were looking for?"}
@@ -336,7 +346,7 @@ const CommonDisplay: React.FC<Props> = ({
         </div>
       ) : null}
 
-      <br></br>
+      <br />
     </Fragment>
   );
 };

src/web/search/anySearchPage.tsx

@@ -1,9 +1,8 @@
 import { GetServerSideProps } from "next";
 
-import { getFrontpage } from "../../backend/frontpage";
 import { getPlatformsConfig, PlatformConfig, platforms } from "../../backend/platforms";
-import { addLabelsToForecasts, FrontendForecast } from "../platforms";
+import { ssrUrql } from "../urql";
-import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
+import { FrontpageDocument, QuestionFragment, SearchDocument } from "./queries.generated";
 
 /* Common code for / and /capture */
 
@@ -15,8 +14,7 @@ export interface QueryParameters {
 }
 
 export interface Props {
-  defaultResults: FrontendForecast[];
+  defaultResults: QuestionFragment[];
-  initialResults: FrontendForecast[];
   initialQueryParameters: QueryParameters;
   defaultQueryParameters: QueryParameters;
   initialNumDisplay: number;
@@ -27,6 +25,7 @@ export interface Props {
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
+  const [ssrCache, client] = ssrUrql();
   const urlQuery = context.query;
 
   const platformsConfig = getPlatformsConfig({ withGuesstimate: true });
@@ -61,28 +60,32 @@ export const getServerSideProps: GetServerSideProps<Props> = async (
   const defaultNumDisplay = 21;
   const initialNumDisplay = Number(urlQuery.numDisplay) || defaultNumDisplay;
 
-  const defaultResults = addLabelsToForecasts(
+  const defaultResults = (await client.query(FrontpageDocument).toPromise())
-    await getFrontpage(),
+    .data.result;
-    platformsConfig
-  );
 
-  const initialResults =
+  if (
     !!initialQueryParameters &&
     initialQueryParameters.query != "" &&
     initialQueryParameters.query != undefined
-      ? await searchAccordingToQueryData(
+  ) {
-          initialQueryParameters,
+    // must match the query from CommonDisplay
-          initialNumDisplay
+    await client
-        )
+      .query(SearchDocument, {
-      : defaultResults;
+        input: {
+          ...initialQueryParameters,
+          limit: initialNumDisplay,
+        },
+      })
+      .toPromise();
+  }
 
   return {
     props: {
+      urqlState: ssrCache.extractData(),
       initialQueryParameters,
       defaultQueryParameters,
       initialNumDisplay,
       defaultNumDisplay,
-      initialResults,
       defaultResults,
       platformsConfig,
     },

src/web/search/queries.generated.tsx

@@ -0,0 +1,20 @@
+import * as Types from '../../graphql/types.generated';
+
+import { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core';
+export type QuestionFragment = { __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: number, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } };
+
+export type FrontpageQueryVariables = Types.Exact<{ [key: string]: never; }>;
+
+
+export type FrontpageQuery = { __typename?: 'Query', result: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: number, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> };
+
+export type SearchQueryVariables = Types.Exact<{
+  input: Types.SearchInput;
+}>;
+
+
+export type SearchQuery = { __typename?: 'Query', result: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: number, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> };
+
+export const QuestionFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"Question"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Question"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"url"}},{"kind":"Field","name":{"kind":"Name","value":"title"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"timestamp"}},{"kind":"Field","name":{"kind":"Name","value":"options"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"probability"}}]}},{"kind":"Field","name":{"kind":"Name","value":"platform"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"label"}}]}},{"kind":"Field","name":{"kind":"Name","value":"qualityIndicators"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"stars"}},{"kind":"Field","name":{"kind":"Name","value":"numForecasts"}}]}},{"kind":"Field","name":{"kind":"Name","value":"visualization"}}]}}]} as unknown as DocumentNode<QuestionFragment, unknown>;
+export const FrontpageDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"Frontpage"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"frontpage"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Question"}}]}}]}},...QuestionFragmentDoc.definitions]} as unknown as DocumentNode<FrontpageQuery, FrontpageQueryVariables>;
+export const SearchDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"Search"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SearchInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"searchQuestions"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Question"}}]}}]}},...QuestionFragmentDoc.definitions]} as unknown as DocumentNode<SearchQuery, SearchQueryVariables>;

src/web/search/queries.graphql

@@ -0,0 +1,32 @@
+fragment Question on Question {
+  id
+  url
+  title
+  description
+  timestamp
+  options {
+    name
+    probability
+  }
+  platform {
+    id
+    label
+  }
+  qualityIndicators {
+    stars
+    numForecasts
+  }
+  visualization
+}
+
+query Frontpage {
+  result: frontpage {
+    ...Question
+  }
+}
+
+query Search($input: SearchInput!) {
+  result: searchQuestions(input: $input) {
+    ...Question
+  }
+}

src/web/urql.ts

@@ -0,0 +1,40 @@
+import { initUrqlClient, SSRExchange } from "next-urql";
+import { cacheExchange, dedupExchange, fetchExchange, ssrExchange } from "urql";
+import customScalarsExchange from "urql-custom-scalars-exchange";
+
+import schema from "../graphql/introspection.json";
+import { getBasePath } from "./utils";
+
+export const graphqlEndpoint = `${getBasePath()}/api/graphql`;
+
+const scalarsExchange = customScalarsExchange({
+  // Types don't match for some reason.
+  // Related:
+  // - https://github.com/apollographql/apollo-tooling/issues/1491
+  // - https://spectrum.chat/urql/help/schema-property-kind-is-missing-in-type~29c8f416-068c-485a-adf1-935686b99d05
+  schema: schema as any,
+  scalars: {
+    /* not compatible with next.js serialization limitations, unfortunately */
+    // Date(value: number) {
+    //   return new Date(value * 1000);
+    // },
+  },
+});
+
+export const getUrqlClientOptions = (ssr: SSRExchange) => ({
+  url: graphqlEndpoint,
+  exchanges: [
+    dedupExchange,
+    scalarsExchange,
+    cacheExchange,
+    ssr,
+    fetchExchange,
+  ],
+});
+
+// for getServerSideProps/getStaticProps only
+export const ssrUrql = () => {
+  const ssrCache = ssrExchange({ isClient: false });
+  const client = initUrqlClient(getUrqlClientOptions(ssrCache), false);
+  return [ssrCache, client] as const;
+};

src/web/utils.ts

@@ -1,11 +1,12 @@
-import { IncomingMessage } from "http";
+export const getBasePath = () => {
-
-export const reqToBasePath = (req: IncomingMessage) => {
   if (process.env.NEXT_PUBLIC_VERCEL_URL) {
-    console.log(process.env.NEXT_PUBLIC_VERCEL_URL);
     return `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`;
   }
 
-  // we could just hardcode http://localhost:3000 here, but then `next dev -p <CUSTOM_PORT>` would break
+  // can be used for local development if you prefer non-default port
-  return "http://" + req.headers.host;
+  if (process.env.NEXT_PUBLIC_SITE_URL) {
+    return process.env.NEXT_PUBLIC_SITE_URL;
+  }
+
+  return "http://localhost:3000";
 };

src/web/worker/getDashboardForecasts.ts

@@ -1,74 +0,0 @@
-import axios from "axios";
-
-import { DashboardItem } from "../../backend/dashboards";
-import { Forecast, getPlatformsConfig } from "../../backend/platforms";
-import { addLabelsToForecasts, FrontendForecast } from "../platforms";
-
-export async function getDashboardForecastsByDashboardId({
-  dashboardId,
-  basePath,
-}: {
-  dashboardId: string;
-  basePath?: string;
-}): Promise<{
-  dashboardForecasts: FrontendForecast[];
-  dashboardItem: DashboardItem;
-}> {
-  console.log("getDashboardForecastsByDashboardId: ");
-  if (typeof window === undefined && !basePath) {
-    throw new Error("`basePath` option is required on server side");
-  }
-
-  let dashboardForecasts: Forecast[] = [];
-  let dashboardItem: DashboardItem | null = null;
-  try {
-    let { data } = await axios({
-      url: `${basePath || ""}/api/dashboard-by-id`,
-      method: "post",
-      data: {
-        id: dashboardId,
-      },
-    });
-    console.log(data);
-
-    dashboardForecasts = data.dashboardContents;
-    dashboardItem = data.dashboardItem as DashboardItem;
-  } catch (error) {
-    console.log(error);
-  } finally {
-    const labeledDashboardForecasts = addLabelsToForecasts(
-      dashboardForecasts,
-      getPlatformsConfig({ withGuesstimate: false })
-    );
-
-    return {
-      dashboardForecasts: labeledDashboardForecasts,
-      dashboardItem,
-    };
-  }
-}
-
-export async function createDashboard(payload) {
-  let data = { dashboardId: null };
-  try {
-    let { title, description, ids, creator, extra } = payload;
-    console.log(payload);
-    let response = await axios({
-      url: "/api/create-dashboard-from-ids",
-      method: "post",
-      data: {
-        title: title || "",
-        description: description || "",
-        ids: ids,
-        creator: creator || "",
-        extra: [],
-      },
-    });
-    data = response.data;
-    console.log(data);
-  } catch (error) {
-    console.log(error);
-  } finally {
-    return data;
-  }
-}

src/web/worker/searchAccordingToQueryData.ts

@@ -1,52 +0,0 @@
-import { FrontendForecast } from "../platforms";
-import { QueryParameters } from "../search/anySearchPage";
-import searchGuesstimate from "./searchGuesstimate";
-import searchWithAlgolia from "./searchWithAlgolia";
-
-export default async function searchAccordingToQueryData(
-  queryData: QueryParameters,
-  limit: number
-): Promise<FrontendForecast[]> {
-  let results: FrontendForecast[] = [];
-
-  try {
-    // defs
-    let query = queryData.query == undefined ? "" : queryData.query;
-    if (query == "") return [];
-    let forecastsThreshold = queryData.forecastsThreshold;
-    let starsThreshold = queryData.starsThreshold;
-    let platformsIncludeGuesstimate =
-      queryData.forecastingPlatforms.includes("guesstimate") &&
-      starsThreshold <= 1;
-
-    // preparation
-    let unawaitedAlgoliaResponse = searchWithAlgolia({
-      queryString: query,
-      hitsPerPage: limit + 50,
-      starsThreshold,
-      filterByPlatforms: queryData.forecastingPlatforms,
-      forecastsThreshold,
-    });
-
-    // consider the guesstimate and the non-guesstimate cases separately.
-    if (platformsIncludeGuesstimate) {
-      let responses = await Promise.all([
-        unawaitedAlgoliaResponse,
-        searchGuesstimate(query),
-      ]); // faster than two separate requests
-      let responsesNotGuesstimate = responses[0];
-      let responsesGuesstimate = responses[1];
-      results = [...responsesNotGuesstimate, ...responsesGuesstimate];
-      //results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
-    } else {
-      results = await unawaitedAlgoliaResponse;
-    }
-
-    return results;
-  } catch (error) {
-    console.log(error);
-  } finally {
-    console.log(results);
-    return results;
-  }
-}

src/web/worker/searchGuesstimate.ts

@@ -1,18 +1,18 @@
 /* Imports */
 import axios from "axios";
 
-import { FrontendForecast } from "../platforms";
+import { AlgoliaQuestion } from "../../backend/utils/algolia";
 
 /* Definitions */
-let urlEndPoint =
+const urlEndPoint =
   "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c";
 
 /* Body */
 
 export default async function searchGuesstimate(
   query
-): Promise<FrontendForecast[]> {
+): Promise<AlgoliaQuestion[]> {
-  let response = await axios({
+  const response = await axios({
     url: urlEndPoint,
     // credentials: "omit",
     headers: {
@@ -21,8 +21,6 @@ export default async function searchGuesstimate(
       "Accept-Language": "en-US,en;q=0.5",
       "content-type": "application/x-www-form-urlencoded",
     },
-    // referrer:
-    //   "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
     data: `{\"params\":\"query=${query.replace(
       / /g,
       "%20"
@@ -31,33 +29,36 @@ export default async function searchGuesstimate(
   });
 
   const models: any[] = response.data.hits;
-  const mappedModels: FrontendForecast[] = models.map((model, index) => {
+  const mappedModels: AlgoliaQuestion[] = models.map((model, index) => {
-    let description = model.description
+    const description = model.description
       ? model.description.replace(/\n/g, " ").replace(/  /g, " ")
       : "";
-    let stars = description.length > 250 ? 2 : 1;
+    const stars = description.length > 250 ? 2 : 1;
-    return {
+    const q: AlgoliaQuestion = {
       id: `guesstimate-${model.id}`,
       title: model.name,
       url: `https://www.getguesstimate.com/models/${model.id}`,
       timestamp: model.created_at, // TODO - check that format matches
       platform: "guesstimate",
-      platformLabel: "Guesstimate",
+      description,
-      description: description,
       options: [],
       qualityindicators: {
         stars: stars,
         numforecasts: 1,
         numforecasters: 1,
       },
+      stars,
+      extra: {
         visualization: model.big_screenshot,
-      ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
+      },
+      // ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
     };
+    return q;
   });
 
   // filter for duplicates. Surprisingly common.
   let uniqueTitles = [];
-  let uniqueModels: FrontendForecast[] = [];
+  let uniqueModels: AlgoliaQuestion[] = [];
   for (let model of mappedModels) {
     if (!uniqueTitles.includes(model.title) && !model.title.includes("copy")) {
       uniqueModels.push(model);

src/web/worker/searchWithAlgolia.ts

@@ -1,6 +1,6 @@
 import algoliasearch from "algoliasearch";
 
-import { FrontendForecast } from "../platforms";
+import { AlgoliaQuestion } from "../../backend/utils/algolia";
 
 const client = algoliasearch(
   process.env.NEXT_PUBLIC_ALGOLIA_APP_ID,
@@ -13,22 +13,21 @@ let buildFilter = ({
   filterByPlatforms,
   forecastsThreshold,
 }) => {
-  let starsFilter = starsThreshold
+  const starsFilter = starsThreshold
     ? `qualityindicators.stars >= ${starsThreshold}`
     : null;
-  let platformsFilter = filterByPlatforms
+  const platformsFilter = filterByPlatforms
     ? filterByPlatforms.map((platform) => `platform:"${platform}"`).join(" OR ")
     : null;
-  console.log(platformsFilter);
+  const numForecastsFilter =
-  // let numForecastsFilter = forecastsThreshold ? `has_numforecasts:true AND qualityindicators.numforecasts >= ${forecastsThreshold}` : null
-  let numForecastsFilter =
     forecastsThreshold > 0
       ? `qualityindicators.numforecasts >= ${forecastsThreshold}`
       : null;
-  let finalFilter = [starsFilter, platformsFilter, numForecastsFilter]
+  const finalFilter = [starsFilter, platformsFilter, numForecastsFilter]
     .filter((f) => f != null)
     .map((f) => `( ${f} )`)
     .join(" AND ");
+
   console.log(
     "searchWithAlgolia.js/searchWithAlgolia/buildFilter",
     finalFilter
@@ -51,18 +50,6 @@ let buildFacetFilter = ({ filterByPlatforms }) => {
   return platformsFilter;
 };
 
-let normalizeArray = (array) => {
-  if (array.length == 0) {
-    return [];
-  }
-  let mean = array.reduce((a, b) => a + b) / array.length;
-  let sd = Math.sqrt(
-    array.map((x) => Math.pow(x - mean, 2)).reduce((a, b) => a + b)
-  );
-  let normalizedArray = array.map((result) => (result - sd) / mean);
-  return normalizedArray;
-};
-
 let noExactMatch = (queryString, result) => {
   queryString = queryString.toLowerCase();
   let title = result.title.toLowerCase();
@@ -75,6 +62,14 @@ let noExactMatch = (queryString, result) => {
   );
 };
 
+interface SearchOpts {
+  queryString: string;
+  hitsPerPage?: number;
+  starsThreshold: number;
+  filterByPlatforms: string[];
+  forecastsThreshold: number;
+}
+
 // only query string
 export default async function searchWithAlgolia({
   queryString,
@@ -82,8 +77,8 @@ export default async function searchWithAlgolia({
   starsThreshold,
   filterByPlatforms,
   forecastsThreshold,
-}): Promise<FrontendForecast[]> {
+}: SearchOpts): Promise<AlgoliaQuestion[]> {
-  let response = await index.search<FrontendForecast>(queryString, {
+  const response = await index.search<AlgoliaQuestion>(queryString, {
     hitsPerPage,
     filters: buildFilter({
       starsThreshold,
@@ -93,7 +88,7 @@ export default async function searchWithAlgolia({
     //facetFilters: buildFacetFilter({filterByPlatforms}),
     getRankingInfo: true,
   });
-  let results: FrontendForecast[] = response.hits;
+  let results = response.hits;
 
   let recursionError = ["metaforecast", "metaforecasts", "metaforecasting"];
   if (
@@ -103,10 +98,10 @@ export default async function searchWithAlgolia({
     results = [
       {
         id: "not-found",
+        objectID: "not-found",
         title: "No search results match your query",
         url: "https://metaforecast.org",
         platform: "metaforecast",
-        platformLabel: "Metaforecast",
         description: "Maybe try a broader query?",
         options: [
           {
@@ -121,24 +116,23 @@ export default async function searchWithAlgolia({
           },
         ],
         timestamp: `${new Date().toISOString().slice(0, 10)}`,
+        stars: 5, // legacy
         qualityindicators: {
           numforecasts: 1,
           numforecasters: 1,
           stars: 5,
         },
-        // noExactSearchResults: true,
+        extra: {},
-        // optionsstringforsearch: "Yes, No",
-        // has_numforecasts: true,
       },
     ];
   } else if (recursionError.includes(queryString.toLowerCase())) {
     results = [
       {
         id: "recursion-error",
+        objectID: "recursion-error",
         title: `Did you mean: ${queryString}?`,
         url: "https://metaforecast.org/recursion?bypassEasterEgg=true",
         platform: "metaforecast",
-        platformLabel: "Metaforecast",
         description:
           "Fatal error: Too much recursion. Click to proceed anyways",
         options: [
@@ -154,14 +148,13 @@ export default async function searchWithAlgolia({
           },
         ],
         timestamp: `${new Date().toISOString().slice(0, 10)}`,
+        stars: 5, // legacy
         qualityindicators: {
           numforecasts: 1,
           numforecasters: 1,
           stars: 5,
         },
-        // noExactSearchResults: true,
+        extra: {},
-        // optionsstringforsearch: "Yes, No",
-        // has_numforecasts: true,
       },
       ...results,
     ];
@@ -172,10 +165,10 @@ export default async function searchWithAlgolia({
   ) {
     results.unshift({
       id: "not-found-2",
+      objectID: "not-found-2",
       title: "No search results appear to match your query",
       url: "https://metaforecast.org",
       platform: "metaforecast",
-      platformLabel: "Metaforecast",
       description: "Maybe try a broader query? That said, we could be wrong.",
       options: [
         {
@@ -190,17 +183,14 @@ export default async function searchWithAlgolia({
         },
       ],
       timestamp: `${new Date().toISOString().slice(0, 10)}`,
+      stars: 1, // legacy
       qualityindicators: {
         numforecasts: 1,
         numforecasters: 1,
         stars: 1,
       },
-      // noExactSearchResults: true,
+      extra: {},
-      // optionsstringforsearch: "Yes, No",
-      // has_numforecasts: true,
     });
-  } else {
-    // results[0].noExactSearchResults = false;
   }
 
   return results;