feat: rewrite frontend with graphql (WIP)

codegen.yml

@@ -1,5 +1,7 @@
-schema:
+schema: src/graphql/build-schema.js
-  - src/graphql/build-schema.js
+documents:
+  - "src/**/*.graphql"
+
 # This should be updated to match your client files
 # documents: 'client/**/!(*.d).{ts,tsx}'
 generates:
@@ -7,9 +9,16 @@ generates:
   schema.graphql:
     plugins:
       - schema-ast
-  # This will contain the generated apollo hooks and schema types needed to make type-safe queries with the apollo client
+
-  # __generated__/operations.ts:
+  src/graphql/types.generated.ts:
-  #   plugins:
+    plugins:
-  #     - typescript
+      - typescript
-  #     - typescript-operations
+
-  #     - typescript-react-apollo
+  src/:
+    preset: near-operation-file
+    presetConfig:
+      extension: .generated.tsx
+      baseTypesPath: graphql/types.generated.ts
+    plugins:
+      - typescript-operations
+      - typed-document-node

graphql.config.yaml

@@ -0,0 +1 @@
+schema: http://localhost:3000/api/graphql

package.json

@@ -59,6 +59,7 @@
     "multiselect-react-dropdown": "^2.0.17",
     "next": "12",
     "next-plausible": "^3.1.6",
+    "next-urql": "^3.3.2",
     "nprogress": "^0.2.0",
     "open": "^7.3.1",
     "papaparse": "^5.3.0",
@@ -76,6 +77,7 @@
     "react-dropdown": "^1.9.2",
     "react-hook-form": "^7.27.0",
     "react-icons": "^4.2.0",
+    "react-is": "^18.0.0",
     "react-markdown": "^8.0.0",
     "react-safe": "^1.3.0",
     "react-select": "^5.2.2",
@@ -85,11 +87,14 @@
     "tailwindcss": "^3.0.22",
     "textversionjs": "^1.1.3",
     "ts-node": "^10.7.0",
-    "tunnel": "^0.0.6"
+    "tunnel": "^0.0.6",
+    "urql": "^2.2.0"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "^2.6.2",
+    "@graphql-codegen/near-operation-file-preset": "^2.2.9",
     "@graphql-codegen/schema-ast": "^2.4.1",
+    "@graphql-codegen/typed-document-node": "^2.2.8",
     "@graphql-codegen/typescript": "^2.4.8",
     "@graphql-codegen/typescript-operations": "^2.3.5",
     "@netlify/plugin-nextjs": "^4.2.4",

schema.graphql

@@ -1,11 +1,71 @@
-type Query {
+type Dashboard {
-  frontpage: [Question!]!
+  creator: String!
-}
+  description: String!
-
+  id: ID!
-"""
+  questions: [Question!]!
-Forecast question.
-"""
-type Question {
-  id: String!
   title: String!
 }
+
+"""Date serialized as the Unix timestamp."""
+scalar Date
+
+type PageInfo {
+  endCursor: String
+  hasNextPage: Boolean!
+  hasPreviousPage: Boolean!
+  startCursor: String
+}
+
+"""Platform supported by metaforecast"""
+type Platform {
+  id: ID!
+  label: String!
+}
+
+type ProbabilityOption {
+  name: String
+  probability: Float
+}
+
+"""Various indicators of the question's quality"""
+type QualityIndicators {
+  numForecasts: Int
+  stars: Int!
+}
+
+type Query {
+  dashboard(id: ID!): Dashboard!
+  frontpage: [Question!]!
+  questions(after: String, before: String, first: Int, last: Int): QueryQuestionsConnection!
+  searchQuestions(input: SearchInput!): [Question!]!
+}
+
+type QueryQuestionsConnection {
+  edges: [QueryQuestionsConnectionEdge]!
+  pageInfo: PageInfo!
+}
+
+type QueryQuestionsConnectionEdge {
+  cursor: String!
+  node: Question!
+}
+
+type Question {
+  description: String!
+  id: ID!
+  options: [ProbabilityOption!]!
+  platform: Platform!
+  qualityIndicators: QualityIndicators!
+  timestamp: Date!
+  title: String!
+  url: String!
+  visualization: String
+}
+
+input SearchInput {
+  forecastingPlatforms: [String!]
+  forecastsThreshold: Int
+  limit: Int
+  query: String!
+  starsThreshold: Int
+}

src/backend/database/prisma.ts

@@ -1,3 +1,15 @@
 import { PrismaClient } from "@prisma/client";
 
-export const prisma = new PrismaClient({});
+declare global {
+  // allow global `var` declarations
+  // eslint-disable-next-line no-var
+  var prisma: PrismaClient | undefined;
+}
+
+export const prisma =
+  global.prisma ||
+  new PrismaClient({
+    log: ["query"],
+  });
+
+if (process.env.NODE_ENV !== "production") global.prisma = prisma;

src/backend/platforms/index.ts

@@ -16,6 +16,20 @@ import { smarkets } from "./smarkets";
 import { wildeford } from "./wildeford";
 import { xrisk } from "./xrisk";
 
+export interface QualityIndicators {
+  stars: number;
+  numforecasts?: number | string;
+  numforecasters?: number;
+  liquidity?: number | string;
+  volume?: number;
+  volume7Days?: number;
+  volume24Hours?: number;
+  address?: number;
+  tradevolume?: string;
+  pool?: any;
+  createdTime?: any;
+}
+
 export interface Question {
   id: string;
   // "fantasyscotus-580"
@@ -53,7 +67,7 @@ export interface Question {
   stars?: number;
   // 2
 
-  qualityindicators: any;
+  qualityindicators: QualityIndicators;
   /*
   {
     "numforecasts": 120,

src/backend/utils/algolia.ts

@@ -1,6 +1,8 @@
 import algoliasearch from "algoliasearch";
 
-import { pgRead } from "../database/pg-wrapper";
+import { Question } from "@prisma/client";
+
+import { prisma } from "../database/prisma";
 import { platforms } from "../platforms";
 
 let cookie = process.env.ALGOLIA_MASTER_API_KEY;
@@ -8,10 +10,14 @@ const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
 const client = algoliasearch(algoliaAppId, cookie);
 const index = client.initIndex("metaforecast");
 
-let getoptionsstringforsearch = (record: any) => {
+export type AlgoliaQuestion = Omit<Question, "timestamp"> & {
+  timestamp: string;
+};
+
+const getoptionsstringforsearch = (record: Question): string => {
   let result = "";
-  if (!!record.options && record.options.length > 0) {
+  if (!!record.options && (record.options as any[]).length > 0) {
-    result = record.options
+    result = (record.options as any[])
       .map((option: any) => option.name || null)
       .filter((x: any) => x != null)
       .join(", ");
@@ -19,23 +25,23 @@ let getoptionsstringforsearch = (record: any) => {
   return result;
 };
 
-export async function rebuildAlgoliaDatabaseTheEasyWay() {
+export async function rebuildAlgoliaDatabase() {
-  let records: any[] = await pgRead({
+  const questions = await prisma.question.findMany();
-    tableName: "questions",
-  });
 
   const platformNameToLabel = Object.fromEntries(
     platforms.map((platform) => [platform.name, platform.label])
   );
 
-  records = records.map((record, index: number) => ({
+  const records: AlgoliaQuestion[] = questions.map(
-    ...record,
+    (question, index: number) => ({
-    platformLabel: platformNameToLabel[record.platform] || record.platform,
+      ...question,
-    has_numforecasts: record.numforecasts ? true : false,
+      timestamp: `${question.timestamp}`,
+      platformLabel:
+        platformNameToLabel[question.platform] || question.platform,
       objectID: index,
-    optionsstringforsearch: getoptionsstringforsearch(record),
+      optionsstringforsearch: getoptionsstringforsearch(question),
-  }));
+    })
-  // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
+  );
 
   if (index.exists()) {
     console.log("Index exists");
@@ -45,5 +51,3 @@ export async function rebuildAlgoliaDatabaseTheEasyWay() {
     );
   }
 }
-
-export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay;

src/graphql/build-schema.js

@@ -2,4 +2,4 @@
 // but we use ts-node instead of @boost/module
 require("ts-node").register({});
 
-module.exports = require("./schema.ts");
+module.exports = require("./schema/index.ts");

src/graphql/builder.ts

@@ -0,0 +1,39 @@
+import SchemaBuilder from "@pothos/core";
+import PrismaPlugin from "@pothos/plugin-prisma";
+import RelayPlugin from "@pothos/plugin-relay";
+
+import { prisma } from "../backend/database/prisma";
+
+import type PrismaTypes from "@pothos/plugin-prisma/generated";
+export const builder = new SchemaBuilder<{
+  PrismaTypes: PrismaTypes;
+  Scalars: {
+    Date: {
+      Input: Date;
+      Output: Date;
+    };
+  };
+}>({
+  plugins: [PrismaPlugin, RelayPlugin],
+  prisma: {
+    client: prisma,
+  },
+  relayOptions: {
+    clientMutationId: "omit",
+    cursorType: "String",
+    // these are required for some reason, though it's not documented and probably a bug
+    brandLoadedObjects: undefined,
+    encodeGlobalID: undefined,
+    decodeGlobalID: undefined,
+  },
+});
+
+builder.scalarType("Date", {
+  description: "Date serialized as the Unix timestamp.",
+  serialize: (d) => d.getTime() / 1000,
+  parseValue: (d) => {
+    return new Date(d as string); // not sure if this is correct, need to check
+  },
+});
+
+builder.queryType({});

src/graphql/schema.ts

@@ -1,104 +0,0 @@
-import SchemaBuilder from "@pothos/core";
-import PrismaPlugin from "@pothos/plugin-prisma";
-import RelayPlugin from "@pothos/plugin-relay";
-import { Question } from "@prisma/client";
-
-import { prisma } from "../backend/database/prisma";
-import { getFrontpage } from "../backend/frontpage";
-
-import type PrismaTypes from "@pothos/plugin-prisma/generated";
-const builder = new SchemaBuilder<{
-  PrismaTypes: PrismaTypes;
-  Scalars: {
-    Date: {
-      Input: Date;
-      Output: Date;
-    };
-  };
-}>({
-  plugins: [PrismaPlugin, RelayPlugin],
-  prisma: {
-    client: prisma,
-  },
-  relayOptions: {
-    clientMutationId: "omit",
-    cursorType: "String",
-    // these are required for some reason, though it's not documented and probably a bug
-    brandLoadedObjects: undefined,
-    encodeGlobalID: undefined,
-    decodeGlobalID: undefined,
-  },
-});
-
-builder.scalarType("Date", {
-  description: "Date serialized as the Unix timestamp.",
-  serialize: (d) => d.getTime() / 1000,
-  parseValue: (d) => {
-    return new Date(d as string); // not sure if this is correct, need to check
-  },
-});
-
-const QuestionObj = builder.prismaObject("Question", {
-  findUnique: (question) => ({ id: question.id }),
-  fields: (t) => ({
-    id: t.exposeID("id"),
-    title: t.exposeString("title"),
-    timestamp: t.field({
-      type: "Date",
-      resolve: (parent) => parent.timestamp,
-    }),
-  }),
-});
-
-builder.queryType({
-  fields: (t) => ({
-    firstQuestion: t.prismaField({
-      type: "Question",
-      resolve: async (query, root, args, ctx, info) =>
-        prisma.question.findUnique({
-          ...query,
-          rejectOnNotFound: true,
-          where: { id: "foretold-e1ca8cc6-33a4-4e38-9ef3-553a050ba0a9" },
-        }),
-    }),
-  }),
-});
-
-builder.queryField("frontpage", (t) =>
-  t.field({
-    type: [QuestionObj],
-    resolve: async () => {
-      const legacyQuestions = await getFrontpage();
-      const ids = legacyQuestions.map((q) => q.id);
-      const questions = await prisma.question.findMany({
-        where: {
-          id: {
-            in: ids,
-          },
-        },
-      });
-      const id2q: { [k: string]: Question } = {};
-      for (const q of questions) {
-        id2q[q.id] = q;
-      }
-
-      return ids.map((id) => id2q[id] || null).filter((q) => q !== null);
-    },
-  })
-);
-
-builder.queryField("questions", (t) =>
-  t.prismaConnection(
-    {
-      type: "Question",
-      cursor: "id",
-      maxSize: 1000,
-      resolve: (query, parent, args, context, info) =>
-        prisma.question.findMany({ ...query }),
-    },
-    {},
-    {}
-  )
-);
-
-export const schema = builder.toSchema({});

src/graphql/schema/dashboards.ts

@@ -0,0 +1,42 @@
+import { Dashboard } from "@prisma/client";
+
+import { prisma } from "../../backend/database/prisma";
+import { builder } from "../builder";
+import { QuestionObj } from "./questions";
+
+const DashboardObj = builder.objectRef<Dashboard>("Dashboard").implement({
+  fields: (t) => ({
+    id: t.exposeID("id"),
+    title: t.exposeString("title"),
+    description: t.exposeString("description"),
+    creator: t.exposeString("creator"),
+    questions: t.field({
+      type: [QuestionObj],
+      resolve: async (parent) => {
+        return await prisma.question.findMany({
+          where: {
+            id: {
+              in: parent.contents as string[],
+            },
+          },
+        });
+      },
+    }),
+  }),
+});
+
+builder.queryField("dashboard", (t) =>
+  t.field({
+    type: DashboardObj,
+    args: {
+      id: t.arg({ type: "ID", required: true }),
+    },
+    resolve: async (parent, args) => {
+      return await prisma.dashboard.findUnique({
+        where: {
+          id: String(args.id),
+        },
+      });
+    },
+  })
+);

src/graphql/schema/frontpage.ts

@@ -0,0 +1,29 @@
+import { Question } from "@prisma/client";
+
+import { prisma } from "../../backend/database/prisma";
+import { getFrontpage } from "../../backend/frontpage";
+import { builder } from "../builder";
+import { QuestionObj } from "./questions";
+
+builder.queryField("frontpage", (t) =>
+  t.field({
+    type: [QuestionObj],
+    resolve: async () => {
+      const legacyQuestions = await getFrontpage();
+      const ids = legacyQuestions.map((q) => q.id);
+      const questions = await prisma.question.findMany({
+        where: {
+          id: {
+            in: ids,
+          },
+        },
+      });
+      const id2q: { [k: string]: Question } = {};
+      for (const q of questions) {
+        id2q[q.id] = q;
+      }
+
+      return ids.map((id) => id2q[id] || null).filter((q) => q !== null);
+    },
+  })
+);

src/graphql/schema/index.ts

@@ -0,0 +1,8 @@
+import "./dashboards";
+import "./frontpage";
+import "./questions";
+import "./search";
+
+import { builder } from "../builder";
+
+export const schema = builder.toSchema({});

src/graphql/schema/questions.ts

@@ -0,0 +1,102 @@
+import { prisma } from "../../backend/database/prisma";
+import { platforms, QualityIndicators } from "../../backend/platforms";
+import { builder } from "../builder";
+
+const PlatformObj = builder.objectRef<string>("Platform").implement({
+  description: "Platform supported by metaforecast",
+  fields: (t) => ({
+    label: t.string({
+      resolve: (platformName) => {
+        if (platformName === "metaforecast") {
+          return "Metaforecast";
+        }
+        if (platformName === "guesstimate") {
+          return "Guesstimate";
+        }
+        // kinda slow and repetitive, TODO - store a map {name => platform} somewhere and `getPlatform` util function?
+        const platform = platforms.find((p) => p.name === platformName);
+        if (!platform) {
+          throw new Error(`Unknown platform ${platformName}`);
+        }
+        return platform.label;
+      },
+    }),
+    id: t.id({
+      resolve: (x) => x,
+    }),
+  }),
+});
+
+export const QualityIndicatorsObj = builder
+  .objectRef<QualityIndicators>("QualityIndicators")
+  .implement({
+    description: "Various indicators of the question's quality",
+    fields: (t) => ({
+      stars: t.exposeInt("stars"),
+      numForecasts: t.int({
+        nullable: true,
+        resolve: (parent) =>
+          parent.numforecasts === undefined
+            ? undefined
+            : Number(parent.numforecasts),
+      }),
+    }),
+  });
+
+export const ProbabilityOptionObj = builder
+  .objectRef<{ name: string; probability: number }>("ProbabilityOption")
+  .implement({
+    fields: (t) => ({
+      name: t.exposeString("name", { nullable: true }),
+      probability: t.exposeFloat("probability", { nullable: true }), // number, 0 to 1
+    }),
+  });
+
+export const QuestionObj = builder.prismaObject("Question", {
+  findUnique: (question) => ({ id: question.id }),
+  fields: (t) => ({
+    id: t.exposeID("id"),
+    title: t.exposeString("title"),
+    description: t.exposeString("description"),
+    url: t.exposeString("url"),
+    timestamp: t.field({
+      type: "Date",
+      resolve: (parent) => parent.timestamp,
+    }),
+    platform: t.field({
+      type: PlatformObj,
+      resolve: (parent) => parent.platform,
+    }),
+    qualityIndicators: t.field({
+      type: QualityIndicatorsObj,
+      resolve: (parent) => parent.qualityindicators as any as QualityIndicators,
+    }),
+    options: t.field({
+      type: [ProbabilityOptionObj],
+      resolve: ({ options }) => {
+        if (!Array.isArray(options)) {
+          return [];
+        }
+        return options as any[];
+      },
+    }),
+    visualization: t.string({
+      resolve: (parent) => (parent.extra as any)?.visualization,
+      nullable: true,
+    }),
+  }),
+});
+
+builder.queryField("questions", (t) =>
+  t.prismaConnection(
+    {
+      type: "Question",
+      cursor: "id",
+      maxSize: 1000,
+      resolve: (query, parent, args, context, info) =>
+        prisma.question.findMany({ ...query }),
+    },
+    {},
+    {}
+  )
+);

src/graphql/schema/search.ts

@@ -0,0 +1,62 @@
+import { AlgoliaQuestion } from "../../backend/utils/algolia";
+import searchGuesstimate from "../../web/worker/searchGuesstimate";
+import searchWithAlgolia from "../../web/worker/searchWithAlgolia";
+import { builder } from "../builder";
+import { QuestionObj } from "./questions";
+
+const SearchInput = builder.inputType("SearchInput", {
+  fields: (t) => ({
+    query: t.string({ required: true }),
+    starsThreshold: t.int(),
+    forecastsThreshold: t.int(),
+    forecastingPlatforms: t.stringList(),
+    limit: t.int(),
+  }),
+});
+
+builder.queryField("searchQuestions", (t) =>
+  t.field({
+    type: [QuestionObj],
+    args: {
+      input: t.arg({ type: SearchInput, required: true }),
+    },
+    resolve: async (parent, { input }) => {
+      // defs
+      const query = input.query === undefined ? "" : input.query;
+      if (query == "") return [];
+      const forecastsThreshold = input.forecastsThreshold;
+      const starsThreshold = input.starsThreshold;
+      const platformsIncludeGuesstimate =
+        input.forecastingPlatforms.includes("guesstimate") &&
+        starsThreshold <= 1;
+
+      // preparation
+      const unawaitedAlgoliaResponse = searchWithAlgolia({
+        queryString: query,
+        hitsPerPage: input.limit + 50,
+        starsThreshold,
+        filterByPlatforms: input.forecastingPlatforms,
+        forecastsThreshold,
+      });
+
+      let results: AlgoliaQuestion[] = [];
+
+      // consider the guesstimate and the non-guesstimate cases separately.
+      if (platformsIncludeGuesstimate) {
+        const [responsesNotGuesstimate, responsesGuesstimate] =
+          await Promise.all([
+            unawaitedAlgoliaResponse,
+            searchGuesstimate(query),
+          ]); // faster than two separate requests
+        results = [...responsesNotGuesstimate, ...responsesGuesstimate];
+      } else {
+        results = await unawaitedAlgoliaResponse;
+      }
+
+      return results.map((q) => ({
+        ...q,
+        timestamp: new Date(q.timestamp),
+      }));
+    },
+  })
+);

src/graphql/types.generated.ts

@@ -0,0 +1,111 @@
+export type Maybe<T> = T | null;
+export type InputMaybe<T> = Maybe<T>;
+export type Exact<T extends { [key: string]: unknown }> = { [K in keyof T]: T[K] };
+export type MakeOptional<T, K extends keyof T> = Omit<T, K> & { [SubKey in K]?: Maybe<T[SubKey]> };
+export type MakeMaybe<T, K extends keyof T> = Omit<T, K> & { [SubKey in K]: Maybe<T[SubKey]> };
+/** All built-in and custom scalars, mapped to their actual values */
+export type Scalars = {
+  ID: string;
+  String: string;
+  Boolean: boolean;
+  Int: number;
+  Float: number;
+  /** Date serialized as the Unix timestamp. */
+  Date: any;
+};
+
+export type Dashboard = {
+  __typename?: 'Dashboard';
+  creator: Scalars['String'];
+  description: Scalars['String'];
+  id: Scalars['ID'];
+  questions: Array<Question>;
+  title: Scalars['String'];
+};
+
+export type PageInfo = {
+  __typename?: 'PageInfo';
+  endCursor?: Maybe<Scalars['String']>;
+  hasNextPage: Scalars['Boolean'];
+  hasPreviousPage: Scalars['Boolean'];
+  startCursor?: Maybe<Scalars['String']>;
+};
+
+/** Platform supported by metaforecast */
+export type Platform = {
+  __typename?: 'Platform';
+  id: Scalars['ID'];
+  label: Scalars['String'];
+};
+
+export type ProbabilityOption = {
+  __typename?: 'ProbabilityOption';
+  name?: Maybe<Scalars['String']>;
+  probability?: Maybe<Scalars['Float']>;
+};
+
+/** Various indicators of the question's quality */
+export type QualityIndicators = {
+  __typename?: 'QualityIndicators';
+  numForecasts?: Maybe<Scalars['Int']>;
+  stars: Scalars['Int'];
+};
+
+export type Query = {
+  __typename?: 'Query';
+  dashboard: Dashboard;
+  frontpage: Array<Question>;
+  questions: QueryQuestionsConnection;
+  searchQuestions: Array<Question>;
+};
+
+
+export type QueryDashboardArgs = {
+  id: Scalars['ID'];
+};
+
+
+export type QueryQuestionsArgs = {
+  after?: InputMaybe<Scalars['String']>;
+  before?: InputMaybe<Scalars['String']>;
+  first?: InputMaybe<Scalars['Int']>;
+  last?: InputMaybe<Scalars['Int']>;
+};
+
+
+export type QuerySearchQuestionsArgs = {
+  input: SearchInput;
+};
+
+export type QueryQuestionsConnection = {
+  __typename?: 'QueryQuestionsConnection';
+  edges: Array<Maybe<QueryQuestionsConnectionEdge>>;
+  pageInfo: PageInfo;
+};
+
+export type QueryQuestionsConnectionEdge = {
+  __typename?: 'QueryQuestionsConnectionEdge';
+  cursor: Scalars['String'];
+  node: Question;
+};
+
+export type Question = {
+  __typename?: 'Question';
+  description: Scalars['String'];
+  id: Scalars['ID'];
+  options: Array<ProbabilityOption>;
+  platform: Platform;
+  qualityIndicators: QualityIndicators;
+  timestamp: Scalars['Date'];
+  title: Scalars['String'];
+  url: Scalars['String'];
+  visualization?: Maybe<Scalars['String']>;
+};
+
+export type SearchInput = {
+  forecastingPlatforms?: InputMaybe<Array<Scalars['String']>>;
+  forecastsThreshold?: InputMaybe<Scalars['Int']>;
+  limit?: InputMaybe<Scalars['Int']>;
+  query: Scalars['String'];
+  starsThreshold?: InputMaybe<Scalars['Int']>;
+};

src/pages/_app.tsx

@@ -1,13 +1,15 @@
 import "nprogress/nprogress.css";
 import "../styles/main.css";
 
+import PlausibleProvider from "next-plausible";
+import { withUrqlClient } from "next-urql";
+import { AppProps } from "next/app";
 import Router from "next/router";
 import NProgress from "nprogress";
 
-import PlausibleProvider from "next-plausible";
+import { graphqlEndpoint } from "../web/urql";
 
 Router.events.on("routeChangeStart", (as, { shallow }) => {
-  console.log(shallow);
   if (!shallow) {
     NProgress.start();
   }
@@ -15,7 +17,7 @@ Router.events.on("routeChangeStart", (as, { shallow }) => {
 Router.events.on("routeChangeComplete", () => NProgress.done());
 Router.events.on("routeChangeError", () => NProgress.done());
 
-function MyApp({ Component, pageProps }) {
+function MyApp({ Component, pageProps }: AppProps) {
   return (
     <PlausibleProvider domain="metaforecast.org">
       <Component {...pageProps} />
@@ -23,4 +25,9 @@ function MyApp({ Component, pageProps }) {
   );
 }
 
-export default MyApp;
+export default withUrqlClient(
+  () => ({
+    url: graphqlEndpoint,
+  }),
+  { ssr: false }
+)(MyApp);

src/pages/dashboards/embed/[id].tsx

@@ -1,49 +1,43 @@
 import { GetServerSideProps, NextPage } from "next";
 import Error from "next/error";
 
-import { DashboardItem } from "../../../backend/dashboards";
+import {
+    DashboardByIdDocument, DashboardFragment
+} from "../../../web/dashboards/queries.generated";
 import { DisplayQuestions } from "../../../web/display/DisplayQuestions";
-import { FrontendQuestion } from "../../../web/platforms";
+import { ssrUrql } from "../../../web/urql";
-import { reqToBasePath } from "../../../web/utils";
-import { getDashboardQuestionsByDashboardId } from "../../../web/worker/getDashboardQuestions";
 
 interface Props {
-  dashboardQuestions: FrontendQuestion[];
+  dashboard?: DashboardFragment;
-  dashboardItem: DashboardItem;
   numCols?: number;
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
+  const [ssrCache, client] = ssrUrql();
   const dashboardId = context.query.id as string;
   const numCols = Number(context.query.numCols);
 
-  const { dashboardItem, dashboardQuestions } =
+  const dashboard = (
-    await getDashboardQuestionsByDashboardId({
+    await client.query(DashboardByIdDocument, { id: dashboardId }).toPromise()
-      dashboardId,
+  ).data?.result;
-      basePath: reqToBasePath(context.req), // required on server side to find the API endpoint
-    });
 
-  if (!dashboardItem) {
+  if (!dashboard) {
     context.res.statusCode = 404;
   }
 
   return {
     props: {
-      dashboardQuestions,
+      urqlState: ssrCache.extractData(),
-      dashboardItem,
+      dashboard,
       numCols: !numCols ? null : numCols < 5 ? numCols : 4,
     },
   };
 };
 
-const EmbedDashboardPage: NextPage<Props> = ({
+const EmbedDashboardPage: NextPage<Props> = ({ dashboard, numCols }) => {
-  dashboardQuestions,
+  if (!dashboard) {
-  dashboardItem,
-  numCols,
-}) => {
-  if (!dashboardItem) {
     return <Error statusCode={404} />;
   }
 
@@ -58,8 +52,8 @@ const EmbedDashboardPage: NextPage<Props> = ({
           } gap-4 mb-6`}
         >
           <DisplayQuestions
-            results={dashboardQuestions}
+            results={dashboard.questions}
-            numDisplay={dashboardQuestions.length}
+            numDisplay={dashboard.questions.length}
             showIdToggle={false}
           />
         </div>

src/pages/dashboards/view/[id].tsx

@@ -2,57 +2,55 @@ import { GetServerSideProps, NextPage } from "next";
 import Error from "next/error";
 import Link from "next/link";
 
-import { DashboardItem } from "../../../backend/dashboards";
+import {
+    DashboardByIdDocument, DashboardFragment
+} from "../../../web/dashboards/queries.generated";
 import { DisplayQuestions } from "../../../web/display/DisplayQuestions";
 import { InfoBox } from "../../../web/display/InfoBox";
 import { Layout } from "../../../web/display/Layout";
 import { LineHeader } from "../../../web/display/LineHeader";
-import { FrontendQuestion } from "../../../web/platforms";
+import { ssrUrql } from "../../../web/urql";
-import { reqToBasePath } from "../../../web/utils";
-import { getDashboardQuestionsByDashboardId } from "../../../web/worker/getDashboardQuestions";
 
 interface Props {
-  dashboardQuestions: FrontendQuestion[];
+  dashboard?: DashboardFragment;
-  dashboardItem: DashboardItem;
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
+  const [ssrCache, client] = ssrUrql();
   const dashboardId = context.query.id as string;
 
-  const { dashboardQuestions, dashboardItem } =
+  const dashboard = (
-    await getDashboardQuestionsByDashboardId({
+    await client.query(DashboardByIdDocument, { id: dashboardId }).toPromise()
-      dashboardId,
+  ).data?.result;
-      basePath: reqToBasePath(context.req), // required on server side to find the API endpoint
-    });
 
-  if (!dashboardItem) {
+  if (!dashboard) {
     context.res.statusCode = 404;
   }
 
   return {
     props: {
-      dashboardQuestions,
+      urqlState: ssrCache.extractData(),
-      dashboardItem,
+      dashboard,
     },
   };
 };
 
-const DashboardMetadata: React.FC<{ dashboardItem: DashboardItem }> = ({
+const DashboardMetadata: React.FC<{ dashboard: DashboardFragment }> = ({
-  dashboardItem,
+  dashboard,
 }) => (
   <div>
-    {dashboardItem?.title ? (
+    {dashboard.title ? (
       <h1 className="text-4xl text-center text-gray-600 mt-2 mb-2">
-        {dashboardItem.title}
+        {dashboard.title}
       </h1>
     ) : null}
 
-    {dashboardItem && dashboardItem.creator ? (
+    {dashboard.creator ? (
       <p className="text-lg text-center text-gray-600 mt-2 mb-2">
         Created by:{" "}
-        {dashboardItem.creator === "Clay Graubard" ? (
+        {dashboard.creator === "Clay Graubard" ? (
           <>
             @
             <a
@@ -63,40 +61,38 @@ const DashboardMetadata: React.FC<{ dashboardItem: DashboardItem }> = ({
             </a>
           </>
         ) : (
-          dashboardItem.creator
+          dashboard.creator
         )}
       </p>
     ) : null}
 
-    {dashboardItem?.description ? (
+    {dashboard.description ? (
       <p className="text-lg text-center text-gray-600 mt-2 mb-2">
-        {dashboardItem.description}
+        {dashboard.description}
       </p>
     ) : null}
   </div>
 );
 
 /* Body */
-const ViewDashboardPage: NextPage<Props> = ({
+const ViewDashboardPage: NextPage<Props> = ({ dashboard }) => {
-  dashboardQuestions,
-  dashboardItem,
-}) => {
   return (
     <Layout page="view-dashboard">
       <div className="flex flex-col my-8 space-y-8">
-        {dashboardItem ? (
+        {dashboard ? (
-          <DashboardMetadata dashboardItem={dashboardItem} />
+          <>
-        ) : (
+            <DashboardMetadata dashboard={dashboard} />
-          <Error statusCode={404} />
-        )}
-
             <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
               <DisplayQuestions
-            results={dashboardQuestions}
+                results={dashboard.questions}
-            numDisplay={dashboardQuestions.length}
+                numDisplay={dashboard.questions.length}
                 showIdToggle={false}
               />
             </div>
+          </>
+        ) : (
+          <Error statusCode={404} />
+        )}
 
         <div className="max-w-xl self-center">
           <InfoBox>

src/pages/index.tsx

@@ -10,7 +10,7 @@ export { getServerSideProps } from "../web/search/anySearchPage";
 
 const IndexPage: NextPage<Props> = (props) => {
   return (
-    <Layout page={"search"}>
+    <Layout page="search">
       <CommonDisplay
         {...props}
         hasSearchbar={true}

src/pages/secretEmbed.tsx

@@ -5,17 +5,18 @@ import React from "react";
 
 import { platforms } from "../backend/platforms";
 import { DisplayQuestion } from "../web/display/DisplayQuestion";
-import { FrontendQuestion } from "../web/platforms";
+import { QuestionFragment, SearchDocument } from "../web/search/queries.generated";
-import searchAccordingToQueryData from "../web/worker/searchAccordingToQueryData";
+import { ssrUrql } from "../web/urql";
 
 interface Props {
-  results: FrontendQuestion[];
+  results: QuestionFragment[];
 }
 
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
-  let urlQuery = context.query; // this is an object, not a string which I have to parse!!
+  const [ssrCache, client] = ssrUrql();
+  let urlQuery = context.query;
 
   let initialQueryParameters = {
     query: "",
@@ -25,14 +26,24 @@ export const getServerSideProps: GetServerSideProps<Props> = async (
     ...urlQuery,
   };
 
-  let results: FrontendQuestion[] = [];
+  let results: QuestionFragment[] = [];
-  if (initialQueryParameters.query != "") {
+  if (initialQueryParameters.query !== "") {
-    results = await searchAccordingToQueryData(initialQueryParameters, 1);
+    results = (
+      await client
+        .query(SearchDocument, {
+          input: {
+            ...initialQueryParameters,
+            limit: 1,
+          },
+        })
+        .toPromise()
+    ).data.result;
   }
 
   return {
     props: {
-      results: results,
+      urqlState: ssrCache.extractData(),
+      results,
     },
   };
 };

src/web/dashboards/queries.generated.tsx

@@ -0,0 +1,15 @@
+import * as Types from '../../graphql/types.generated';
+
+import { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core';
+import { QuestionFragmentDoc } from '../search/queries.generated';
+export type DashboardFragment = { __typename?: 'Dashboard', id: string, title: string, description: string, creator: string, questions: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: any, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> };
+
+export type DashboardByIdQueryVariables = Types.Exact<{
+  id: Types.Scalars['ID'];
+}>;
+
+
+export type DashboardByIdQuery = { __typename?: 'Query', result: { __typename?: 'Dashboard', id: string, title: string, description: string, creator: string, questions: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: any, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> } };
+
+export const DashboardFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"Dashboard"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Dashboard"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"title"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"creator"}},{"kind":"Field","name":{"kind":"Name","value":"questions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Question"}}]}}]}},...QuestionFragmentDoc.definitions]} as unknown as DocumentNode<DashboardFragment, unknown>;
+export const DashboardByIdDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"DashboardById"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"dashboard"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Dashboard"}}]}}]}},...DashboardFragmentDoc.definitions]} as unknown as DocumentNode<DashboardByIdQuery, DashboardByIdQueryVariables>;

src/web/dashboards/queries.graphql

@@ -0,0 +1,15 @@
+fragment Dashboard on Dashboard {
+  id
+  title
+  description
+  creator
+  questions {
+    ...Question
+  }
+}
+
+query DashboardById($id: ID!) {
+  result: dashboard(id: $id) {
+    ...Dashboard
+  }
+}

src/web/display/DisplayOneQuestionForCapture.tsx

@@ -2,11 +2,11 @@ import domtoimage from "dom-to-image"; // https://github.com/tsayen/dom-to-image
 import { useEffect, useRef, useState } from "react";
 import { CopyToClipboard } from "react-copy-to-clipboard";
 
-import { FrontendQuestion } from "../platforms";
+import { QuestionFragment } from "../search/queries.generated";
 import { uploadToImgur } from "../worker/uploadToImgur";
 import { DisplayQuestion } from "./DisplayQuestion";
 
-function displayOneQuestionInner(result: FrontendQuestion, containerRef) {
+function displayOneQuestionInner(result: QuestionFragment, containerRef) {
   return (
     <div ref={containerRef}>
       {result ? (
@@ -168,7 +168,7 @@ let generateMetaculusSource = (result, hasDisplayBeenCaptured) => {
 };
 
 interface Props {
-  result: FrontendQuestion;
+  result: QuestionFragment;
 }
 
 export const DisplayOneQuestionForCapture: React.FC<Props> = ({ result }) => {

src/web/display/DisplayQuestion/index.tsx

@@ -1,7 +1,7 @@
 import { FaRegClipboard } from "react-icons/fa";
 import ReactMarkdown from "react-markdown";
 
-import { FrontendQuestion } from "../../platforms";
+import { QuestionFragment } from "../../search/queries.generated";
 import { Card } from "../Card";
 import { QuestionFooter } from "./QuestionFooter";
 
@@ -268,7 +268,7 @@ const LastUpdated: React.FC<{ timestamp: string }> = ({ timestamp }) => (
 // Main component
 
 interface Props {
-  question: FrontendQuestion;
+  question: QuestionFragment;
   showTimeStamp: boolean;
   expandFooterToFullWidth: boolean;
   showIdToggle?: boolean;
@@ -280,10 +280,9 @@ export const DisplayQuestion: React.FC<Props> = ({
     title,
     url,
     platform,
-    platformLabel,
     description,
     options,
-    qualityindicators,
+    qualityIndicators,
     timestamp,
     visualization,
   },
@@ -292,7 +291,7 @@ export const DisplayQuestion: React.FC<Props> = ({
   showIdToggle,
 }) => {
   const displayTimestampAtBottom =
-    checkIfDisplayTimeStampAtBottom(qualityindicators);
+    checkIfDisplayTimeStampAtBottom(qualityIndicators);
 
   const yesNoOptions =
     options.length === 2 &&
@@ -349,13 +348,13 @@ export const DisplayQuestion: React.FC<Props> = ({
               </div>
             )}
 
-            {platform !== "guesstimate" && options.length < 3 && (
+            {platform.id !== "guesstimate" && options.length < 3 && (
               <div className="text-gray-500">
                 <DisplayMarkdown description={description} />
               </div>
             )}
 
-            {platform === "guesstimate" && (
+            {platform.id === "guesstimate" && (
               <img
                 className="rounded-sm"
                 src={visualization}
@@ -373,11 +372,11 @@ export const DisplayQuestion: React.FC<Props> = ({
           </div>
           <div className="w-full">
             <QuestionFooter
-              stars={qualityindicators.stars}
+              stars={qualityIndicators.stars}
-              platform={platform}
+              platform={platform.id}
-              platformLabel={platformLabel || platform} // author || platformLabel,
+              platformLabel={platform.label}
-              numforecasts={qualityindicators.numforecasts}
+              numforecasts={qualityIndicators.numForecasts}
-              qualityindicators={qualityindicators}
+              qualityindicators={qualityIndicators}
               timestamp={timestamp}
               showTimeStamp={showTimeStamp && displayTimestampAtBottom}
               expandFooterToFullWidth={expandFooterToFullWidth}

src/web/display/DisplayQuestions.tsx

@@ -1,10 +1,10 @@
 import React from "react";
 
-import { FrontendQuestion } from "../platforms";
+import { QuestionFragment } from "../search/queries.generated";
 import { DisplayQuestion } from "./DisplayQuestion";
 
 interface Props {
-  results: FrontendQuestion[];
+  results: QuestionFragment[];
   numDisplay: number;
   showIdToggle: boolean;
 }

src/web/hooks.ts

@@ -13,3 +13,15 @@ export const useNoInitialEffect = (
     return effect();
   }, deps);
 };
+
+export const useIsFirstRender = (): boolean => {
+  const isFirst = React.useRef(true);
+
+  if (isFirst.current) {
+    isFirst.current = false;
+
+    return true;
+  }
+
+  return isFirst.current;
+};

src/web/platforms.ts

@@ -1,21 +0,0 @@
-import { PlatformConfig, Question } from "../backend/platforms";
-
-export type FrontendQuestion = Question & {
-  platformLabel: string;
-  visualization?: any;
-};
-
-// ok on client side
-export const addLabelsToQuestions = (
-  questions: Question[],
-  platformsConfig: PlatformConfig[]
-): FrontendQuestion[] => {
-  const platformNameToLabel = Object.fromEntries(
-    platformsConfig.map((platform) => [platform.name, platform.label])
-  );
-
-  return questions.map((result) => ({
-    ...result,
-    platformLabel: platformNameToLabel[result.platform] || result.platform,
-  }));
-};

src/web/search/CommonDisplay.tsx

@@ -1,14 +1,14 @@
 import { useRouter } from "next/router";
-import React, { Fragment, useState } from "react";
+import React, { Fragment, useMemo, useState } from "react";
+import { useQuery } from "urql";
 
 import { ButtonsForStars } from "../display/ButtonsForStars";
 import { MultiSelectPlatform } from "../display/MultiSelectPlatform";
 import { QueryForm } from "../display/QueryForm";
 import { SliderElement } from "../display/SliderElement";
-import { useNoInitialEffect } from "../hooks";
+import { useIsFirstRender, useNoInitialEffect } from "../hooks";
-import { FrontendQuestion } from "../platforms";
-import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
 import { Props as AnySearchPageProps, QueryParameters } from "./anySearchPage";
+import { QuestionFragment, SearchDocument } from "./queries.generated";
 
 interface Props extends AnySearchPageProps {
   hasSearchbar: boolean;
@@ -17,7 +17,7 @@ interface Props extends AnySearchPageProps {
   placeholder: string;
   displaySeeMoreHint: boolean;
   displayQuestionsWrapper: (opts: {
-    results: FrontendQuestion[];
+    results: QuestionFragment[];
     numDisplay: number;
     whichResultToDisplayAndCapture: number;
     showIdToggle: boolean;
@@ -27,7 +27,6 @@ interface Props extends AnySearchPageProps {
 /* Body */
 const CommonDisplay: React.FC<Props> = ({
   defaultResults,
-  initialResults,
   initialQueryParameters,
   defaultQueryParameters,
   initialNumDisplay,
@@ -40,8 +39,9 @@ const CommonDisplay: React.FC<Props> = ({
   displaySeeMoreHint,
   displayQuestionsWrapper,
 }) => {
-  const router = useRouter();
   /* States */
+  const router = useRouter();
+  const isFirstRender = useIsFirstRender();
 
   const [queryParameters, setQueryParameters] = useState<QueryParameters>(
     initialQueryParameters
@@ -52,52 +52,56 @@ const CommonDisplay: React.FC<Props> = ({
   // used to distinguish numDisplay updates which force search and don't force search, see effects below
   const [forceSearch, setForceSearch] = useState(0);
 
-  const [results, setResults] = useState(initialResults);
   const [advancedOptions, showAdvancedOptions] = useState(false);
   const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] =
     useState(0);
   const [showIdToggle, setShowIdToggle] = useState(false);
 
-  /* Functions which I want to have access to the Home namespace */
+  const [typing, setTyping] = useState(false);
-  // I don't want to create an "defaultResults" object for each search.
-  async function executeSearchOrAnswerWithDefaultResults() {
-    const queryData = {
-      ...queryParameters,
-      numDisplay,
-    };
 
-    const filterManually = (
+  // must match the query from anySearchPage.ts getServerSideProps
-      queryData: QueryParameters,
+  const [queryResults, reexecuteQuery] = useQuery({
-      results: FrontendQuestion[]
+    query: SearchDocument,
-    ) => {
+    variables: {
+      input: {
+        ...queryParameters,
+        limit: numDisplay,
+      },
+    },
+    pause: !isFirstRender,
+  });
+
+  const queryIsEmpty =
+    queryParameters.query === undefined || queryParameters.query === "";
+
+  const results: QuestionFragment[] = useMemo(() => {
+    if (typing || queryResults.fetching) return []; // TODO - return results but show spinner or darken out all cards?
+
+    if (queryIsEmpty) {
+      const filterManually = (results: QuestionFragment[]) => {
         if (
-        queryData.forecastingPlatforms &&
+          queryParameters.forecastingPlatforms &&
-        queryData.forecastingPlatforms.length > 0
+          queryParameters.forecastingPlatforms.length > 0
         ) {
           results = results.filter((result) =>
-          queryData.forecastingPlatforms.includes(result.platform)
+            queryParameters.forecastingPlatforms.includes(result.platform.id)
           );
         }
-      if (queryData.starsThreshold === 4) {
+        if (queryParameters.starsThreshold === 4) {
           results = results.filter(
-          (result) => result.qualityindicators.stars >= 4
+            (result) => result.qualityIndicators.stars >= 4
           );
         }
-      if (queryData.forecastsThreshold) {
+        if (queryParameters.forecastsThreshold) {
-        // results = results.filter(result => (result.qualityindicators && result.qualityindicators.numforecasts > forecastsThreshold))
+          // TODO / FIXME / remove?
         }
         return results;
       };
-
+      return filterManually(defaultResults);
-    const queryIsEmpty =
+    } else {
-      !queryData || queryData.query == "" || queryData.query == undefined;
+      return queryResults.data?.result;
-
-    const results = queryIsEmpty
-      ? filterManually(queryData, defaultResults)
-      : await searchAccordingToQueryData(queryData, numDisplay);
-
-    setResults(results);
     }
+  }, [queryResults.data, queryParameters]);
 
   // I don't want the component which display questions (DisplayQuestions) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home.
   const getInfoToDisplayQuestionsFunction = () => {
@@ -145,10 +149,13 @@ const CommonDisplay: React.FC<Props> = ({
   useNoInitialEffect(updateRoute, [numDisplay]);
 
   useNoInitialEffect(() => {
-    setResults([]);
+    setTyping(true);
-    const newTimeoutId = setTimeout(() => {
+    const newTimeoutId = setTimeout(async () => {
       updateRoute();
-      executeSearchOrAnswerWithDefaultResults();
+      if (!queryIsEmpty) {
+        reexecuteQuery();
+      }
+      setTyping(false);
     }, 500);
 
     // avoid sending results if user has not stopped typing.
@@ -310,7 +317,7 @@ const CommonDisplay: React.FC<Props> = ({
       <div>{getInfoToDisplayQuestionsFunction()}</div>
 
       {displaySeeMoreHint &&
-      (!results || (results.length != 0 && numDisplay < results.length)) ? (
+      (!results || (results.length !== 0 && numDisplay < results.length)) ? (
         <div>
           <p className="mt-4 mb-4">
             {"Can't find what you were looking for?"}
@@ -336,7 +343,7 @@ const CommonDisplay: React.FC<Props> = ({
         </div>
       ) : null}
 
-      <br></br>
+      <br />
     </Fragment>
   );
 };

src/web/search/anySearchPage.tsx

@@ -1,9 +1,8 @@
 import { GetServerSideProps } from "next";
 
-import { getFrontpage } from "../../backend/frontpage";
 import { getPlatformsConfig, PlatformConfig, platforms } from "../../backend/platforms";
-import { addLabelsToQuestions, FrontendQuestion } from "../platforms";
+import { ssrUrql } from "../urql";
-import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
+import { FrontpageDocument, QuestionFragment, SearchDocument } from "./queries.generated";
 
 /* Common code for / and /capture */
 
@@ -15,8 +14,7 @@ export interface QueryParameters {
 }
 
 export interface Props {
-  defaultResults: FrontendQuestion[];
+  defaultResults: QuestionFragment[];
-  initialResults: FrontendQuestion[];
   initialQueryParameters: QueryParameters;
   defaultQueryParameters: QueryParameters;
   initialNumDisplay: number;
@@ -27,6 +25,7 @@ export interface Props {
 export const getServerSideProps: GetServerSideProps<Props> = async (
   context
 ) => {
+  const [ssrCache, client] = ssrUrql();
   const urlQuery = context.query;
 
   const platformsConfig = getPlatformsConfig({ withGuesstimate: true });
@@ -61,28 +60,32 @@ export const getServerSideProps: GetServerSideProps<Props> = async (
   const defaultNumDisplay = 21;
   const initialNumDisplay = Number(urlQuery.numDisplay) || defaultNumDisplay;
 
-  const defaultResults = addLabelsToQuestions(
+  const defaultResults = (await client.query(FrontpageDocument).toPromise())
-    await getFrontpage(),
+    .data.result;
-    platformsConfig
-  );
 
-  const initialResults =
+  if (
     !!initialQueryParameters &&
     initialQueryParameters.query != "" &&
     initialQueryParameters.query != undefined
-      ? await searchAccordingToQueryData(
+  ) {
-          initialQueryParameters,
+    // must match the query from CommonDisplay
-          initialNumDisplay
+    await client
-        )
+      .query(SearchDocument, {
-      : defaultResults;
+        input: {
+          ...initialQueryParameters,
+          limit: initialNumDisplay,
+        },
+      })
+      .toPromise();
+  }
 
   return {
     props: {
+      urqlState: ssrCache.extractData(),
       initialQueryParameters,
       defaultQueryParameters,
       initialNumDisplay,
       defaultNumDisplay,
-      initialResults,
       defaultResults,
       platformsConfig,
     },

src/web/search/queries.generated.tsx

@@ -0,0 +1,20 @@
+import * as Types from '../../graphql/types.generated';
+
+import { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core';
+export type QuestionFragment = { __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: any, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } };
+
+export type FrontpageQueryVariables = Types.Exact<{ [key: string]: never; }>;
+
+
+export type FrontpageQuery = { __typename?: 'Query', result: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: any, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> };
+
+export type SearchQueryVariables = Types.Exact<{
+  input: Types.SearchInput;
+}>;
+
+
+export type SearchQuery = { __typename?: 'Query', result: Array<{ __typename?: 'Question', id: string, url: string, title: string, description: string, timestamp: any, visualization?: string | null, options: Array<{ __typename?: 'ProbabilityOption', name?: string | null, probability?: number | null }>, platform: { __typename?: 'Platform', id: string, label: string }, qualityIndicators: { __typename?: 'QualityIndicators', stars: number, numForecasts?: number | null } }> };
+
+export const QuestionFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"Question"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"Question"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"url"}},{"kind":"Field","name":{"kind":"Name","value":"title"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"timestamp"}},{"kind":"Field","name":{"kind":"Name","value":"options"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"probability"}}]}},{"kind":"Field","name":{"kind":"Name","value":"platform"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"label"}}]}},{"kind":"Field","name":{"kind":"Name","value":"qualityIndicators"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"stars"}},{"kind":"Field","name":{"kind":"Name","value":"numForecasts"}}]}},{"kind":"Field","name":{"kind":"Name","value":"visualization"}}]}}]} as unknown as DocumentNode<QuestionFragment, unknown>;
+export const FrontpageDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"Frontpage"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"frontpage"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Question"}}]}}]}},...QuestionFragmentDoc.definitions]} as unknown as DocumentNode<FrontpageQuery, FrontpageQueryVariables>;
+export const SearchDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"Search"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SearchInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"result"},"name":{"kind":"Name","value":"searchQuestions"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"Question"}}]}}]}},...QuestionFragmentDoc.definitions]} as unknown as DocumentNode<SearchQuery, SearchQueryVariables>;

src/web/search/queries.graphql

@@ -0,0 +1,32 @@
+fragment Question on Question {
+  id
+  url
+  title
+  description
+  timestamp
+  options {
+    name
+    probability
+  }
+  platform {
+    id
+    label
+  }
+  qualityIndicators {
+    stars
+    numForecasts
+  }
+  visualization
+}
+
+query Frontpage {
+  result: frontpage {
+    ...Question
+  }
+}
+
+query Search($input: SearchInput!) {
+  result: searchQuestions(input: $input) {
+    ...Question
+  }
+}

src/web/urql.ts

@@ -0,0 +1,19 @@
+import { initUrqlClient } from "next-urql";
+import { cacheExchange, dedupExchange, fetchExchange, ssrExchange } from "urql";
+
+import { getBasePath } from "./utils";
+
+export const graphqlEndpoint = `${getBasePath()}/api/graphql`;
+
+// for getServerSideProps/getStaticProps only
+export const ssrUrql = () => {
+  const ssrCache = ssrExchange({ isClient: false });
+  const client = initUrqlClient(
+    {
+      url: graphqlEndpoint,
+      exchanges: [dedupExchange, cacheExchange, ssrCache, fetchExchange],
+    },
+    false
+  );
+  return [ssrCache, client] as const;
+};

src/web/utils.ts

@@ -1,11 +1,12 @@
-import { IncomingMessage } from "http";
+export const getBasePath = () => {
-
-export const reqToBasePath = (req: IncomingMessage) => {
   if (process.env.NEXT_PUBLIC_VERCEL_URL) {
-    console.log(process.env.NEXT_PUBLIC_VERCEL_URL);
     return `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`;
   }
 
-  // we could just hardcode http://localhost:3000 here, but then `next dev -p <CUSTOM_PORT>` would break
+  // can be used for local development if you prefer non-default port
-  return "http://" + req.headers.host;
+  if (process.env.NEXT_PUBLIC_SITE_URL) {
+    return process.env.NEXT_PUBLIC_SITE_URL;
+  }
+
+  return "http://localhost:3000";
 };

src/web/worker/getDashboardQuestions.ts

@@ -1,53 +1,5 @@
 import axios from "axios";
 
-import { DashboardItem } from "../../backend/dashboards";
-import { getPlatformsConfig, Question } from "../../backend/platforms";
-import { addLabelsToQuestions, FrontendQuestion } from "../platforms";
-
-export async function getDashboardQuestionsByDashboardId({
-  dashboardId,
-  basePath,
-}: {
-  dashboardId: string;
-  basePath?: string;
-}): Promise<{
-  dashboardQuestions: FrontendQuestion[];
-  dashboardItem: DashboardItem;
-}> {
-  console.log("getDashboardQuestionsByDashboardId: ");
-  if (typeof window === undefined && !basePath) {
-    throw new Error("`basePath` option is required on server side");
-  }
-
-  let dashboardQuestions: Question[] = [];
-  let dashboardItem: DashboardItem | null = null;
-  try {
-    let { data } = await axios({
-      url: `${basePath || ""}/api/dashboard-by-id`,
-      method: "post",
-      data: {
-        id: dashboardId,
-      },
-    });
-    console.log(data);
-
-    dashboardQuestions = data.dashboardContents;
-    dashboardItem = data.dashboardItem as DashboardItem;
-  } catch (error) {
-    console.log(error);
-  } finally {
-    const labeledDashboardQuestions = addLabelsToQuestions(
-      dashboardQuestions,
-      getPlatformsConfig({ withGuesstimate: false })
-    );
-
-    return {
-      dashboardQuestions: labeledDashboardQuestions,
-      dashboardItem,
-    };
-  }
-}
-
 export async function createDashboard(payload) {
   let data = { dashboardId: null };
   try {

src/web/worker/searchAccordingToQueryData.ts

@@ -1,52 +0,0 @@
-import { FrontendQuestion } from "../platforms";
-import { QueryParameters } from "../search/anySearchPage";
-import searchGuesstimate from "./searchGuesstimate";
-import searchWithAlgolia from "./searchWithAlgolia";
-
-export default async function searchAccordingToQueryData(
-  queryData: QueryParameters,
-  limit: number
-): Promise<FrontendQuestion[]> {
-  let results: FrontendQuestion[] = [];
-
-  try {
-    // defs
-    let query = queryData.query == undefined ? "" : queryData.query;
-    if (query == "") return [];
-    let forecastsThreshold = queryData.forecastsThreshold;
-    let starsThreshold = queryData.starsThreshold;
-    let platformsIncludeGuesstimate =
-      queryData.forecastingPlatforms.includes("guesstimate") &&
-      starsThreshold <= 1;
-
-    // preparation
-    let unawaitedAlgoliaResponse = searchWithAlgolia({
-      queryString: query,
-      hitsPerPage: limit + 50,
-      starsThreshold,
-      filterByPlatforms: queryData.forecastingPlatforms,
-      forecastsThreshold,
-    });
-
-    // consider the guesstimate and the non-guesstimate cases separately.
-    if (platformsIncludeGuesstimate) {
-      let responses = await Promise.all([
-        unawaitedAlgoliaResponse,
-        searchGuesstimate(query),
-      ]); // faster than two separate requests
-      let responsesNotGuesstimate = responses[0];
-      let responsesGuesstimate = responses[1];
-      results = [...responsesNotGuesstimate, ...responsesGuesstimate];
-      //results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
-    } else {
-      results = await unawaitedAlgoliaResponse;
-    }
-
-    return results;
-  } catch (error) {
-    console.log(error);
-  } finally {
-    console.log(results);
-    return results;
-  }
-}

src/web/worker/searchGuesstimate.ts

@@ -1,18 +1,18 @@
 /* Imports */
 import axios from "axios";
 
-import { FrontendQuestion } from "../platforms";
+import { AlgoliaQuestion } from "../../backend/utils/algolia";
 
 /* Definitions */
-let urlEndPoint =
+const urlEndPoint =
   "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c";
 
 /* Body */
 
 export default async function searchGuesstimate(
   query
-): Promise<FrontendQuestion[]> {
+): Promise<AlgoliaQuestion[]> {
-  let response = await axios({
+  const response = await axios({
     url: urlEndPoint,
     // credentials: "omit",
     headers: {
@@ -21,8 +21,6 @@ export default async function searchGuesstimate(
       "Accept-Language": "en-US,en;q=0.5",
       "content-type": "application/x-www-form-urlencoded",
     },
-    // referrer:
-    //   "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
     data: `{\"params\":\"query=${query.replace(
       / /g,
       "%20"
@@ -31,25 +29,26 @@ export default async function searchGuesstimate(
   });
 
   const models: any[] = response.data.hits;
-  const mappedModels: FrontendQuestion[] = models.map((model, index) => {
+  const mappedModels: AlgoliaQuestion[] = models.map((model, index) => {
-    let description = model.description
+    const description = model.description
       ? model.description.replace(/\n/g, " ").replace(/  /g, " ")
       : "";
-    let stars = description.length > 250 ? 2 : 1;
+    const stars = description.length > 250 ? 2 : 1;
     return {
       id: `guesstimate-${model.id}`,
       title: model.name,
       url: `https://www.getguesstimate.com/models/${model.id}`,
       timestamp: model.created_at, // TODO - check that format matches
       platform: "guesstimate",
-      platformLabel: "Guesstimate",
+      description,
-      description: description,
       options: [],
       qualityindicators: {
         stars: stars,
         numforecasts: 1,
         numforecasters: 1,
       },
+      stars,
+      extra: {},
       visualization: model.big_screenshot,
       ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
     };
@@ -57,7 +56,7 @@ export default async function searchGuesstimate(
 
   // filter for duplicates. Surprisingly common.
   let uniqueTitles = [];
-  let uniqueModels: FrontendQuestion[] = [];
+  let uniqueModels: AlgoliaQuestion[] = [];
   for (let model of mappedModels) {
     if (!uniqueTitles.includes(model.title) && !model.title.includes("copy")) {
       uniqueModels.push(model);

src/web/worker/searchWithAlgolia.ts

@@ -1,6 +1,6 @@
 import algoliasearch from "algoliasearch";
 
-import { FrontendQuestion } from "../platforms";
+import { AlgoliaQuestion } from "../../backend/utils/algolia";
 
 const client = algoliasearch(
   process.env.NEXT_PUBLIC_ALGOLIA_APP_ID,
@@ -13,22 +13,21 @@ let buildFilter = ({
   filterByPlatforms,
   forecastsThreshold,
 }) => {
-  let starsFilter = starsThreshold
+  const starsFilter = starsThreshold
     ? `qualityindicators.stars >= ${starsThreshold}`
     : null;
-  let platformsFilter = filterByPlatforms
+  const platformsFilter = filterByPlatforms
     ? filterByPlatforms.map((platform) => `platform:"${platform}"`).join(" OR ")
     : null;
-  console.log(platformsFilter);
+  const numForecastsFilter =
-  // let numForecastsFilter = forecastsThreshold ? `has_numforecasts:true AND qualityindicators.numforecasts >= ${forecastsThreshold}` : null
-  let numForecastsFilter =
     forecastsThreshold > 0
       ? `qualityindicators.numforecasts >= ${forecastsThreshold}`
       : null;
-  let finalFilter = [starsFilter, platformsFilter, numForecastsFilter]
+  const finalFilter = [starsFilter, platformsFilter, numForecastsFilter]
     .filter((f) => f != null)
     .map((f) => `( ${f} )`)
     .join(" AND ");
+
   console.log(
     "searchWithAlgolia.js/searchWithAlgolia/buildFilter",
     finalFilter
@@ -51,18 +50,6 @@ let buildFacetFilter = ({ filterByPlatforms }) => {
   return platformsFilter;
 };
 
-let normalizeArray = (array) => {
-  if (array.length == 0) {
-    return [];
-  }
-  let mean = array.reduce((a, b) => a + b) / array.length;
-  let sd = Math.sqrt(
-    array.map((x) => Math.pow(x - mean, 2)).reduce((a, b) => a + b)
-  );
-  let normalizedArray = array.map((result) => (result - sd) / mean);
-  return normalizedArray;
-};
-
 let noExactMatch = (queryString, result) => {
   queryString = queryString.toLowerCase();
   let title = result.title.toLowerCase();
@@ -75,6 +62,14 @@ let noExactMatch = (queryString, result) => {
   );
 };
 
+interface SearchOpts {
+  queryString: string;
+  hitsPerPage?: number;
+  starsThreshold: number;
+  filterByPlatforms: string[];
+  forecastsThreshold: number;
+}
+
 // only query string
 export default async function searchWithAlgolia({
   queryString,
@@ -82,8 +77,8 @@ export default async function searchWithAlgolia({
   starsThreshold,
   filterByPlatforms,
   forecastsThreshold,
-}): Promise<FrontendQuestion[]> {
+}: SearchOpts): Promise<AlgoliaQuestion[]> {
-  let response = await index.search<FrontendQuestion>(queryString, {
+  const response = await index.search<AlgoliaQuestion>(queryString, {
     hitsPerPage,
     filters: buildFilter({
       starsThreshold,
@@ -93,7 +88,7 @@ export default async function searchWithAlgolia({
     //facetFilters: buildFacetFilter({filterByPlatforms}),
     getRankingInfo: true,
   });
-  let results: FrontendQuestion[] = response.hits;
+  let results = response.hits;
 
   let recursionError = ["metaforecast", "metaforecasts", "metaforecasting"];
   if (
@@ -103,10 +98,10 @@ export default async function searchWithAlgolia({
     results = [
       {
         id: "not-found",
+        objectID: "not-found",
         title: "No search results match your query",
         url: "https://metaforecast.org",
         platform: "metaforecast",
-        platformLabel: "Metaforecast",
         description: "Maybe try a broader query?",
         options: [
           {
@@ -121,24 +116,23 @@ export default async function searchWithAlgolia({
           },
         ],
         timestamp: `${new Date().toISOString().slice(0, 10)}`,
+        stars: 5, // legacy
         qualityindicators: {
           numforecasts: 1,
           numforecasters: 1,
           stars: 5,
         },
-        // noExactSearchResults: true,
+        extra: {},
-        // optionsstringforsearch: "Yes, No",
-        // has_numforecasts: true,
       },
     ];
   } else if (recursionError.includes(queryString.toLowerCase())) {
     results = [
       {
         id: "recursion-error",
+        objectID: "recursion-error",
         title: `Did you mean: ${queryString}?`,
         url: "https://metaforecast.org/recursion?bypassEasterEgg=true",
         platform: "metaforecast",
-        platformLabel: "Metaforecast",
         description:
           "Fatal error: Too much recursion. Click to proceed anyways",
         options: [
@@ -154,14 +148,13 @@ export default async function searchWithAlgolia({
           },
         ],
         timestamp: `${new Date().toISOString().slice(0, 10)}`,
+        stars: 5, // legacy
         qualityindicators: {
           numforecasts: 1,
           numforecasters: 1,
           stars: 5,
         },
-        // noExactSearchResults: true,
+        extra: {},
-        // optionsstringforsearch: "Yes, No",
-        // has_numforecasts: true,
       },
       ...results,
     ];
@@ -172,10 +165,10 @@ export default async function searchWithAlgolia({
   ) {
     results.unshift({
       id: "not-found-2",
+      objectID: "not-found-2",
       title: "No search results appear to match your query",
       url: "https://metaforecast.org",
       platform: "metaforecast",
-      platformLabel: "Metaforecast",
       description: "Maybe try a broader query? That said, we could be wrong.",
       options: [
         {
@@ -190,17 +183,14 @@ export default async function searchWithAlgolia({
         },
       ],
       timestamp: `${new Date().toISOString().slice(0, 10)}`,
+      stars: 1, // legacy
       qualityindicators: {
         numforecasts: 1,
         numforecasters: 1,
         stars: 1,
       },
-      // noExactSearchResults: true,
+      extra: {},
-      // optionsstringforsearch: "Yes, No",
-      // has_numforecasts: true,
     });
-  } else {
-    // results[0].noExactSearchResults = false;
   }
 
   return results;