feat: back to frontpage-in-db approach

package.json

@@ -17,7 +17,7 @@
   },
   "homepage": "https://github.com/QURIresearch/metaforecasts#readme",
   "scripts": {
-    "cli": "ts-node src/backend/index.js",
+    "cli": "ts-node src/backend/index.ts",
     "reload": "heroku run:detached node src/backend/utils/doEverythingForScheduler.js",
     "setCookies": "./src/backend/utils/setCookies.sh",
     "next-dev": "next dev",

src/backend/database/pg-wrapper.js

@@ -302,11 +302,34 @@ export async function pgInitializeHistories() {
   }
 }
 
+async function pgInitializeFrontpage() {
+  let YOLO = false;
+  if (YOLO) {
+    await runPgCommand({
+      command: dropTable("latest", "frontpage"),
+      pool: readWritePool,
+    });
+    await runPgCommand({
+      command: `CREATE TABLE latest.frontpage (
+        id serial primary key,
+        frontpage_full jsonb,
+        frontpage_sliced jsonb
+      );`,
+      pool: readWritePool,
+    });
+  } else {
+    console.log(
+      "pgInitializeFrontpage: This command is dangerous, set YOLO to true in the code to invoke it"
+    );
+  }
+}
+
 export async function pgInitialize() {
   await pgInitializeScaffolding();
   await pgInitializeLatest();
   await pgInitializeHistories();
   await pgInitializeDashboards();
+  await pgInitializeFrontpage();
 }
 
 // Read

src/backend/frontpage.ts

@@ -2,23 +2,22 @@ import { pgRead, readWritePool } from './database/pg-wrapper';
 
 export async function getFrontpageRaw() {
   const client = await readWritePool.connect();
-  const res = await client.query(`
+  const res = await client.query(
-    SELECT * FROM latest.combined
+    "SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1"
-    WHERE
+  );
-      (qualityindicators->>'stars')::int >= 3
+  if (!res.rows.length) return [];
-      AND description != ''
+  console.log(res.rows[0].frontpage_sliced);
-      AND JSON_ARRAY_LENGTH(options) > 0
+  return res.rows[0].frontpage_sliced;
-    ORDER BY RANDOM() LIMIT 50
-  `);
-
-  return res.rows;
 }
 
 export async function getFrontpageFullRaw() {
-  return await pgRead({
+  const client = await readWritePool.connect();
-    schema: "latest",
+  const res = await client.query(
-    tableName: "combined",
+    "SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1"
-  });
+  );
+  if (!res.rows.length) return [];
+  console.log(res.rows[0]);
+  return res.rows[0].frontpage_full;
 }
 
 export async function getFrontpage() {
@@ -36,3 +35,34 @@ export async function getFrontpage() {
     return frontPageForecastsCompatibleWithFuse;
   }
 }
+
+export async function rebuildFrontpage() {
+  const frontpageFull = await pgRead({
+    schema: "latest",
+    tableName: "combined",
+  });
+
+  const client = await readWritePool.connect();
+  const frontpageSliced = (
+    await client.query(`
+    SELECT * FROM latest.combined
+    WHERE
+      (qualityindicators->>'stars')::int >= 3
+      AND description != ''
+      AND JSON_ARRAY_LENGTH(options) > 0
+    ORDER BY RANDOM() LIMIT 50
+  `)
+  ).rows;
+
+  const start = Date.now();
+  await client.query(
+    "INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
+    [JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
+  );
+
+  const end = Date.now();
+  const difference = end - start;
+  console.log(
+    `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
+  );
+}

src/backend/index.ts

@@ -1,13 +1,16 @@
 /* Imports */
-import "dotenv/config";
+import 'dotenv/config';
-import readline from "readline";
+
-import { pgInitialize } from "./database/pg-wrapper.js";
+import readline from 'readline';
-import { doEverything, tryCatchTryAgain } from "./flow/doEverything.js";
+
-import { updateHistory } from "./flow/history/updateHistory.js";
+import { pgInitialize } from './database/pg-wrapper.js';
-import { mergeEverything } from "./flow/mergeEverything.js";
+import { doEverything, tryCatchTryAgain } from './flow/doEverything.js';
-import { rebuildNetlifySiteWithNewData } from "./flow/rebuildNetliftySiteWithNewData.js";
+import { updateHistory } from './flow/history/updateHistory.js';
-import { platformFetchers } from "./platforms/all-platforms.js";
+import { mergeEverything } from './flow/mergeEverything.js';
-import { rebuildAlgoliaDatabase } from "./utils/algolia.js";
+import { rebuildNetlifySiteWithNewData } from './flow/rebuildNetliftySiteWithNewData.js';
+import { rebuildFrontpage } from './frontpage';
+import { platformFetchers } from './platforms/all-platforms.js';
+import { rebuildAlgoliaDatabase } from './utils/algolia.js';
 
 /* Support functions */
 let functions = [
@@ -18,6 +21,7 @@ let functions = [
   rebuildNetlifySiteWithNewData,
   doEverything,
   pgInitialize,
+  rebuildFrontpage,
 ];
 let functionNames = functions.map((fun) => fun.name);
 
@@ -34,6 +38,7 @@ let generateWhatToDoMessage = () => {
     // `\n[${functionNames.length-1}]: Add to history` +
     `All of the above`,
     `Initialize postgres database`,
+    "Rebuild frontpage",
   ];
   let otherMessagesWithNums = otherMessages.map(
     (message, i) => `[${i + l}]: ${message}`

src/pages/capture.tsx

@@ -1,41 +1,24 @@
-import { GetStaticProps, NextPage } from 'next';
+import { NextPage } from 'next';
 import React from 'react';
 
-import { getFrontpage } from '../backend/frontpage';
-import CommonDisplay from '../web/display/commonDisplay';
 import { displayForecastsWrapperForCapture } from '../web/display/displayForecastsWrappers';
+import { Props } from '../web/search/anySearchPage';
+import CommonDisplay from '../web/search/commonDisplay';
 import Layout from './layout';
 
-/* get Props */
+export { getServerSideProps } from "../web/search/anySearchPage";
 
-interface Props {
+const CapturePage: NextPage<Props> = ({
-  defaultResults: any;
+  defaultResults,
-}
+  initialResults,
-
+  initialQueryParameters,
-export const getStaticProps: GetStaticProps<Props> = async (context) => {
+}) => {
-  let frontPageForecasts = await getFrontpage();
-  frontPageForecasts = frontPageForecasts.map((forecast) => ({
-    ...forecast,
-    item: {
-      ...forecast.item,
-      timestamp: forecast.item.timestamp.toJSON(),
-    },
-  }));
-
-  return {
-    props: {
-      defaultResults: frontPageForecasts,
-    },
-    revalidate: 3600 * 6,
-  };
-};
-
-/* Body */
-const CapturePage: NextPage<Props> = ({ defaultResults }) => {
   return (
     <Layout page={"capture"}>
       <CommonDisplay
         defaultResults={defaultResults}
+        initialResults={initialResults}
+        initialQueryParameters={initialQueryParameters}
         hasSearchbar={true}
         hasCapture={true}
         hasAdvancedOptions={false}

src/pages/index.tsx

@@ -1,41 +1,24 @@
-import { GetStaticProps, NextPage } from 'next';
+import { NextPage } from 'next';
 import React from 'react';
 
-import { getFrontpage } from '../backend/frontpage';
-import CommonDisplay from '../web/display/commonDisplay';
 import { displayForecastsWrapperForSearch } from '../web/display/displayForecastsWrappers';
+import { Props } from '../web/search/anySearchPage';
+import CommonDisplay from '../web/search/commonDisplay';
 import Layout from './layout';
 
-/* get Props */
+export { getServerSideProps } from "../web/search/anySearchPage";
 
-interface Props {
+const IndexPage: NextPage<Props> = ({
-  defaultResults: any;
+  defaultResults,
-}
+  initialResults,
-
+  initialQueryParameters,
-export const getStaticProps: GetStaticProps<Props> = async (context) => {
+}) => {
-  let frontPageForecasts = await getFrontpage();
-  frontPageForecasts = frontPageForecasts.map((forecast) => ({
-    ...forecast,
-    item: {
-      ...forecast.item,
-      timestamp: forecast.item.timestamp.toJSON(),
-    },
-  }));
-
-  return {
-    props: {
-      defaultResults: frontPageForecasts,
-    },
-    revalidate: 3600 * 6,
-  };
-};
-
-/* Body */
-const IndexPage: NextPage<Props> = ({ defaultResults }) => {
   return (
     <Layout page={"search"}>
       <CommonDisplay
         defaultResults={defaultResults}
+        initialResults={initialResults}
+        initialQueryParameters={initialQueryParameters}
         hasSearchbar={true}
         hasCapture={false}
         hasAdvancedOptions={true}

src/web/search/CommonDisplay.tsx

@@ -1,12 +1,12 @@
 import { useRouter } from 'next/router';
 import React, { Fragment, useEffect, useState } from 'react';
 
+import ButtonsForStars from '../display/buttonsForStars';
+import Form from '../display/form';
+import MultiSelectPlatform from '../display/multiSelectPlatforms';
+import { SliderElement } from '../display/slider';
 import { platformsWithLabels, PlatformWithLabel } from '../platforms';
 import searchAccordingToQueryData from '../worker/searchAccordingToQueryData';
-import ButtonsForStars from './buttonsForStars';
-import Form from './form';
-import MultiSelectPlatform from './multiSelectPlatforms';
-import { SliderElement } from './slider';
 
 interface QueryParametersWithoutNum {
   query: string;
@@ -21,6 +21,8 @@ export interface QueryParameters extends QueryParametersWithoutNum {
 
 interface Props {
   defaultResults: any;
+  initialResults: any;
+  initialQueryParameters: QueryParameters;
   hasSearchbar: boolean;
   hasCapture: boolean;
   hasAdvancedOptions: boolean;
@@ -34,17 +36,19 @@ interface Props {
   }) => React.ReactNode;
 }
 
-const defaultQueryParameters: QueryParametersWithoutNum = {
+export const defaultQueryParameters: QueryParametersWithoutNum = {
   query: "",
   starsThreshold: 2,
   forecastsThreshold: 0,
   forecastingPlatforms: platformsWithLabels, // weird key value format,
 };
-const defaultNumDisplay = 21;
+export const defaultNumDisplay = 21;
 
 /* Body */
 const CommonDisplay: React.FC<Props> = ({
   defaultResults,
+  initialResults,
+  initialQueryParameters,
   hasSearchbar,
   hasCapture,
   hasAdvancedOptions,
@@ -56,36 +60,21 @@ const CommonDisplay: React.FC<Props> = ({
   /* States */
 
   const [queryParameters, setQueryParameters] =
-    useState<QueryParametersWithoutNum>(defaultQueryParameters);
+    useState<QueryParametersWithoutNum>(initialQueryParameters);
 
-  const [numDisplay, setNumDisplay] = useState(0);
+  const [numDisplay, setNumDisplay] = useState(
-
+    initialQueryParameters.numDisplay ?? defaultNumDisplay
-  const [ready, setReady] = useState(false);
+  );
 
   // used to distinguish numDisplay updates which force search and don't force search, see effects below
   const [forceSearch, setForceSearch] = useState(0);
 
-  const [results, setResults] = useState([]);
+  const [results, setResults] = useState(initialResults);
   const [advancedOptions, showAdvancedOptions] = useState(false);
   const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] =
     useState(0);
   const [showIdToggle, setShowIdToggle] = useState(false);
 
-  useEffect(() => {
-    if (!router.isReady) return;
-
-    setQueryParameters({
-      ...defaultQueryParameters,
-      ...router.query,
-    });
-    setNumDisplay(
-      typeof router.query.numDisplay === "string"
-        ? parseInt(router.query.numDisplay)
-        : defaultNumDisplay
-    );
-    setReady(true);
-  }, [router.isReady]);
-
   /* Functions which I want to have access to the Home namespace */
   // I don't want to create an "defaultResults" object for each search.
   async function executeSearchOrAnswerWithDefaultResults() {
@@ -173,7 +162,6 @@ const CommonDisplay: React.FC<Props> = ({
   useEffect(updateRoute, [numDisplay]);
 
   useEffect(() => {
-    if (!ready) return;
     setResults([]);
     let newTimeoutId = setTimeout(() => {
       updateRoute();
@@ -184,7 +172,7 @@ const CommonDisplay: React.FC<Props> = ({
     return () => {
       clearTimeout(newTimeoutId);
     };
-  }, [ready, queryParameters, forceSearch]);
+  }, [queryParameters, forceSearch]);
 
   /* State controllers */
 

src/web/search/anySearchPage.tsx

@@ -0,0 +1,42 @@
+import { GetServerSideProps } from 'next';
+
+import { getFrontpage } from '../../backend/frontpage';
+import searchAccordingToQueryData from '../worker/searchAccordingToQueryData';
+import { defaultNumDisplay, defaultQueryParameters, QueryParameters } from './commonDisplay';
+
+/* Common code for / and /capture */
+
+export interface Props {
+  defaultResults: any;
+  initialResults: any;
+  initialQueryParameters: QueryParameters;
+}
+
+export const getServerSideProps: GetServerSideProps<Props> = async (
+  context
+) => {
+  let urlQuery = context.query;
+
+  let initialQueryParameters: QueryParameters = {
+    ...defaultQueryParameters,
+    numDisplay: defaultNumDisplay,
+    ...urlQuery, // FIXME - parse numerical fields
+  };
+
+  let defaultResults = await getFrontpage();
+
+  const initialResults =
+    !!initialQueryParameters &&
+    initialQueryParameters.query != "" &&
+    initialQueryParameters.query != undefined
+      ? await searchAccordingToQueryData(initialQueryParameters)
+      : defaultResults;
+
+  return {
+    props: {
+      initialQueryParameters,
+      initialResults,
+      defaultResults,
+    },
+  };
+};