From 42c0f0967b603d12bedc44f2829e5793b83dd88a Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Sat, 26 Mar 2022 02:51:11 +0300 Subject: [PATCH] feat: back to frontpage-in-db approach --- package.json | 2 +- src/backend/database/pg-wrapper.js | 23 ++++++++ src/backend/frontpage.ts | 58 ++++++++++++++----- src/backend/{index.js => index.ts} | 23 +++++--- src/pages/capture.tsx | 39 ++++--------- src/pages/index.tsx | 39 ++++--------- .../CommonDisplay.tsx} | 44 +++++--------- src/web/search/anySearchPage.tsx | 42 ++++++++++++++ 8 files changed, 162 insertions(+), 108 deletions(-) rename src/backend/{index.js => index.ts} (78%) rename src/web/{display/commonDisplay.tsx => search/CommonDisplay.tsx} (92%) create mode 100644 src/web/search/anySearchPage.tsx diff --git a/package.json b/package.json index b73e5c1..36031dc 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ }, "homepage": "https://github.com/QURIresearch/metaforecasts#readme", "scripts": { - "cli": "ts-node src/backend/index.js", + "cli": "ts-node src/backend/index.ts", "reload": "heroku run:detached node src/backend/utils/doEverythingForScheduler.js", "setCookies": "./src/backend/utils/setCookies.sh", "next-dev": "next dev", diff --git a/src/backend/database/pg-wrapper.js b/src/backend/database/pg-wrapper.js index f7434ea..a6df7d3 100644 --- a/src/backend/database/pg-wrapper.js +++ b/src/backend/database/pg-wrapper.js @@ -302,11 +302,34 @@ export async function pgInitializeHistories() { } } +async function pgInitializeFrontpage() { + let YOLO = false; + if (YOLO) { + await runPgCommand({ + command: dropTable("latest", "frontpage"), + pool: readWritePool, + }); + await runPgCommand({ + command: `CREATE TABLE latest.frontpage ( + id serial primary key, + frontpage_full jsonb, + frontpage_sliced jsonb + );`, + pool: readWritePool, + }); + } else { + console.log( + "pgInitializeFrontpage: This command is dangerous, set YOLO to true in the code to invoke it" + ); + } +} + export async function pgInitialize() { await pgInitializeScaffolding(); await pgInitializeLatest(); await pgInitializeHistories(); await pgInitializeDashboards(); + await pgInitializeFrontpage(); } // Read diff --git a/src/backend/frontpage.ts b/src/backend/frontpage.ts index c85f373..419a356 100644 --- a/src/backend/frontpage.ts +++ b/src/backend/frontpage.ts @@ -2,23 +2,22 @@ import { pgRead, readWritePool } from './database/pg-wrapper'; export async function getFrontpageRaw() { const client = await readWritePool.connect(); - const res = await client.query(` - SELECT * FROM latest.combined - WHERE - (qualityindicators->>'stars')::int >= 3 - AND description != '' - AND JSON_ARRAY_LENGTH(options) > 0 - ORDER BY RANDOM() LIMIT 50 - `); - - return res.rows; + const res = await client.query( + "SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1" + ); + if (!res.rows.length) return []; + console.log(res.rows[0].frontpage_sliced); + return res.rows[0].frontpage_sliced; } export async function getFrontpageFullRaw() { - return await pgRead({ - schema: "latest", - tableName: "combined", - }); + const client = await readWritePool.connect(); + const res = await client.query( + "SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1" + ); + if (!res.rows.length) return []; + console.log(res.rows[0]); + return res.rows[0].frontpage_full; } export async function getFrontpage() { @@ -36,3 +35,34 @@ export async function getFrontpage() { return frontPageForecastsCompatibleWithFuse; } } + +export async function rebuildFrontpage() { + const frontpageFull = await pgRead({ + schema: "latest", + tableName: "combined", + }); + + const client = await readWritePool.connect(); + const frontpageSliced = ( + await client.query(` + SELECT * FROM latest.combined + WHERE + (qualityindicators->>'stars')::int >= 3 + AND description != '' + AND JSON_ARRAY_LENGTH(options) > 0 + ORDER BY RANDOM() LIMIT 50 + `) + ).rows; + + const start = Date.now(); + await client.query( + "INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)", + [JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)] + ); + + const end = Date.now(); + const difference = end - start; + console.log( + `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` + ); +} diff --git a/src/backend/index.js b/src/backend/index.ts similarity index 78% rename from src/backend/index.js rename to src/backend/index.ts index 94e1d84..350c5bc 100644 --- a/src/backend/index.js +++ b/src/backend/index.ts @@ -1,13 +1,16 @@ /* Imports */ -import "dotenv/config"; -import readline from "readline"; -import { pgInitialize } from "./database/pg-wrapper.js"; -import { doEverything, tryCatchTryAgain } from "./flow/doEverything.js"; -import { updateHistory } from "./flow/history/updateHistory.js"; -import { mergeEverything } from "./flow/mergeEverything.js"; -import { rebuildNetlifySiteWithNewData } from "./flow/rebuildNetliftySiteWithNewData.js"; -import { platformFetchers } from "./platforms/all-platforms.js"; -import { rebuildAlgoliaDatabase } from "./utils/algolia.js"; +import 'dotenv/config'; + +import readline from 'readline'; + +import { pgInitialize } from './database/pg-wrapper.js'; +import { doEverything, tryCatchTryAgain } from './flow/doEverything.js'; +import { updateHistory } from './flow/history/updateHistory.js'; +import { mergeEverything } from './flow/mergeEverything.js'; +import { rebuildNetlifySiteWithNewData } from './flow/rebuildNetliftySiteWithNewData.js'; +import { rebuildFrontpage } from './frontpage'; +import { platformFetchers } from './platforms/all-platforms.js'; +import { rebuildAlgoliaDatabase } from './utils/algolia.js'; /* Support functions */ let functions = [ @@ -18,6 +21,7 @@ let functions = [ rebuildNetlifySiteWithNewData, doEverything, pgInitialize, + rebuildFrontpage, ]; let functionNames = functions.map((fun) => fun.name); @@ -34,6 +38,7 @@ let generateWhatToDoMessage = () => { // `\n[${functionNames.length-1}]: Add to history` + `All of the above`, `Initialize postgres database`, + "Rebuild frontpage", ]; let otherMessagesWithNums = otherMessages.map( (message, i) => `[${i + l}]: ${message}` diff --git a/src/pages/capture.tsx b/src/pages/capture.tsx index 32579f4..40e09fe 100644 --- a/src/pages/capture.tsx +++ b/src/pages/capture.tsx @@ -1,41 +1,24 @@ -import { GetStaticProps, NextPage } from 'next'; +import { NextPage } from 'next'; import React from 'react'; -import { getFrontpage } from '../backend/frontpage'; -import CommonDisplay from '../web/display/commonDisplay'; import { displayForecastsWrapperForCapture } from '../web/display/displayForecastsWrappers'; +import { Props } from '../web/search/anySearchPage'; +import CommonDisplay from '../web/search/commonDisplay'; import Layout from './layout'; -/* get Props */ +export { getServerSideProps } from "../web/search/anySearchPage"; -interface Props { - defaultResults: any; -} - -export const getStaticProps: GetStaticProps = async (context) => { - let frontPageForecasts = await getFrontpage(); - frontPageForecasts = frontPageForecasts.map((forecast) => ({ - ...forecast, - item: { - ...forecast.item, - timestamp: forecast.item.timestamp.toJSON(), - }, - })); - - return { - props: { - defaultResults: frontPageForecasts, - }, - revalidate: 3600 * 6, - }; -}; - -/* Body */ -const CapturePage: NextPage = ({ defaultResults }) => { +const CapturePage: NextPage = ({ + defaultResults, + initialResults, + initialQueryParameters, +}) => { return ( = async (context) => { - let frontPageForecasts = await getFrontpage(); - frontPageForecasts = frontPageForecasts.map((forecast) => ({ - ...forecast, - item: { - ...forecast.item, - timestamp: forecast.item.timestamp.toJSON(), - }, - })); - - return { - props: { - defaultResults: frontPageForecasts, - }, - revalidate: 3600 * 6, - }; -}; - -/* Body */ -const IndexPage: NextPage = ({ defaultResults }) => { +const IndexPage: NextPage = ({ + defaultResults, + initialResults, + initialQueryParameters, +}) => { return ( React.ReactNode; } -const defaultQueryParameters: QueryParametersWithoutNum = { +export const defaultQueryParameters: QueryParametersWithoutNum = { query: "", starsThreshold: 2, forecastsThreshold: 0, forecastingPlatforms: platformsWithLabels, // weird key value format, }; -const defaultNumDisplay = 21; +export const defaultNumDisplay = 21; /* Body */ const CommonDisplay: React.FC = ({ defaultResults, + initialResults, + initialQueryParameters, hasSearchbar, hasCapture, hasAdvancedOptions, @@ -56,36 +60,21 @@ const CommonDisplay: React.FC = ({ /* States */ const [queryParameters, setQueryParameters] = - useState(defaultQueryParameters); + useState(initialQueryParameters); - const [numDisplay, setNumDisplay] = useState(0); - - const [ready, setReady] = useState(false); + const [numDisplay, setNumDisplay] = useState( + initialQueryParameters.numDisplay ?? defaultNumDisplay + ); // used to distinguish numDisplay updates which force search and don't force search, see effects below const [forceSearch, setForceSearch] = useState(0); - const [results, setResults] = useState([]); + const [results, setResults] = useState(initialResults); const [advancedOptions, showAdvancedOptions] = useState(false); const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] = useState(0); const [showIdToggle, setShowIdToggle] = useState(false); - useEffect(() => { - if (!router.isReady) return; - - setQueryParameters({ - ...defaultQueryParameters, - ...router.query, - }); - setNumDisplay( - typeof router.query.numDisplay === "string" - ? parseInt(router.query.numDisplay) - : defaultNumDisplay - ); - setReady(true); - }, [router.isReady]); - /* Functions which I want to have access to the Home namespace */ // I don't want to create an "defaultResults" object for each search. async function executeSearchOrAnswerWithDefaultResults() { @@ -173,7 +162,6 @@ const CommonDisplay: React.FC = ({ useEffect(updateRoute, [numDisplay]); useEffect(() => { - if (!ready) return; setResults([]); let newTimeoutId = setTimeout(() => { updateRoute(); @@ -184,7 +172,7 @@ const CommonDisplay: React.FC = ({ return () => { clearTimeout(newTimeoutId); }; - }, [ready, queryParameters, forceSearch]); + }, [queryParameters, forceSearch]); /* State controllers */ diff --git a/src/web/search/anySearchPage.tsx b/src/web/search/anySearchPage.tsx new file mode 100644 index 0000000..aaa5628 --- /dev/null +++ b/src/web/search/anySearchPage.tsx @@ -0,0 +1,42 @@ +import { GetServerSideProps } from 'next'; + +import { getFrontpage } from '../../backend/frontpage'; +import searchAccordingToQueryData from '../worker/searchAccordingToQueryData'; +import { defaultNumDisplay, defaultQueryParameters, QueryParameters } from './commonDisplay'; + +/* Common code for / and /capture */ + +export interface Props { + defaultResults: any; + initialResults: any; + initialQueryParameters: QueryParameters; +} + +export const getServerSideProps: GetServerSideProps = async ( + context +) => { + let urlQuery = context.query; + + let initialQueryParameters: QueryParameters = { + ...defaultQueryParameters, + numDisplay: defaultNumDisplay, + ...urlQuery, // FIXME - parse numerical fields + }; + + let defaultResults = await getFrontpage(); + + const initialResults = + !!initialQueryParameters && + initialQueryParameters.query != "" && + initialQueryParameters.query != undefined + ? await searchAccordingToQueryData(initialQueryParameters) + : defaultResults; + + return { + props: { + initialQueryParameters, + initialResults, + defaultResults, + }, + }; +};