feat: back to frontpage-in-db approach

This commit is contained in:
Vyacheslav Matyukhin 2022-03-26 02:51:11 +03:00
parent 80ee4c4055
commit 42c0f0967b
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
8 changed files with 162 additions and 108 deletions

View File

@ -17,7 +17,7 @@
}, },
"homepage": "https://github.com/QURIresearch/metaforecasts#readme", "homepage": "https://github.com/QURIresearch/metaforecasts#readme",
"scripts": { "scripts": {
"cli": "ts-node src/backend/index.js", "cli": "ts-node src/backend/index.ts",
"reload": "heroku run:detached node src/backend/utils/doEverythingForScheduler.js", "reload": "heroku run:detached node src/backend/utils/doEverythingForScheduler.js",
"setCookies": "./src/backend/utils/setCookies.sh", "setCookies": "./src/backend/utils/setCookies.sh",
"next-dev": "next dev", "next-dev": "next dev",

View File

@ -302,11 +302,34 @@ export async function pgInitializeHistories() {
} }
} }
async function pgInitializeFrontpage() {
let YOLO = false;
if (YOLO) {
await runPgCommand({
command: dropTable("latest", "frontpage"),
pool: readWritePool,
});
await runPgCommand({
command: `CREATE TABLE latest.frontpage (
id serial primary key,
frontpage_full jsonb,
frontpage_sliced jsonb
);`,
pool: readWritePool,
});
} else {
console.log(
"pgInitializeFrontpage: This command is dangerous, set YOLO to true in the code to invoke it"
);
}
}
export async function pgInitialize() { export async function pgInitialize() {
await pgInitializeScaffolding(); await pgInitializeScaffolding();
await pgInitializeLatest(); await pgInitializeLatest();
await pgInitializeHistories(); await pgInitializeHistories();
await pgInitializeDashboards(); await pgInitializeDashboards();
await pgInitializeFrontpage();
} }
// Read // Read

View File

@ -2,23 +2,22 @@ import { pgRead, readWritePool } from './database/pg-wrapper';
export async function getFrontpageRaw() { export async function getFrontpageRaw() {
const client = await readWritePool.connect(); const client = await readWritePool.connect();
const res = await client.query(` const res = await client.query(
SELECT * FROM latest.combined "SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1"
WHERE );
(qualityindicators->>'stars')::int >= 3 if (!res.rows.length) return [];
AND description != '' console.log(res.rows[0].frontpage_sliced);
AND JSON_ARRAY_LENGTH(options) > 0 return res.rows[0].frontpage_sliced;
ORDER BY RANDOM() LIMIT 50
`);
return res.rows;
} }
export async function getFrontpageFullRaw() { export async function getFrontpageFullRaw() {
return await pgRead({ const client = await readWritePool.connect();
schema: "latest", const res = await client.query(
tableName: "combined", "SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1"
}); );
if (!res.rows.length) return [];
console.log(res.rows[0]);
return res.rows[0].frontpage_full;
} }
export async function getFrontpage() { export async function getFrontpage() {
@ -36,3 +35,34 @@ export async function getFrontpage() {
return frontPageForecastsCompatibleWithFuse; return frontPageForecastsCompatibleWithFuse;
} }
} }
export async function rebuildFrontpage() {
const frontpageFull = await pgRead({
schema: "latest",
tableName: "combined",
});
const client = await readWritePool.connect();
const frontpageSliced = (
await client.query(`
SELECT * FROM latest.combined
WHERE
(qualityindicators->>'stars')::int >= 3
AND description != ''
AND JSON_ARRAY_LENGTH(options) > 0
ORDER BY RANDOM() LIMIT 50
`)
).rows;
const start = Date.now();
await client.query(
"INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
[JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
);
const end = Date.now();
const difference = end - start;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}

View File

@ -1,13 +1,16 @@
/* Imports */ /* Imports */
import "dotenv/config"; import 'dotenv/config';
import readline from "readline";
import { pgInitialize } from "./database/pg-wrapper.js"; import readline from 'readline';
import { doEverything, tryCatchTryAgain } from "./flow/doEverything.js";
import { updateHistory } from "./flow/history/updateHistory.js"; import { pgInitialize } from './database/pg-wrapper.js';
import { mergeEverything } from "./flow/mergeEverything.js"; import { doEverything, tryCatchTryAgain } from './flow/doEverything.js';
import { rebuildNetlifySiteWithNewData } from "./flow/rebuildNetliftySiteWithNewData.js"; import { updateHistory } from './flow/history/updateHistory.js';
import { platformFetchers } from "./platforms/all-platforms.js"; import { mergeEverything } from './flow/mergeEverything.js';
import { rebuildAlgoliaDatabase } from "./utils/algolia.js"; import { rebuildNetlifySiteWithNewData } from './flow/rebuildNetliftySiteWithNewData.js';
import { rebuildFrontpage } from './frontpage';
import { platformFetchers } from './platforms/all-platforms.js';
import { rebuildAlgoliaDatabase } from './utils/algolia.js';
/* Support functions */ /* Support functions */
let functions = [ let functions = [
@ -18,6 +21,7 @@ let functions = [
rebuildNetlifySiteWithNewData, rebuildNetlifySiteWithNewData,
doEverything, doEverything,
pgInitialize, pgInitialize,
rebuildFrontpage,
]; ];
let functionNames = functions.map((fun) => fun.name); let functionNames = functions.map((fun) => fun.name);
@ -34,6 +38,7 @@ let generateWhatToDoMessage = () => {
// `\n[${functionNames.length-1}]: Add to history` + // `\n[${functionNames.length-1}]: Add to history` +
`All of the above`, `All of the above`,
`Initialize postgres database`, `Initialize postgres database`,
"Rebuild frontpage",
]; ];
let otherMessagesWithNums = otherMessages.map( let otherMessagesWithNums = otherMessages.map(
(message, i) => `[${i + l}]: ${message}` (message, i) => `[${i + l}]: ${message}`

View File

@ -1,41 +1,24 @@
import { GetStaticProps, NextPage } from 'next'; import { NextPage } from 'next';
import React from 'react'; import React from 'react';
import { getFrontpage } from '../backend/frontpage';
import CommonDisplay from '../web/display/commonDisplay';
import { displayForecastsWrapperForCapture } from '../web/display/displayForecastsWrappers'; import { displayForecastsWrapperForCapture } from '../web/display/displayForecastsWrappers';
import { Props } from '../web/search/anySearchPage';
import CommonDisplay from '../web/search/commonDisplay';
import Layout from './layout'; import Layout from './layout';
/* get Props */ export { getServerSideProps } from "../web/search/anySearchPage";
interface Props { const CapturePage: NextPage<Props> = ({
defaultResults: any; defaultResults,
} initialResults,
initialQueryParameters,
export const getStaticProps: GetStaticProps<Props> = async (context) => { }) => {
let frontPageForecasts = await getFrontpage();
frontPageForecasts = frontPageForecasts.map((forecast) => ({
...forecast,
item: {
...forecast.item,
timestamp: forecast.item.timestamp.toJSON(),
},
}));
return {
props: {
defaultResults: frontPageForecasts,
},
revalidate: 3600 * 6,
};
};
/* Body */
const CapturePage: NextPage<Props> = ({ defaultResults }) => {
return ( return (
<Layout page={"capture"}> <Layout page={"capture"}>
<CommonDisplay <CommonDisplay
defaultResults={defaultResults} defaultResults={defaultResults}
initialResults={initialResults}
initialQueryParameters={initialQueryParameters}
hasSearchbar={true} hasSearchbar={true}
hasCapture={true} hasCapture={true}
hasAdvancedOptions={false} hasAdvancedOptions={false}

View File

@ -1,41 +1,24 @@
import { GetStaticProps, NextPage } from 'next'; import { NextPage } from 'next';
import React from 'react'; import React from 'react';
import { getFrontpage } from '../backend/frontpage';
import CommonDisplay from '../web/display/commonDisplay';
import { displayForecastsWrapperForSearch } from '../web/display/displayForecastsWrappers'; import { displayForecastsWrapperForSearch } from '../web/display/displayForecastsWrappers';
import { Props } from '../web/search/anySearchPage';
import CommonDisplay from '../web/search/commonDisplay';
import Layout from './layout'; import Layout from './layout';
/* get Props */ export { getServerSideProps } from "../web/search/anySearchPage";
interface Props { const IndexPage: NextPage<Props> = ({
defaultResults: any; defaultResults,
} initialResults,
initialQueryParameters,
export const getStaticProps: GetStaticProps<Props> = async (context) => { }) => {
let frontPageForecasts = await getFrontpage();
frontPageForecasts = frontPageForecasts.map((forecast) => ({
...forecast,
item: {
...forecast.item,
timestamp: forecast.item.timestamp.toJSON(),
},
}));
return {
props: {
defaultResults: frontPageForecasts,
},
revalidate: 3600 * 6,
};
};
/* Body */
const IndexPage: NextPage<Props> = ({ defaultResults }) => {
return ( return (
<Layout page={"search"}> <Layout page={"search"}>
<CommonDisplay <CommonDisplay
defaultResults={defaultResults} defaultResults={defaultResults}
initialResults={initialResults}
initialQueryParameters={initialQueryParameters}
hasSearchbar={true} hasSearchbar={true}
hasCapture={false} hasCapture={false}
hasAdvancedOptions={true} hasAdvancedOptions={true}

View File

@ -1,12 +1,12 @@
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import React, { Fragment, useEffect, useState } from 'react'; import React, { Fragment, useEffect, useState } from 'react';
import ButtonsForStars from '../display/buttonsForStars';
import Form from '../display/form';
import MultiSelectPlatform from '../display/multiSelectPlatforms';
import { SliderElement } from '../display/slider';
import { platformsWithLabels, PlatformWithLabel } from '../platforms'; import { platformsWithLabels, PlatformWithLabel } from '../platforms';
import searchAccordingToQueryData from '../worker/searchAccordingToQueryData'; import searchAccordingToQueryData from '../worker/searchAccordingToQueryData';
import ButtonsForStars from './buttonsForStars';
import Form from './form';
import MultiSelectPlatform from './multiSelectPlatforms';
import { SliderElement } from './slider';
interface QueryParametersWithoutNum { interface QueryParametersWithoutNum {
query: string; query: string;
@ -21,6 +21,8 @@ export interface QueryParameters extends QueryParametersWithoutNum {
interface Props { interface Props {
defaultResults: any; defaultResults: any;
initialResults: any;
initialQueryParameters: QueryParameters;
hasSearchbar: boolean; hasSearchbar: boolean;
hasCapture: boolean; hasCapture: boolean;
hasAdvancedOptions: boolean; hasAdvancedOptions: boolean;
@ -34,17 +36,19 @@ interface Props {
}) => React.ReactNode; }) => React.ReactNode;
} }
const defaultQueryParameters: QueryParametersWithoutNum = { export const defaultQueryParameters: QueryParametersWithoutNum = {
query: "", query: "",
starsThreshold: 2, starsThreshold: 2,
forecastsThreshold: 0, forecastsThreshold: 0,
forecastingPlatforms: platformsWithLabels, // weird key value format, forecastingPlatforms: platformsWithLabels, // weird key value format,
}; };
const defaultNumDisplay = 21; export const defaultNumDisplay = 21;
/* Body */ /* Body */
const CommonDisplay: React.FC<Props> = ({ const CommonDisplay: React.FC<Props> = ({
defaultResults, defaultResults,
initialResults,
initialQueryParameters,
hasSearchbar, hasSearchbar,
hasCapture, hasCapture,
hasAdvancedOptions, hasAdvancedOptions,
@ -56,36 +60,21 @@ const CommonDisplay: React.FC<Props> = ({
/* States */ /* States */
const [queryParameters, setQueryParameters] = const [queryParameters, setQueryParameters] =
useState<QueryParametersWithoutNum>(defaultQueryParameters); useState<QueryParametersWithoutNum>(initialQueryParameters);
const [numDisplay, setNumDisplay] = useState(0); const [numDisplay, setNumDisplay] = useState(
initialQueryParameters.numDisplay ?? defaultNumDisplay
const [ready, setReady] = useState(false); );
// used to distinguish numDisplay updates which force search and don't force search, see effects below // used to distinguish numDisplay updates which force search and don't force search, see effects below
const [forceSearch, setForceSearch] = useState(0); const [forceSearch, setForceSearch] = useState(0);
const [results, setResults] = useState([]); const [results, setResults] = useState(initialResults);
const [advancedOptions, showAdvancedOptions] = useState(false); const [advancedOptions, showAdvancedOptions] = useState(false);
const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] = const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] =
useState(0); useState(0);
const [showIdToggle, setShowIdToggle] = useState(false); const [showIdToggle, setShowIdToggle] = useState(false);
useEffect(() => {
if (!router.isReady) return;
setQueryParameters({
...defaultQueryParameters,
...router.query,
});
setNumDisplay(
typeof router.query.numDisplay === "string"
? parseInt(router.query.numDisplay)
: defaultNumDisplay
);
setReady(true);
}, [router.isReady]);
/* Functions which I want to have access to the Home namespace */ /* Functions which I want to have access to the Home namespace */
// I don't want to create an "defaultResults" object for each search. // I don't want to create an "defaultResults" object for each search.
async function executeSearchOrAnswerWithDefaultResults() { async function executeSearchOrAnswerWithDefaultResults() {
@ -173,7 +162,6 @@ const CommonDisplay: React.FC<Props> = ({
useEffect(updateRoute, [numDisplay]); useEffect(updateRoute, [numDisplay]);
useEffect(() => { useEffect(() => {
if (!ready) return;
setResults([]); setResults([]);
let newTimeoutId = setTimeout(() => { let newTimeoutId = setTimeout(() => {
updateRoute(); updateRoute();
@ -184,7 +172,7 @@ const CommonDisplay: React.FC<Props> = ({
return () => { return () => {
clearTimeout(newTimeoutId); clearTimeout(newTimeoutId);
}; };
}, [ready, queryParameters, forceSearch]); }, [queryParameters, forceSearch]);
/* State controllers */ /* State controllers */

View File

@ -0,0 +1,42 @@
import { GetServerSideProps } from 'next';
import { getFrontpage } from '../../backend/frontpage';
import searchAccordingToQueryData from '../worker/searchAccordingToQueryData';
import { defaultNumDisplay, defaultQueryParameters, QueryParameters } from './commonDisplay';
/* Common code for / and /capture */
export interface Props {
defaultResults: any;
initialResults: any;
initialQueryParameters: QueryParameters;
}
export const getServerSideProps: GetServerSideProps<Props> = async (
context
) => {
let urlQuery = context.query;
let initialQueryParameters: QueryParameters = {
...defaultQueryParameters,
numDisplay: defaultNumDisplay,
...urlQuery, // FIXME - parse numerical fields
};
let defaultResults = await getFrontpage();
const initialResults =
!!initialQueryParameters &&
initialQueryParameters.query != "" &&
initialQueryParameters.query != undefined
? await searchAccordingToQueryData(initialQueryParameters)
: defaultResults;
return {
props: {
initialQueryParameters,
initialResults,
defaultResults,
},
};
};