quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: back to frontpage-in-db approach

Browse Source
This commit is contained in:
Vyacheslav Matyukhin 2022-03-26 02:51:11 +03:00
parent 80ee4c4055
commit 42c0f0967b
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
8 changed files with 162 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -17,7 +17,7 @@
},
"homepage": "https://github.com/QURIresearch/metaforecasts#readme",
"scripts": {
"cli": "ts-node src/backend/index.js",
"cli": "ts-node src/backend/index.ts",
"reload": "heroku run:detached node src/backend/utils/doEverythingForScheduler.js",
"setCookies": "./src/backend/utils/setCookies.sh",
"next-dev": "next dev",

23
src/backend/database/pg-wrapper.js
View File

@ -302,11 +302,34 @@ export async function pgInitializeHistories() {
}
}
async function pgInitializeFrontpage() {
let YOLO = false;
if (YOLO) {
await runPgCommand({
command: dropTable("latest", "frontpage"),
pool: readWritePool,
});
await runPgCommand({
command: `CREATE TABLE latest.frontpage (
id serial primary key,
frontpage_full jsonb,
frontpage_sliced jsonb
);`,
pool: readWritePool,
});
} else {
console.log(
"pgInitializeFrontpage: This command is dangerous, set YOLO to true in the code to invoke it"
);
}
}
export async function pgInitialize() {
await pgInitializeScaffolding();
await pgInitializeLatest();
await pgInitializeHistories();
await pgInitializeDashboards();
await pgInitializeFrontpage();
}
// Read

58
src/backend/frontpage.ts
View File

@ -2,23 +2,22 @@ import { pgRead, readWritePool } from './database/pg-wrapper';
export async function getFrontpageRaw() {
const client = await readWritePool.connect();
const res = await client.query(`
SELECT * FROM latest.combined
WHERE
(qualityindicators->>'stars')::int >= 3
AND description != ''
AND JSON_ARRAY_LENGTH(options) > 0
ORDER BY RANDOM() LIMIT 50
`);
return res.rows;
const res = await client.query(
"SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1"
);
if (!res.rows.length) return [];
console.log(res.rows[0].frontpage_sliced);
return res.rows[0].frontpage_sliced;
}
export async function getFrontpageFullRaw() {
return await pgRead({
schema: "latest",
tableName: "combined",
});
const client = await readWritePool.connect();
const res = await client.query(
"SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1"
);
if (!res.rows.length) return [];
console.log(res.rows[0]);
return res.rows[0].frontpage_full;
}
export async function getFrontpage() {
@ -36,3 +35,34 @@ export async function getFrontpage() {
return frontPageForecastsCompatibleWithFuse;
}
}
export async function rebuildFrontpage() {
const frontpageFull = await pgRead({
schema: "latest",
tableName: "combined",
});
const client = await readWritePool.connect();
const frontpageSliced = (
await client.query(`
SELECT * FROM latest.combined
WHERE
(qualityindicators->>'stars')::int >= 3
AND description != ''
AND JSON_ARRAY_LENGTH(options) > 0
ORDER BY RANDOM() LIMIT 50
`)
).rows;
const start = Date.now();
await client.query(
"INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
[JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
);
const end = Date.now();
const difference = end - start;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}

23
src/backend/index.js → src/backend/index.ts
View File

@ -1,13 +1,16 @@
/* Imports */
import "dotenv/config";
import readline from "readline";
import { pgInitialize } from "./database/pg-wrapper.js";
import { doEverything, tryCatchTryAgain } from "./flow/doEverything.js";
import { updateHistory } from "./flow/history/updateHistory.js";
import { mergeEverything } from "./flow/mergeEverything.js";
import { rebuildNetlifySiteWithNewData } from "./flow/rebuildNetliftySiteWithNewData.js";
import { platformFetchers } from "./platforms/all-platforms.js";
import { rebuildAlgoliaDatabase } from "./utils/algolia.js";
import 'dotenv/config';
import readline from 'readline';
import { pgInitialize } from './database/pg-wrapper.js';
import { doEverything, tryCatchTryAgain } from './flow/doEverything.js';
import { updateHistory } from './flow/history/updateHistory.js';
import { mergeEverything } from './flow/mergeEverything.js';
import { rebuildNetlifySiteWithNewData } from './flow/rebuildNetliftySiteWithNewData.js';
import { rebuildFrontpage } from './frontpage';
import { platformFetchers } from './platforms/all-platforms.js';
import { rebuildAlgoliaDatabase } from './utils/algolia.js';
/* Support functions */
let functions = [
@ -18,6 +21,7 @@ let functions = [
rebuildNetlifySiteWithNewData,
doEverything,
pgInitialize,
rebuildFrontpage,
];
let functionNames = functions.map((fun) => fun.name);
@ -34,6 +38,7 @@ let generateWhatToDoMessage = () => {
// `\n[${functionNames.length-1}]: Add to history` +
`All of the above`,
`Initialize postgres database`,
"Rebuild frontpage",
];
let otherMessagesWithNums = otherMessages.map(
(message, i) => `[${i + l}]: ${message}`

39
src/pages/capture.tsx
View File

@ -1,41 +1,24 @@
import { GetStaticProps, NextPage } from 'next';
import { NextPage } from 'next';
import React from 'react';
import { getFrontpage } from '../backend/frontpage';
import CommonDisplay from '../web/display/commonDisplay';
import { displayForecastsWrapperForCapture } from '../web/display/displayForecastsWrappers';
import { Props } from '../web/search/anySearchPage';
import CommonDisplay from '../web/search/commonDisplay';
import Layout from './layout';
/* get Props */
export { getServerSideProps } from "../web/search/anySearchPage";
interface Props {
defaultResults: any;
}
export const getStaticProps: GetStaticProps<Props> = async (context) => {
let frontPageForecasts = await getFrontpage();
frontPageForecasts = frontPageForecasts.map((forecast) => ({
...forecast,
item: {
...forecast.item,
timestamp: forecast.item.timestamp.toJSON(),
},
}));
return {
props: {
defaultResults: frontPageForecasts,
},
revalidate: 3600 * 6,
};
};
/* Body */
const CapturePage: NextPage<Props> = ({ defaultResults }) => {
const CapturePage: NextPage<Props> = ({
defaultResults,
initialResults,
initialQueryParameters,
}) => {
return (
<Layout page={"capture"}>
<CommonDisplay
defaultResults={defaultResults}
initialResults={initialResults}
initialQueryParameters={initialQueryParameters}
hasSearchbar={true}
hasCapture={true}
hasAdvancedOptions={false}

39
src/pages/index.tsx
View File

@ -1,41 +1,24 @@
import { GetStaticProps, NextPage } from 'next';
import { NextPage } from 'next';
import React from 'react';
import { getFrontpage } from '../backend/frontpage';
import CommonDisplay from '../web/display/commonDisplay';
import { displayForecastsWrapperForSearch } from '../web/display/displayForecastsWrappers';
import { Props } from '../web/search/anySearchPage';
import CommonDisplay from '../web/search/commonDisplay';
import Layout from './layout';
/* get Props */
export { getServerSideProps } from "../web/search/anySearchPage";
interface Props {
defaultResults: any;
}
export const getStaticProps: GetStaticProps<Props> = async (context) => {
let frontPageForecasts = await getFrontpage();
frontPageForecasts = frontPageForecasts.map((forecast) => ({
...forecast,
item: {
...forecast.item,
timestamp: forecast.item.timestamp.toJSON(),
},
}));
return {
props: {
defaultResults: frontPageForecasts,
},
revalidate: 3600 * 6,
};
};
/* Body */
const IndexPage: NextPage<Props> = ({ defaultResults }) => {
const IndexPage: NextPage<Props> = ({
defaultResults,
initialResults,
initialQueryParameters,
}) => {
return (
<Layout page={"search"}>
<CommonDisplay
defaultResults={defaultResults}
initialResults={initialResults}
initialQueryParameters={initialQueryParameters}
hasSearchbar={true}
hasCapture={false}
hasAdvancedOptions={true}

44
src/web/display/commonDisplay.tsx → src/web/search/CommonDisplay.tsx
View File

@ -1,12 +1,12 @@
import { useRouter } from 'next/router';
import React, { Fragment, useEffect, useState } from 'react';
import ButtonsForStars from '../display/buttonsForStars';
import Form from '../display/form';
import MultiSelectPlatform from '../display/multiSelectPlatforms';
import { SliderElement } from '../display/slider';
import { platformsWithLabels, PlatformWithLabel } from '../platforms';
import searchAccordingToQueryData from '../worker/searchAccordingToQueryData';
import ButtonsForStars from './buttonsForStars';
import Form from './form';
import MultiSelectPlatform from './multiSelectPlatforms';
import { SliderElement } from './slider';
interface QueryParametersWithoutNum {
query: string;
@ -21,6 +21,8 @@ export interface QueryParameters extends QueryParametersWithoutNum {
interface Props {
defaultResults: any;
initialResults: any;
initialQueryParameters: QueryParameters;
hasSearchbar: boolean;
hasCapture: boolean;
hasAdvancedOptions: boolean;
@ -34,17 +36,19 @@ interface Props {
}) => React.ReactNode;
}
const defaultQueryParameters: QueryParametersWithoutNum = {
export const defaultQueryParameters: QueryParametersWithoutNum = {
query: "",
starsThreshold: 2,
forecastsThreshold: 0,
forecastingPlatforms: platformsWithLabels, // weird key value format,
};
const defaultNumDisplay = 21;
export const defaultNumDisplay = 21;
/* Body */
const CommonDisplay: React.FC<Props> = ({
defaultResults,
initialResults,
initialQueryParameters,
hasSearchbar,
hasCapture,
hasAdvancedOptions,
@ -56,36 +60,21 @@ const CommonDisplay: React.FC<Props> = ({
/* States */
const [queryParameters, setQueryParameters] =
useState<QueryParametersWithoutNum>(defaultQueryParameters);
useState<QueryParametersWithoutNum>(initialQueryParameters);
const [numDisplay, setNumDisplay] = useState(0);
const [ready, setReady] = useState(false);
const [numDisplay, setNumDisplay] = useState(
initialQueryParameters.numDisplay ?? defaultNumDisplay
);
// used to distinguish numDisplay updates which force search and don't force search, see effects below
const [forceSearch, setForceSearch] = useState(0);
const [results, setResults] = useState([]);
const [results, setResults] = useState(initialResults);
const [advancedOptions, showAdvancedOptions] = useState(false);
const [whichResultToDisplayAndCapture, setWhichResultToDisplayAndCapture] =
useState(0);
const [showIdToggle, setShowIdToggle] = useState(false);
useEffect(() => {
if (!router.isReady) return;
setQueryParameters({
...defaultQueryParameters,
...router.query,
});
setNumDisplay(
typeof router.query.numDisplay === "string"
? parseInt(router.query.numDisplay)
: defaultNumDisplay
);
setReady(true);
}, [router.isReady]);
/* Functions which I want to have access to the Home namespace */
// I don't want to create an "defaultResults" object for each search.
async function executeSearchOrAnswerWithDefaultResults() {
@ -173,7 +162,6 @@ const CommonDisplay: React.FC<Props> = ({
useEffect(updateRoute, [numDisplay]);
useEffect(() => {
if (!ready) return;
setResults([]);
let newTimeoutId = setTimeout(() => {
updateRoute();
@ -184,7 +172,7 @@ const CommonDisplay: React.FC<Props> = ({
return () => {
clearTimeout(newTimeoutId);
};
}, [ready, queryParameters, forceSearch]);
}, [queryParameters, forceSearch]);
/* State controllers */

42
src/web/search/anySearchPage.tsx Normal file
View File

@ -0,0 +1,42 @@
import { GetServerSideProps } from 'next';
import { getFrontpage } from '../../backend/frontpage';
import searchAccordingToQueryData from '../worker/searchAccordingToQueryData';
import { defaultNumDisplay, defaultQueryParameters, QueryParameters } from './commonDisplay';
/* Common code for / and /capture */
export interface Props {
defaultResults: any;
initialResults: any;
initialQueryParameters: QueryParameters;
}
export const getServerSideProps: GetServerSideProps<Props> = async (
context
) => {
let urlQuery = context.query;
let initialQueryParameters: QueryParameters = {
...defaultQueryParameters,
numDisplay: defaultNumDisplay,
...urlQuery, // FIXME - parse numerical fields
};
let defaultResults = await getFrontpage();
const initialResults =
!!initialQueryParameters &&
initialQueryParameters.query != "" &&
initialQueryParameters.query != undefined
? await searchAccordingToQueryData(initialQueryParameters)
: defaultResults;
return {
props: {
initialQueryParameters,
initialResults,
defaultResults,
},
};
};