feat: rewrite frontpage code with direct db queries

This commit is contained in:
Vyacheslav Matyukhin 2022-03-24 21:08:42 +03:00
parent 6f5f53acf3
commit 796d227537
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
3 changed files with 12 additions and 109 deletions

View File

@ -302,41 +302,11 @@ export async function pgInitializeHistories() {
} }
} }
async function pgInitializeFrontpage() {
let buildFrontpage = () =>
`CREATE TABLE latest.frontpage (
id serial primary key,
frontpage_sliced jsonb,
frontpage_full jsonb
);`;
let YOLO = false;
if (YOLO) {
console.log("Create frontpage table and its index");
await runPgCommand({
command: dropTable("latest", "frontpage"),
pool: readWritePool,
});
await runPgCommand({
command: buildFrontpage(),
pool: readWritePool,
});
console.log("");
} else {
console.log(
"pgInitializeFrontpage: This command is dangerous, set YOLO to true in the code to invoke it"
);
}
}
export async function pgInitialize() { export async function pgInitialize() {
await pgInitializeScaffolding(); await pgInitializeScaffolding();
await pgInitializeLatest(); await pgInitializeLatest();
await pgInitializeHistories(); await pgInitializeHistories();
await pgInitializeDashboards(); await pgInitializeDashboards();
await pgInitializeFrontpage();
} }
// Read // Read

View File

@ -2,20 +2,23 @@ import { pgRead, readWritePool } from './database/pg-wrapper';
export async function getFrontpageRaw() { export async function getFrontpageRaw() {
const client = await readWritePool.connect(); const client = await readWritePool.connect();
const res = await client.query( const res = await client.query(`
"SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1" SELECT * FROM latest.combined
); WHERE
(qualityindicators->>'stars')::int >= 3
AND description != ''
AND JSON_ARRAY_LENGTH(options) > 0
ORDER BY RANDOM() LIMIT 50
`);
if (!res.rows.length) return []; if (!res.rows.length) return [];
return res.rows[0].frontpage_sliced; return res.rows[0].frontpage_sliced;
} }
export async function getFrontpageFullRaw() { export async function getFrontpageFullRaw() {
const client = await readWritePool.connect(); return await pgRead({
const res = await client.query( schema: "latest",
"SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1" tableName: "combined",
); });
if (!res.rows.length) return [];
return res.rows[0].frontpage_full;
} }
export async function getFrontpage() { export async function getFrontpage() {
@ -33,57 +36,3 @@ export async function getFrontpage() {
return frontPageForecastsCompatibleWithFuse; return frontPageForecastsCompatibleWithFuse;
} }
} }
// Helpers
let shuffle = (array) => {
// https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array
let currentIndex = array.length,
randomIndex;
// While there remain elements to shuffle...
while (currentIndex != 0) {
// Pick a remaining element...
randomIndex = Math.floor(Math.random() * currentIndex);
currentIndex--;
// And swap it with the current element.
[array[currentIndex], array[randomIndex]] = [
array[randomIndex],
array[currentIndex],
];
}
return array;
};
// Main
export async function downloadFrontpage() {
let init = Date.now();
const frontpageFull = await pgRead({
schema: "latest",
tableName: "combined",
});
let frontpageFiltered = frontpageFull.filter(
(forecast) =>
forecast.qualityindicators &&
forecast.qualityindicators.stars >= 3 &&
forecast.options &&
forecast.options.length > 0 &&
forecast.description != ""
);
let frontpageSliced = shuffle(frontpageFiltered).slice(0, 50);
const client = await readWritePool.connect();
await client.query(
"INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
[JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
);
let end = Date.now();
let difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}

View File

@ -1,16 +0,0 @@
import { NextApiRequest, NextApiResponse } from 'next/types';
import { downloadFrontpage } from '../../../backend/frontpage';
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
// TODO - check auth token
if (req.method !== "POST") {
res.status(400).send("Expected POST method");
return;
}
await downloadFrontpage();
res.status(200).send("Updated");
}