quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: faster upserts

Browse Source
This commit is contained in:
Vyacheslav Matyukhin 2022-03-29 22:48:06 +03:00
parent 6dfbe0c05e
commit 838fecd556
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
5 changed files with 229 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

243
package-lock.json generated
View File

@ -64,6 +64,7 @@
"devDependencies": { "devDependencies": {
"@netlify/plugin-nextjs": "^4.2.4", "@netlify/plugin-nextjs": "^4.2.4",
"@svgr/cli": "^6.2.1", "@svgr/cli": "^6.2.1",
"@types/pg": "^8.6.5",
"netlify-cli": "^9.13.6" "netlify-cli": "^9.13.6"
} }
}, },
@ -979,22 +980,6 @@
"integrity": "sha512-nrIgY6t17FQ9xxwH3jj0a6EOiQ/WDHUos35Hghtr+SWN/ntHIQ7UpuvSi0vaLzZVHQWaDupKI+liO5vANcDeTQ==", "integrity": "sha512-nrIgY6t17FQ9xxwH3jj0a6EOiQ/WDHUos35Hghtr+SWN/ntHIQ7UpuvSi0vaLzZVHQWaDupKI+liO5vANcDeTQ==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/@next/swc-android-arm64": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-android-arm64/-/swc-android-arm64-12.1.0.tgz",
"integrity": "sha512-/280MLdZe0W03stA69iL+v6I+J1ascrQ6FrXBlXGCsGzrfMaGr7fskMa0T5AhQIVQD4nA/46QQWxG//DYuFBcA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-darwin-arm64": { "node_modules/@next/swc-darwin-arm64": {
"version": "12.1.0", "version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-darwin-arm64/-/swc-darwin-arm64-12.1.0.tgz", "resolved": "https://registry.npmjs.org/@next%2fswc-darwin-arm64/-/swc-darwin-arm64-12.1.0.tgz",
@ -1011,150 +996,6 @@
"node": ">= 10" "node": ">= 10"
} }
}, },
"node_modules/@next/swc-darwin-x64": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-darwin-x64/-/swc-darwin-x64-12.1.0.tgz",
"integrity": "sha512-ieAz0/J0PhmbZBB8+EA/JGdhRHBogF8BWaeqR7hwveb6SYEIJaDNQy0I+ZN8gF8hLj63bEDxJAs/cEhdnTq+ug==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-linux-arm-gnueabihf": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.1.0.tgz",
"integrity": "sha512-njUd9hpl6o6A5d08dC0cKAgXKCzm5fFtgGe6i0eko8IAdtAPbtHxtpre3VeSxdZvuGFh+hb0REySQP9T1ttkog==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-linux-arm64-gnu": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.1.0.tgz",
"integrity": "sha512-OqangJLkRxVxMhDtcb7Qn1xjzFA3s50EIxY7mljbSCLybU+sByPaWAHY4px97ieOlr2y4S0xdPKkQ3BCAwyo6Q==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-linux-arm64-musl": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-arm64-musl/-/swc-linux-arm64-musl-12.1.0.tgz",
"integrity": "sha512-hB8cLSt4GdmOpcwRe2UzI5UWn6HHO/vLkr5OTuNvCJ5xGDwpPXelVkYW/0+C3g5axbDW2Tym4S+MQCkkH9QfWA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-linux-x64-gnu": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-x64-gnu/-/swc-linux-x64-gnu-12.1.0.tgz",
"integrity": "sha512-OKO4R/digvrVuweSw/uBM4nSdyzsBV5EwkUeeG4KVpkIZEe64ZwRpnFB65bC6hGwxIBnTv5NMSnJ+0K/WmG78A==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-linux-x64-musl": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-x64-musl/-/swc-linux-x64-musl-12.1.0.tgz",
"integrity": "sha512-JohhgAHZvOD3rQY7tlp7NlmvtvYHBYgY0x5ZCecUT6eCCcl9lv6iV3nfu82ErkxNk1H893fqH0FUpznZ/H3pSw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-win32-arm64-msvc": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.1.0.tgz",
"integrity": "sha512-T/3gIE6QEfKIJ4dmJk75v9hhNiYZhQYAoYm4iVo1TgcsuaKLFa+zMPh4056AHiG6n9tn2UQ1CFE8EoybEsqsSw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-win32-ia32-msvc": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.1.0.tgz",
"integrity": "sha512-iwnKgHJdqhIW19H9PRPM9j55V6RdcOo6rX+5imx832BCWzkDbyomWnlzBfr6ByUYfhohb8QuH4hSGEikpPqI0Q==",
"cpu": [
"ia32"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@next/swc-win32-x64-msvc": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-win32-x64-msvc/-/swc-win32-x64-msvc-12.1.0.tgz",
"integrity": "sha512-aBvcbMwuanDH4EMrL2TthNJy+4nP59Bimn8egqv6GHMVj0a44cU6Au4PjOhLNqEh9l+IpRGBqMTzec94UdC5xg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
}
},
"node_modules/@nodelib/fs.scandir": { "node_modules/@nodelib/fs.scandir": {
"version": "2.1.5", "version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib%2ffs.scandir/-/fs.scandir-2.1.5.tgz", "resolved": "https://registry.npmjs.org/@nodelib%2ffs.scandir/-/fs.scandir-2.1.5.tgz",
@ -1691,6 +1532,17 @@
"integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/@types/pg": {
"version": "8.6.5",
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.5.tgz",
"integrity": "sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw==",
"dev": true,
"dependencies": {
"@types/node": "*",
"pg-protocol": "*",
"pg-types": "^2.2.0"
}
},
"node_modules/@types/prop-types": { "node_modules/@types/prop-types": {
"version": "15.7.4", "version": "15.7.4",
"resolved": "https://registry.npmjs.org/@types%2fprop-types/-/prop-types-15.7.4.tgz", "resolved": "https://registry.npmjs.org/@types%2fprop-types/-/prop-types-15.7.4.tgz",
@ -35625,72 +35477,12 @@
"resolved": "https://registry.npmjs.org/@next%2fenv/-/env-12.1.0.tgz", "resolved": "https://registry.npmjs.org/@next%2fenv/-/env-12.1.0.tgz",
"integrity": "sha512-nrIgY6t17FQ9xxwH3jj0a6EOiQ/WDHUos35Hghtr+SWN/ntHIQ7UpuvSi0vaLzZVHQWaDupKI+liO5vANcDeTQ==" "integrity": "sha512-nrIgY6t17FQ9xxwH3jj0a6EOiQ/WDHUos35Hghtr+SWN/ntHIQ7UpuvSi0vaLzZVHQWaDupKI+liO5vANcDeTQ=="
}, },
"@next/swc-android-arm64": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-android-arm64/-/swc-android-arm64-12.1.0.tgz",
"integrity": "sha512-/280MLdZe0W03stA69iL+v6I+J1ascrQ6FrXBlXGCsGzrfMaGr7fskMa0T5AhQIVQD4nA/46QQWxG//DYuFBcA==",
"optional": true
},
"@next/swc-darwin-arm64": { "@next/swc-darwin-arm64": {
"version": "12.1.0", "version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-darwin-arm64/-/swc-darwin-arm64-12.1.0.tgz", "resolved": "https://registry.npmjs.org/@next%2fswc-darwin-arm64/-/swc-darwin-arm64-12.1.0.tgz",
"integrity": "sha512-R8vcXE2/iONJ1Unf5Ptqjk6LRW3bggH+8drNkkzH4FLEQkHtELhvcmJwkXcuipyQCsIakldAXhRbZmm3YN1vXg==", "integrity": "sha512-R8vcXE2/iONJ1Unf5Ptqjk6LRW3bggH+8drNkkzH4FLEQkHtELhvcmJwkXcuipyQCsIakldAXhRbZmm3YN1vXg==",
"optional": true "optional": true
}, },
"@next/swc-darwin-x64": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-darwin-x64/-/swc-darwin-x64-12.1.0.tgz",
"integrity": "sha512-ieAz0/J0PhmbZBB8+EA/JGdhRHBogF8BWaeqR7hwveb6SYEIJaDNQy0I+ZN8gF8hLj63bEDxJAs/cEhdnTq+ug==",
"optional": true
},
"@next/swc-linux-arm-gnueabihf": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.1.0.tgz",
"integrity": "sha512-njUd9hpl6o6A5d08dC0cKAgXKCzm5fFtgGe6i0eko8IAdtAPbtHxtpre3VeSxdZvuGFh+hb0REySQP9T1ttkog==",
"optional": true
},
"@next/swc-linux-arm64-gnu": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.1.0.tgz",
"integrity": "sha512-OqangJLkRxVxMhDtcb7Qn1xjzFA3s50EIxY7mljbSCLybU+sByPaWAHY4px97ieOlr2y4S0xdPKkQ3BCAwyo6Q==",
"optional": true
},
"@next/swc-linux-arm64-musl": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-arm64-musl/-/swc-linux-arm64-musl-12.1.0.tgz",
"integrity": "sha512-hB8cLSt4GdmOpcwRe2UzI5UWn6HHO/vLkr5OTuNvCJ5xGDwpPXelVkYW/0+C3g5axbDW2Tym4S+MQCkkH9QfWA==",
"optional": true
},
"@next/swc-linux-x64-gnu": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-x64-gnu/-/swc-linux-x64-gnu-12.1.0.tgz",
"integrity": "sha512-OKO4R/digvrVuweSw/uBM4nSdyzsBV5EwkUeeG4KVpkIZEe64ZwRpnFB65bC6hGwxIBnTv5NMSnJ+0K/WmG78A==",
"optional": true
},
"@next/swc-linux-x64-musl": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-linux-x64-musl/-/swc-linux-x64-musl-12.1.0.tgz",
"integrity": "sha512-JohhgAHZvOD3rQY7tlp7NlmvtvYHBYgY0x5ZCecUT6eCCcl9lv6iV3nfu82ErkxNk1H893fqH0FUpznZ/H3pSw==",
"optional": true
},
"@next/swc-win32-arm64-msvc": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.1.0.tgz",
"integrity": "sha512-T/3gIE6QEfKIJ4dmJk75v9hhNiYZhQYAoYm4iVo1TgcsuaKLFa+zMPh4056AHiG6n9tn2UQ1CFE8EoybEsqsSw==",
"optional": true
},
"@next/swc-win32-ia32-msvc": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.1.0.tgz",
"integrity": "sha512-iwnKgHJdqhIW19H9PRPM9j55V6RdcOo6rX+5imx832BCWzkDbyomWnlzBfr6ByUYfhohb8QuH4hSGEikpPqI0Q==",
"optional": true
},
"@next/swc-win32-x64-msvc": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/@next%2fswc-win32-x64-msvc/-/swc-win32-x64-msvc-12.1.0.tgz",
"integrity": "sha512-aBvcbMwuanDH4EMrL2TthNJy+4nP59Bimn8egqv6GHMVj0a44cU6Au4PjOhLNqEh9l+IpRGBqMTzec94UdC5xg==",
"optional": true
},
"@nodelib/fs.scandir": { "@nodelib/fs.scandir": {
"version": "2.1.5", "version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib%2ffs.scandir/-/fs.scandir-2.1.5.tgz", "resolved": "https://registry.npmjs.org/@nodelib%2ffs.scandir/-/fs.scandir-2.1.5.tgz",
@ -36019,6 +35811,17 @@
"resolved": "https://registry.npmjs.org/@types%2fparse-json/-/parse-json-4.0.0.tgz", "resolved": "https://registry.npmjs.org/@types%2fparse-json/-/parse-json-4.0.0.tgz",
"integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA=="
}, },
"@types/pg": {
"version": "8.6.5",
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.5.tgz",
"integrity": "sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw==",
"dev": true,
"requires": {
"@types/node": "*",
"pg-protocol": "*",
"pg-types": "^2.2.0"
}
},
"@types/prop-types": { "@types/prop-types": {
"version": "15.7.4", "version": "15.7.4",
"resolved": "https://registry.npmjs.org/@types%2fprop-types/-/prop-types-15.7.4.tgz", "resolved": "https://registry.npmjs.org/@types%2fprop-types/-/prop-types-15.7.4.tgz",

3
package.json
View File

@ -82,6 +82,7 @@
"devDependencies": { "devDependencies": {
"@netlify/plugin-nextjs": "^4.2.4", "@netlify/plugin-nextjs": "^4.2.4",
"@svgr/cli": "^6.2.1", "@svgr/cli": "^6.2.1",
"netlify-cli": "^9.13.6" "netlify-cli": "^9.13.6",
"@types/pg": "^8.6.5"
} }
} }

249
src/backend/database/pg-wrapper.ts
View File

@ -1,11 +1,10 @@
import pkg from "pg"; import { Pool, PoolClient } from "pg";
import { platforms } from "../platforms"; import { Forecast, platforms } from "../platforms";
import { hash } from "../utils/hash"; import { hash } from "../utils/hash";
import { measureTime } from "../utils/measureTime";
import { roughSizeOfObject } from "../utils/roughSize"; import { roughSizeOfObject } from "../utils/roughSize";
const { Pool } = pkg;
// Definitions // Definitions
const schemas = ["latest", "history"]; const schemas = ["latest", "history"];
const year = Number(new Date().toISOString().slice(0, 4)); const year = Number(new Date().toISOString().slice(0, 4));
@ -26,10 +25,6 @@ const tableNamesWhiteListHistory = [
...allowed_years, ...allowed_years,
...allowed_year_month_histories, ...allowed_year_month_histories,
]; ];
const tableNamesWhitelist = [
...tableNamesWhitelistLatest,
...tableNamesWhiteListHistory,
];
const createFullName = (schemaName, namesArray) => const createFullName = (schemaName, namesArray) =>
namesArray.map((name) => `${schemaName}.${name}`); namesArray.map((name) => `${schemaName}.${name}`);
const tableWhiteList = [ const tableWhiteList = [
@ -63,28 +58,33 @@ const readOnlyPool = new Pool({
}); });
// Helpers // Helpers
export const runPgCommand = async ({ command, pool }) => { export const runPgCommand = async ({
command,
pool,
}: {
command: string;
pool: Pool;
}) => {
console.log(command); console.log(command);
const client = await pool.connect(); const client = await pool.connect();
let result; let result;
try { try {
let response = await client.query(command); let response = await client.query(command);
// console.log(response);
result = { results: response ? response.rows : null }; result = { results: response ? response.rows : null };
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} finally { } finally {
client.release(); client.release();
} }
// console.log(results)
return result; return result;
}; };
// Initialize // Initialize
let dropTable = (schema, table) => `DROP TABLE IF EXISTS ${schema}.${table}`; let dropTable = (schema: string, table: string) =>
let createIndex = (schema, table) => `DROP TABLE IF EXISTS ${schema}.${table}`;
let createIndex = (schema: string, table: string) =>
`CREATE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`; `CREATE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
let createUniqueIndex = (schema, table) => let createUniqueIndex = (schema: string, table: string) =>
`CREATE UNIQUE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`; `CREATE UNIQUE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
async function pgInitializeScaffolding() { async function pgInitializeScaffolding() {
@ -97,7 +97,7 @@ async function pgInitializeScaffolding() {
await runPgCommand({ command, pool: readWritePool }); await runPgCommand({ command, pool: readWritePool });
} }
let buildGrantSelectForSchema = (schema) => let buildGrantSelectForSchema = (schema: string) =>
`GRANT SELECT ON ALL TABLES IN SCHEMA ${schema} TO public_read_only_user`; `GRANT SELECT ON ALL TABLES IN SCHEMA ${schema} TO public_read_only_user`;
for (let schema of schemas) { for (let schema of schemas) {
await runPgCommand({ await runPgCommand({
@ -106,7 +106,7 @@ async function pgInitializeScaffolding() {
}); });
} }
let alterDefaultPrivilegesForSchema = (schema) => let alterDefaultPrivilegesForSchema = (schema: string) =>
`ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT SELECT ON TABLES TO public_read_only_user`; `ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT SELECT ON TABLES TO public_read_only_user`;
for (let schema of schemas) { for (let schema of schemas) {
await runPgCommand({ await runPgCommand({
@ -144,8 +144,8 @@ async function pgInitializeScaffolding() {
} }
let buildMetaforecastTable = ( let buildMetaforecastTable = (
schema, schema: string,
table table: string
) => `CREATE TABLE ${schema}.${table} ( ) => `CREATE TABLE ${schema}.${table} (
id text, id text,
title text, title text,
@ -245,7 +245,10 @@ async function pgInitializeDashboards() {
} }
} }
let buildHistoryTable = (schema, table) => `CREATE TABLE ${schema}.${table} ( let buildHistoryTable = (
schema: string,
table: string
) => `CREATE TABLE ${schema}.${table} (
id text, id text,
title text, title text,
url text, url text,
@ -338,7 +341,15 @@ export async function pgInitialize() {
} }
// Read // Read
async function pgReadWithPool({ schema, tableName, pool }) { async function pgReadWithPool({
schema,
tableName,
pool,
}: {
schema: string;
tableName: string;
pool: Pool;
}) {
if (tableWhiteList.includes(`${schema}.${tableName}`)) { if (tableWhiteList.includes(`${schema}.${tableName}`)) {
let command = `SELECT * from ${schema}.${tableName}`; let command = `SELECT * from ${schema}.${tableName}`;
let response = await runPgCommand({ command, pool }); let response = await runPgCommand({ command, pool });
@ -351,11 +362,23 @@ async function pgReadWithPool({ schema, tableName, pool }) {
} }
} }
export async function pgRead({ schema, tableName }) { export async function pgRead({
schema,
tableName,
}: {
schema: string;
tableName: string;
}) {
return await pgReadWithPool({ schema, tableName, pool: readWritePool }); return await pgReadWithPool({ schema, tableName, pool: readWritePool });
} }
export async function pgReadWithReadCredentials({ schema, tableName }) { export async function pgReadWithReadCredentials({
schema,
tableName,
}: {
schema: string;
tableName: string;
}) {
// currently does not work. // currently does not work.
/* return await pgReadWithPool({ /* return await pgReadWithPool({
schema, schema,
@ -366,8 +389,16 @@ export async function pgReadWithReadCredentials({ schema, tableName }) {
return await pgReadWithPool({ schema, tableName, pool: readWritePool }); return await pgReadWithPool({ schema, tableName, pool: readWritePool });
} }
export async function pgGetByIds({ ids, schema, table }) { export async function pgGetByIds({
let idstring = `( ${ids.map((id) => `'${id}'`).join(", ")} )`; // (1, 2, 3) ids,
schema,
table,
}: {
ids: string[];
schema: string;
table: string;
}) {
let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3)
let command = `SELECT * from ${schema}.${table} where id in ${idstring}`; let command = `SELECT * from ${schema}.${table} where id in ${idstring}`;
// see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn // see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn
let response = await runPgCommand({ command, pool: readWritePool }); let response = await runPgCommand({ command, pool: readWritePool });
@ -376,9 +407,48 @@ export async function pgGetByIds({ ids, schema, table }) {
return results; return results;
} }
export async function pgInsert({ datum, schema, tableName }) { export async function pgBulkInsert({
if (tableWhiteList.includes(`${schema}.${tableName}`)) { data,
let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`; schema,
tableName,
client,
}: {
data: Forecast[];
schema: string;
tableName: string;
client: PoolClient;
}) {
if (!tableWhiteList.includes(`${schema}.${tableName}`)) {
throw Error(
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
);
}
const generateQuery = (rows: number) => {
let text = `INSERT INTO ${schema}.${tableName} VALUES`;
const cols = 10;
const parts: string[] = [];
for (let r = 0; r < rows; r++) {
const bits = [];
for (let c = 1; c <= cols; c++) {
bits.push(`$${cols * r + c}`);
}
parts.push("(" + bits.join(", ") + ")");
}
text += parts.join(", ");
return text;
};
let from = 0;
const chunkSize = 20;
while (from < data.length - 1) {
const take = Math.min(chunkSize, data.length - from);
const query = generateQuery(take);
const chunk = [];
for (let i = from; i < from + take; i++) {
const datum = data[i];
let timestamp = let timestamp =
datum.timestamp && datum.timestamp &&
!!datum.timestamp.slice && !!datum.timestamp.slice &&
@ -386,7 +456,7 @@ export async function pgInsert({ datum, schema, tableName }) {
? datum.timestamp ? datum.timestamp
: new Date().toISOString(); : new Date().toISOString();
timestamp = timestamp.slice(0, 19).replace("T", " "); timestamp = timestamp.slice(0, 19).replace("T", " ");
let values = [ const values = [
datum.id, datum.id,
datum.title, datum.title,
datum.url, datum.url,
@ -399,50 +469,15 @@ export async function pgInsert({ datum, schema, tableName }) {
JSON.stringify(datum.qualityindicators || []), JSON.stringify(datum.qualityindicators || []),
JSON.stringify(datum.extra || []), JSON.stringify(datum.extra || []),
]; ];
const client = await readWritePool.connect(); chunk.push(...values);
let result;
try {
result = await client.query(text, values);
} catch (error) {
console.log(error);
} finally {
client.release();
} }
// console.log(result)
return result; console.log(`Inserting ${from + 1}..${from + take}`);
} else { from += take;
throw Error( await client.query(query, chunk);
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
);
} }
} }
/* For reference:
pgInsert({
"id": "fantasyscotus-580",
"title": "In Wooden v. U.S., the SCOTUS will affirm the lower court's decision",
"url": "https://fantasyscotus.net/user-predictions/case/wooden-v-us/",
"platform": "FantasySCOTUS",
"description": "62.50% (75 out of 120) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of Affirm 6-3. Historically, FantasySCOTUS has chosen the correct side 50.00% of the time.",
"options": [
{
"name": "Yes",
"probability": 0.625,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 0.375,
"type": "PROBABILITY"
}
],
"timestamp": "2022-02-11T21:42:19.291Z",
"qualityindicators": {
"numforecasts": 120,
"stars": 2
}
}
)
*/
export async function pgInsertIntoDashboard({ datum, schema, tableName }) { export async function pgInsertIntoDashboard({ datum, schema, tableName }) {
if (tableWhiteList.includes(`${schema}.${tableName}`)) { if (tableWhiteList.includes(`${schema}.${tableName}`)) {
let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7)`; let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7)`;
@ -502,21 +537,20 @@ pgInsertIntoDashboard({
}); });
*/ */
export async function pgUpsert({ contents, schema, tableName }) { export async function pgUpsert({ contents, schema, tableName }) {
if (tableWhiteList.includes(`${schema}.${tableName}`)) { if (!tableWhiteList.includes(`${schema}.${tableName}`)) {
let init = Date.now(); console.log("tableWhiteList:");
if (schema == "latest") { console.log(tableWhiteList);
await runPgCommand({ throw Error(
command: dropTable(schema, tableName), `Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
pool: readWritePool, );
}); }
await runPgCommand({
command: buildMetaforecastTable(schema, tableName), await measureTime(async () => {
pool: readWritePool, const client = await readWritePool.connect();
}); try {
await runPgCommand({ await client.query("BEGIN");
command: createUniqueIndex(schema, tableName), if (schema === "latest") {
pool: readWritePool, client.query(`DELETE FROM latest.${tableName}`);
});
} }
console.log( console.log(
`Upserting ${contents.length} rows into postgres table ${schema}.${tableName}.` `Upserting ${contents.length} rows into postgres table ${schema}.${tableName}.`
@ -528,17 +562,8 @@ export async function pgUpsert({ contents, schema, tableName }) {
2 2
)} minutes` )} minutes`
); );
let i = 0;
for (let datum of contents) { await pgBulkInsert({ data: contents, schema, tableName, client });
await pgInsert({ datum, schema, tableName });
if (i < 10) {
console.log(`Inserted ${datum.id}`);
i++;
} else if (i == 10) {
console.log("...");
i++;
}
}
console.log( console.log(
`Inserted ${ `Inserted ${
contents.length contents.length
@ -546,31 +571,15 @@ export async function pgUpsert({ contents, schema, tableName }) {
contents contents
)} MB into ${schema}.${tableName}.` )} MB into ${schema}.${tableName}.`
); );
let check = await pgRead({ schema, tableName });
console.log(
`Received ${
check.length
} rows with approximate cummulative size ${roughSizeOfObject(
check
)} MB from ${schema}.${tableName}.`
);
console.log("Sample: "); console.log("Sample: ");
console.log(JSON.stringify(check.slice(0, 1), null, 4)); console.log(JSON.stringify(contents.slice(0, 1), null, 4));
await client.query("COMMIT");
let end = Date.now(); } catch (e) {
let difference = end - init; await client.query("ROLLBACK");
console.log( throw e;
`Took ${difference / 1000} seconds, or ${ } finally {
difference / (1000 * 60) client.release();
} minutes.`
);
//console.log(JSON.stringify(check.slice(0, 1), null, 4));
} else {
console.log("tableWhiteList:");
console.log(tableWhiteList);
throw Error(
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
);
} }
});
} }

35
src/backend/platforms/index.ts
View File

@ -18,13 +18,48 @@ import { xrisk } from "./xrisk";
export interface Forecast { export interface Forecast {
id: string; id: string;
// "fantasyscotus-580"
title: string; title: string;
// "In Wooden v. U.S., the SCOTUS will affirm the lower court's decision"
url: string; url: string;
// "https://fantasyscotus.net/user-predictions/case/wooden-v-us/"
description: string; description: string;
// "62.50% (75 out of 120) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of Affirm 6-3. Historically, FantasySCOTUS has chosen the correct side 50.00% of the time."
platform: string; platform: string;
// "FantasySCOTUS"
options: any[]; options: any[];
/*
[
{
"name": "Yes",
"probability": 0.625,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 0.375,
"type": "PROBABILITY"
}
]
*/
timestamp: string; timestamp: string;
// "2022-02-11T21:42:19.291Z"
stars?: number;
// 2
qualityindicators: any; qualityindicators: any;
/*
{
"numforecasts": 120,
"stars": 2
}
*/
extra?: any; extra?: any;
} }

9
src/backend/utils/measureTime.ts Normal file
View File

@ -0,0 +1,9 @@
export const measureTime = async (f: () => Promise<void>) => {
const init = Date.now();
await f();
const end = Date.now();
const difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
};