394 lines
11 KiB
JavaScript
394 lines
11 KiB
JavaScript
import pkg from "pg";
|
|
const { Pool } = pkg;
|
|
import { platformNames } from "../platforms/all/platformNames.js";
|
|
import { getSecret } from "../utils/getSecrets.js";
|
|
import { roughSizeOfObject } from "../utils/roughSize.js";
|
|
import { hash } from "../utils/hash.js";
|
|
|
|
// Definitions
|
|
const schemas = ["latest", "history"];
|
|
const tableNamesWhitelist = ["combined", ...platformNames];
|
|
const createFullName = (schemaName, namesArray) =>
|
|
namesArray.map((name) => `${schemaName}.${name}`);
|
|
const tableWhiteList = [
|
|
...createFullName("latest", tableNamesWhitelist),
|
|
...createFullName("history", tableNamesWhitelist),
|
|
"latest.dashboards",
|
|
];
|
|
|
|
/* Postgres database connection code */
|
|
const databaseURL =
|
|
process.env.DIGITALOCEAN_POSTGRES || getSecret("digitalocean-postgres");
|
|
// process.env.DATABASE_URL || getSecret("heroku-postgres")
|
|
const readWritePool = new Pool({
|
|
connectionString: databaseURL,
|
|
ssl: {
|
|
rejectUnauthorized: false,
|
|
},
|
|
});
|
|
|
|
const readOnlyDatabaseURL =
|
|
getSecret("digitalocean-postgres-public") ||
|
|
"postgresql://public_read_only_user:gOcihnLhqRIQUQYt@postgres-red-do-user-10290909-0.b.db.ondigitalocean.com:25060/metaforecastpg?sslmode=require";
|
|
const readOnlyPool = new Pool({
|
|
connectionString: readOnlyDatabaseURL,
|
|
ssl: {
|
|
rejectUnauthorized: false,
|
|
},
|
|
});
|
|
|
|
// Helpers
|
|
const runPgCommand = async ({ command, pool }) => {
|
|
console.log(command);
|
|
const client = await pool.connect();
|
|
let result;
|
|
try {
|
|
let response = await client.query(command);
|
|
console.log(response);
|
|
result = { results: response ? response.rows : null };
|
|
} catch (error) {
|
|
console.log(error);
|
|
} finally {
|
|
client.release();
|
|
}
|
|
// console.log(results)
|
|
return result;
|
|
};
|
|
|
|
// Initialize
|
|
let dropTable = (schema, table) => `DROP TABLE IF EXISTS ${schema}.${table}`;
|
|
let buildMetaforecastTable = (
|
|
schema,
|
|
table
|
|
) => `CREATE TABLE ${schema}.${table} (
|
|
id text,
|
|
title text,
|
|
url text,
|
|
platform text,
|
|
description text,
|
|
options json,
|
|
timestamp timestamp,
|
|
stars int,
|
|
qualityindicators json,
|
|
extra json
|
|
);`;
|
|
|
|
let buildDashboard = () =>
|
|
`CREATE TABLE latest.dashboards (
|
|
id text,
|
|
title text,
|
|
description text,
|
|
contents json,
|
|
timestamp timestamp,
|
|
creator text,
|
|
extra json
|
|
);`;
|
|
|
|
let createIndex = (schema, table) =>
|
|
`CREATE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
|
|
let createUniqueIndex = (schema, table) =>
|
|
`CREATE UNIQUE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
|
|
|
|
export async function setPermissionsForPublicUser() {
|
|
let initCommands = [
|
|
"REVOKE ALL ON DATABASE metaforecastpg FROM public_read_only_user;",
|
|
"GRANT CONNECT ON DATABASE metaforecastpg TO public_read_only_user;",
|
|
];
|
|
for (let command of initCommands) {
|
|
await runPgCommand({ command, pool: readWritePool });
|
|
}
|
|
|
|
let buildGrantSelectForSchema = (schema) =>
|
|
`GRANT SELECT ON ALL TABLES IN SCHEMA ${schema} TO public_read_only_user`;
|
|
for (let schema of schemas) {
|
|
await runPgCommand({
|
|
command: buildGrantSelectForSchema(schema),
|
|
pool: readWritePool,
|
|
});
|
|
}
|
|
|
|
let alterDefaultPrivilegesForSchema = (schema) =>
|
|
`ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT SELECT ON TABLES TO public_read_only_user`;
|
|
for (let schema of schemas) {
|
|
await runPgCommand({
|
|
command: alterDefaultPrivilegesForSchema(schema),
|
|
pool: readWritePool,
|
|
});
|
|
}
|
|
}
|
|
export async function pgInitialize() {
|
|
let YOLO = false;
|
|
if (YOLO) {
|
|
console.log("Create schemas");
|
|
for (let schema of schemas) {
|
|
await runPgCommand({
|
|
command: `CREATE SCHEMA IF NOT EXISTS ${schema}`,
|
|
pool: readWritePool,
|
|
});
|
|
}
|
|
console.log("");
|
|
|
|
console.log("Set search path");
|
|
await runPgCommand({
|
|
command: `SET search_path TO ${schemas.join(",")},public;`,
|
|
pool: readWritePool,
|
|
});
|
|
console.log("");
|
|
|
|
console.log("Set public user permissions");
|
|
await setPermissionsForPublicUser();
|
|
console.log("");
|
|
|
|
console.log("Create tables & their indexes");
|
|
for (let schema of schemas) {
|
|
for (let table of tableNamesWhitelist) {
|
|
await runPgCommand({
|
|
command: dropTable(schema, table),
|
|
pool: readWritePool,
|
|
});
|
|
await runPgCommand({
|
|
command: buildMetaforecastTable(schema, table),
|
|
pool: readWritePool,
|
|
});
|
|
if (schema == "history") {
|
|
await runPgCommand({
|
|
command: createIndex(schema, table),
|
|
pool: readWritePool,
|
|
});
|
|
} else {
|
|
await runPgCommand({
|
|
command: createUniqueIndex(schema, table),
|
|
pool: readWritePool,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
console.log(
|
|
"This command is dangerous, set YOLO to true in the code to invoke it"
|
|
);
|
|
console.log("Create dashboard table and its index");
|
|
|
|
await runPgCommand({
|
|
command: buildDashboard(),
|
|
pool: readWritePool,
|
|
});
|
|
|
|
await runPgCommand({
|
|
command: createUniqueIndex("latest", "dashboards"),
|
|
pool: readWritePool,
|
|
});
|
|
console.log("");
|
|
}
|
|
}
|
|
// pgInitialize()
|
|
|
|
// Read
|
|
async function pgReadWithPool({ schema, tableName, pool }) {
|
|
if (tableWhiteList.includes(`${schema}.${tableName}`)) {
|
|
let command = `SELECT * from ${schema}.${tableName}`;
|
|
let response = await runPgCommand({ command, pool });
|
|
let results = response.results;
|
|
return results;
|
|
} else {
|
|
throw Error(
|
|
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
|
);
|
|
}
|
|
}
|
|
|
|
export async function pgRead({ schema, tableName }) {
|
|
return await pgReadWithPool({ schema, tableName, pool: readWritePool });
|
|
}
|
|
|
|
export async function pgReadWithReadCredentials({ schema, tableName }) {
|
|
// currently does not work.
|
|
/* return await pgReadWithPool({
|
|
schema,
|
|
tableName,
|
|
pool: readOnlyPool,
|
|
});
|
|
*/
|
|
return await pgReadWithPool({ schema, tableName, pool: readWritePool });
|
|
}
|
|
|
|
export async function pgGetByIds({ ids, schema, table }) {
|
|
let idstring = `( ${ids.map((id) => `'${id}'`).join(", ")} )`; // (1, 2, 3)
|
|
let command = `SELECT * from ${schema}.${table} where id in ${idstring}`;
|
|
// see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn
|
|
let response = await runPgCommand({ command, pool: readWritePool });
|
|
let results = response.results;
|
|
console.log(results);
|
|
return results;
|
|
}
|
|
|
|
export async function pgInsert({ datum, schema, tableName }) {
|
|
if (tableWhiteList.includes(`${schema}.${tableName}`)) {
|
|
let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`;
|
|
let timestamp = datum.timestamp || new Date().toISOString();
|
|
timestamp = timestamp.slice(0, 19).replace("T", " ");
|
|
let values = [
|
|
datum.id,
|
|
datum.title,
|
|
datum.url,
|
|
datum.platform,
|
|
datum.description || "",
|
|
JSON.stringify(datum.options || []),
|
|
timestamp, // fix
|
|
datum.stars ||
|
|
(datum.qualityindicators ? datum.qualityindicators.stars : 2),
|
|
JSON.stringify(datum.qualityindicators || []),
|
|
JSON.stringify(datum.extra || []),
|
|
];
|
|
const client = await readWritePool.connect();
|
|
let result;
|
|
try {
|
|
result = await client.query(text, values);
|
|
} catch (error) {
|
|
console.log(error);
|
|
} finally {
|
|
client.release();
|
|
}
|
|
// console.log(result)
|
|
return result;
|
|
} else {
|
|
throw Error(
|
|
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
|
);
|
|
}
|
|
}
|
|
/* For reference:
|
|
pgInsert({
|
|
"id": "fantasyscotus-580",
|
|
"title": "In Wooden v. U.S., the SCOTUS will affirm the lower court's decision",
|
|
"url": "https://fantasyscotus.net/user-predictions/case/wooden-v-us/",
|
|
"platform": "FantasySCOTUS",
|
|
"description": "62.50% (75 out of 120) of FantasySCOTUS players predict that the lower court's decision will be affirmed. FantasySCOTUS overall predicts an outcome of Affirm 6-3. Historically, FantasySCOTUS has chosen the correct side 50.00% of the time.",
|
|
"options": [
|
|
{
|
|
"name": "Yes",
|
|
"probability": 0.625,
|
|
"type": "PROBABILITY"
|
|
},
|
|
{
|
|
"name": "No",
|
|
"probability": 0.375,
|
|
"type": "PROBABILITY"
|
|
}
|
|
],
|
|
"timestamp": "2022-02-11T21:42:19.291Z",
|
|
"qualityindicators": {
|
|
"numforecasts": 120,
|
|
"stars": 2
|
|
}
|
|
}
|
|
)
|
|
*/
|
|
export async function pgInsertIntoDashboard({ datum, schema, tableName }) {
|
|
if (tableWhiteList.includes(`${schema}.${tableName}`)) {
|
|
let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7)`;
|
|
let timestamp = datum.timestamp || new Date().toISOString();
|
|
timestamp = timestamp.slice(0, 19).replace("T", " ");
|
|
let values = [
|
|
hash(JSON.stringify(datum.contents)),
|
|
datum.title || "",
|
|
datum.description || "",
|
|
JSON.stringify(datum.contents || []),
|
|
timestamp, // fixed
|
|
datum.creator || "",
|
|
JSON.stringify(datum.extra || []),
|
|
];
|
|
const client = await readWritePool.connect();
|
|
let result;
|
|
try {
|
|
result = await client.query(text, values);
|
|
} catch (error) {
|
|
console.log(error);
|
|
} finally {
|
|
client.release();
|
|
}
|
|
// console.log(result)
|
|
return result;
|
|
} else {
|
|
throw Error(
|
|
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
|
);
|
|
}
|
|
}
|
|
/* For reference
|
|
id text,
|
|
title text,
|
|
description text,
|
|
contents json,
|
|
timestamp timestamp,
|
|
creator text,
|
|
extra json
|
|
*/
|
|
|
|
pgInsertIntoDashboard({
|
|
datum: {
|
|
title: "Test dashboard",
|
|
description: "A test dashboard",
|
|
contents: [
|
|
"rootclaim-did-former-new-england-patriots-tight-end-aaron-hernandez-commit-suicide-19060",
|
|
"metaculus-3912",
|
|
"givewellopenphil-2021-133",
|
|
"rootclaim-what-happened-to-barry-and-honey-sherman-19972",
|
|
"rootclaim-what-caused-the-disappearance-of-malaysia-airlines-flight-370",
|
|
],
|
|
creator: "Nuño Sempere",
|
|
},
|
|
schema: "latest",
|
|
tableName: "dashboards",
|
|
});
|
|
|
|
export async function pgUpsert({ contents, schema, tableName }) {
|
|
if (tableWhiteList.includes(`${schema}.${tableName}`)) {
|
|
if (schema == "latest") {
|
|
await runPgCommand({
|
|
command: dropTable(schema, tableName),
|
|
pool: readWritePool,
|
|
});
|
|
await runPgCommand({
|
|
command: buildMetaforecastTable(schema, tableName),
|
|
pool: readWritePool,
|
|
});
|
|
await runPgCommand({
|
|
command: createUniqueIndex(schema, tableName),
|
|
pool: readWritePool,
|
|
});
|
|
}
|
|
console.log(`Upserting into postgres table ${schema}.${tableName}`);
|
|
let i = 0;
|
|
for (let datum of contents) {
|
|
await pgInsert({ datum, schema, tableName });
|
|
if (i < 10) {
|
|
console.log(`Inserted ${datum.id}`);
|
|
i++;
|
|
} else if (i == 10) {
|
|
console.log("...");
|
|
i++;
|
|
}
|
|
}
|
|
console.log(
|
|
`Inserted rows with approximate cummulative size ${roughSizeOfObject(
|
|
contents
|
|
)} MB into ${schema}.${tableName}.`
|
|
);
|
|
let check = await pgRead({ schema, tableName });
|
|
console.log(
|
|
`Received rows with approximate cummulative size ${roughSizeOfObject(
|
|
check
|
|
)} MB from ${schema}.${tableName}.`
|
|
);
|
|
console.log("Sample: ");
|
|
console.log(JSON.stringify(check.slice(0, 1), null, 4));
|
|
|
|
//console.log(JSON.stringify(check.slice(0, 1), null, 4));
|
|
} else {
|
|
throw Error(
|
|
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
|
);
|
|
}
|
|
}
|