Merge pull request #42 from QURIresearch/no-schemas
Should be fine. Tables in `public` schema are ready.
Fixed secretEmbed and dashboards, fixed small bugs in platforms code (I'm starting to dislike the fact that platform fetchers have to populate `platform` field and set `id` to `${platform}-${shortId}`, it seems error-prone; maybe platforms should remove an object with `id: shortId` and without `platform`, and then an external code would populate it as needed... but then we'd need a different type for "fetcher output result", and I'm still hesitant).
Steps for the next half-hour:
- [ ] move tables to new DB instance in the US (copied with pg_dump/pg_restore)
- [ ] merge code, deploy on heroku/netlify
- [ ] switch to the new db
- [ ] update algolia index
- [ ] check that everything works
- [ ] drop schemas (from new DB, not a problem even in case of significant issues since we have backups and the old DB)
This commit is contained in:
Vyacheslav Matyukhin
GitHub
commit
5ccbb7fabc
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -26,12 +26,6 @@ npm-debug.log*
|
|||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# local env files
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
|
|
@ -41,5 +35,4 @@ package-lock.json ## use yarn.lock instead
|
|||
# Local Netlify folder
|
||||
.netlify
|
||||
|
||||
/.env
|
||||
/.env.production
|
||||
/.env*
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
import { pgRead, pgReadWithReadCredentials, pgUpsert } from "./pg-wrapper";
|
||||
|
||||
const dateUpToYear = () => new Date().toISOString().slice(0, 4);
|
||||
const dateUpToMonth = () =>
|
||||
new Date().toISOString().slice(0, 7).replace("-", "_");
|
||||
|
||||
export async function databaseUpsert({ contents, group }) {
|
||||
// No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
|
||||
// (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){
|
||||
switch (group) {
|
||||
case "combined":
|
||||
await pgUpsert({ contents, schema: "latest", tableName: "combined" });
|
||||
break;
|
||||
case "history":
|
||||
await pgUpsert({
|
||||
contents,
|
||||
schema: "history",
|
||||
tableName: `h${dateUpToYear()}`,
|
||||
});
|
||||
await pgUpsert({
|
||||
contents,
|
||||
schema: "history",
|
||||
tableName: `h${dateUpToMonth()}`,
|
||||
});
|
||||
break;
|
||||
default:
|
||||
await pgUpsert({ contents, schema: "latest", tableName: group });
|
||||
}
|
||||
}
|
||||
|
||||
const readWithReader = async (
|
||||
group: string,
|
||||
reader: (opts: { schema: string; tableName: string }) => Promise<any>
|
||||
) => {
|
||||
const schema = group === "history" ? "history" : "latest";
|
||||
const tableName = group === "history" ? `h${dateUpToMonth()}` : group;
|
||||
const response = await reader({
|
||||
schema,
|
||||
tableName,
|
||||
});
|
||||
|
||||
console.log("Postgres: ");
|
||||
console.log(response.slice(0, 2));
|
||||
console.log("");
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
export async function databaseRead({ group }) {
|
||||
return await readWithReader(group, pgRead);
|
||||
}
|
||||
|
||||
export async function databaseReadWithReadCredentials({ group }) {
|
||||
return await readWithReader(group, pgReadWithReadCredentials);
|
||||
}
|
||||
|
|
@ -1,37 +1,13 @@
|
|||
import { Pool, PoolClient } from "pg";
|
||||
|
||||
import { Forecast, platforms } from "../platforms";
|
||||
import { Forecast } from "../platforms";
|
||||
import { hash } from "../utils/hash";
|
||||
import { measureTime } from "../utils/measureTime";
|
||||
import { roughSizeOfObject } from "../utils/roughSize";
|
||||
|
||||
// Definitions
|
||||
const schemas = ["latest", "history"];
|
||||
const year = Number(new Date().toISOString().slice(0, 4));
|
||||
const allowed_years = [year, year + 1].map((year) => `h${year}`); // tables can't begin with number
|
||||
const allowed_months = [...Array(12).keys()]
|
||||
.map((x) => x + 1)
|
||||
.map((x) => (String(x).length == 1 ? `0${x}` : x));
|
||||
const allowed_year_month_histories = [].concat(
|
||||
...allowed_years.map((year) =>
|
||||
allowed_months.map((month) => `${year}_${month}`)
|
||||
)
|
||||
); // h2022_01
|
||||
const tableNamesWhitelistLatest = [
|
||||
"combined",
|
||||
...platforms.map((platform) => platform.name),
|
||||
];
|
||||
const tableNamesWhiteListHistory = [
|
||||
...allowed_years,
|
||||
...allowed_year_month_histories,
|
||||
];
|
||||
const createFullName = (schemaName, namesArray) =>
|
||||
namesArray.map((name) => `${schemaName}.${name}`);
|
||||
const tableWhiteList = [
|
||||
...createFullName("latest", tableNamesWhitelistLatest),
|
||||
...createFullName("history", tableNamesWhiteListHistory),
|
||||
"latest.dashboards",
|
||||
];
|
||||
const forecastTableNames = ["questions", "history"];
|
||||
|
||||
const allTableNames = [...forecastTableNames, "dashboards", "frontpage"];
|
||||
|
||||
/* Postgres database connection code */
|
||||
const databaseURL = process.env.DIGITALOCEAN_POSTGRES;
|
||||
|
|
@ -80,12 +56,11 @@ export const runPgCommand = async ({
|
|||
};
|
||||
|
||||
// Initialize
|
||||
let dropTable = (schema: string, table: string) =>
|
||||
`DROP TABLE IF EXISTS ${schema}.${table}`;
|
||||
let createIndex = (schema: string, table: string) =>
|
||||
`CREATE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
|
||||
let createUniqueIndex = (schema: string, table: string) =>
|
||||
`CREATE UNIQUE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
|
||||
let dropTable = (table: string) => `DROP TABLE IF EXISTS ${table}`;
|
||||
let createIndex = (table: string) =>
|
||||
`CREATE INDEX ${table}_id_index ON ${table} (id);`;
|
||||
let createUniqueIndex = (table: string) =>
|
||||
`CREATE UNIQUE INDEX ${table}_id_index ON ${table} (id);`;
|
||||
|
||||
async function pgInitializeScaffolding() {
|
||||
async function setPermissionsForPublicUser() {
|
||||
|
|
@ -97,42 +72,20 @@ async function pgInitializeScaffolding() {
|
|||
await runPgCommand({ command, pool: readWritePool });
|
||||
}
|
||||
|
||||
let buildGrantSelectForSchema = (schema: string) =>
|
||||
`GRANT SELECT ON ALL TABLES IN SCHEMA ${schema} TO public_read_only_user`;
|
||||
for (let schema of schemas) {
|
||||
await runPgCommand({
|
||||
command: buildGrantSelectForSchema(schema),
|
||||
command:
|
||||
"GRANT SELECT ON ALL TABLES IN SCHEMA public TO public_read_only_user",
|
||||
pool: readWritePool,
|
||||
});
|
||||
}
|
||||
|
||||
let alterDefaultPrivilegesForSchema = (schema: string) =>
|
||||
`ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT SELECT ON TABLES TO public_read_only_user`;
|
||||
for (let schema of schemas) {
|
||||
await runPgCommand({
|
||||
command: alterDefaultPrivilegesForSchema(schema),
|
||||
command:
|
||||
"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO public_read_only_user",
|
||||
pool: readWritePool,
|
||||
});
|
||||
}
|
||||
}
|
||||
let YOLO = false;
|
||||
if (YOLO) {
|
||||
console.log("Create schemas");
|
||||
for (let schema of schemas) {
|
||||
await runPgCommand({
|
||||
command: `CREATE SCHEMA IF NOT EXISTS ${schema}`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
}
|
||||
console.log("");
|
||||
|
||||
console.log("Set search path");
|
||||
await runPgCommand({
|
||||
command: `SET search_path TO ${schemas.join(",")},public;`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
|
||||
console.log("Set public user permissions");
|
||||
await setPermissionsForPublicUser();
|
||||
console.log("");
|
||||
|
|
@ -143,10 +96,7 @@ async function pgInitializeScaffolding() {
|
|||
}
|
||||
}
|
||||
|
||||
let buildMetaforecastTable = (
|
||||
schema: string,
|
||||
table: string
|
||||
) => `CREATE TABLE ${schema}.${table} (
|
||||
let buildMetaforecastTable = (table: string) => `CREATE TABLE ${table} (
|
||||
id text,
|
||||
title text,
|
||||
url text,
|
||||
|
|
@ -159,45 +109,34 @@ let buildMetaforecastTable = (
|
|||
extra json
|
||||
);`;
|
||||
|
||||
async function pgInitializeLatest() {
|
||||
async function pgInitializeQuestions() {
|
||||
let YOLO = false;
|
||||
if (YOLO) {
|
||||
console.log("Create tables & their indexes");
|
||||
let schema = "latest";
|
||||
for (let table of tableNamesWhitelistLatest) {
|
||||
const table = "questions";
|
||||
await runPgCommand({
|
||||
command: dropTable(schema, table),
|
||||
command: dropTable(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: buildMetaforecastTable(schema, table),
|
||||
command: buildMetaforecastTable(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
/*
|
||||
if (schema == "history") {
|
||||
await runPgCommand({
|
||||
command: createIndex(schema, table),
|
||||
command: createUniqueIndex(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
} else {
|
||||
*/
|
||||
await runPgCommand({
|
||||
command: createUniqueIndex(schema, table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
//}
|
||||
}
|
||||
console.log("");
|
||||
} else {
|
||||
console.log(
|
||||
"pgInitializeLatest: This command is dangerous, set YOLO to true in the code to invoke it"
|
||||
"pgInitializeQuestions: This command is dangerous, set YOLO to true in the code to invoke it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function pgInitializeDashboards() {
|
||||
let buildDashboard = () =>
|
||||
`CREATE TABLE latest.dashboards (
|
||||
`CREATE TABLE dashboards (
|
||||
id text,
|
||||
title text,
|
||||
description text,
|
||||
|
|
@ -208,23 +147,10 @@ async function pgInitializeDashboards() {
|
|||
);`;
|
||||
let YOLO = false;
|
||||
if (YOLO) {
|
||||
await runPgCommand({
|
||||
command: `CREATE SCHEMA IF NOT EXISTS history;`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
|
||||
console.log("Set search path");
|
||||
await runPgCommand({
|
||||
command: `SET search_path TO ${schemas.join(",")},public;`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
|
||||
console.log("Create dashboard table and its index");
|
||||
|
||||
await runPgCommand({
|
||||
command: dropTable("latest", "dashboards"),
|
||||
command: dropTable("dashboards"),
|
||||
pool: readWritePool,
|
||||
});
|
||||
|
||||
|
|
@ -234,7 +160,7 @@ async function pgInitializeDashboards() {
|
|||
});
|
||||
|
||||
await runPgCommand({
|
||||
command: createUniqueIndex("latest", "dashboards"),
|
||||
command: createUniqueIndex("dashboards"),
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
|
|
@ -245,10 +171,7 @@ async function pgInitializeDashboards() {
|
|||
}
|
||||
}
|
||||
|
||||
let buildHistoryTable = (
|
||||
schema: string,
|
||||
table: string
|
||||
) => `CREATE TABLE ${schema}.${table} (
|
||||
let buildHistoryTable = (table: string) => `CREATE TABLE ${table} (
|
||||
id text,
|
||||
title text,
|
||||
url text,
|
||||
|
|
@ -263,45 +186,19 @@ let buildHistoryTable = (
|
|||
export async function pgInitializeHistories() {
|
||||
let YOLO = false;
|
||||
if (YOLO) {
|
||||
console.log("Drop all previous history tables (Danger!)");
|
||||
console.log("Create history table & index");
|
||||
await runPgCommand({
|
||||
command: `DROP SCHEMA history CASCADE;`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
|
||||
console.log("Create schemas");
|
||||
for (let schema of schemas) {
|
||||
await runPgCommand({
|
||||
command: `CREATE SCHEMA IF NOT EXISTS ${schema}`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
}
|
||||
console.log("");
|
||||
|
||||
console.log("Set search path");
|
||||
await runPgCommand({
|
||||
command: `SET search_path TO ${schemas.join(",")},public;`,
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
|
||||
console.log("Create tables & their indexes");
|
||||
let schema = "history";
|
||||
for (let table of tableNamesWhiteListHistory) {
|
||||
await runPgCommand({
|
||||
command: dropTable(schema, table),
|
||||
command: dropTable("history"),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: buildHistoryTable(schema, table),
|
||||
command: buildHistoryTable("history"),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: createIndex(schema, table), // Not unique!!
|
||||
command: createIndex("history"), // Not unique!!
|
||||
pool: readWritePool,
|
||||
});
|
||||
}
|
||||
console.log("");
|
||||
} else {
|
||||
console.log(
|
||||
|
|
@ -314,11 +211,11 @@ async function pgInitializeFrontpage() {
|
|||
let YOLO = false;
|
||||
if (YOLO) {
|
||||
await runPgCommand({
|
||||
command: dropTable("latest", "frontpage"),
|
||||
command: dropTable("frontpage"),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: `CREATE TABLE latest.frontpage (
|
||||
command: `CREATE TABLE frontpage (
|
||||
id serial primary key,
|
||||
frontpage_full jsonb,
|
||||
frontpage_sliced jsonb
|
||||
|
|
@ -334,7 +231,7 @@ async function pgInitializeFrontpage() {
|
|||
|
||||
export async function pgInitialize() {
|
||||
await pgInitializeScaffolding();
|
||||
await pgInitializeLatest();
|
||||
await pgInitializeQuestions();
|
||||
await pgInitializeHistories();
|
||||
await pgInitializeDashboards();
|
||||
await pgInitializeFrontpage();
|
||||
|
|
@ -342,64 +239,50 @@ export async function pgInitialize() {
|
|||
|
||||
// Read
|
||||
async function pgReadWithPool({
|
||||
schema,
|
||||
tableName,
|
||||
pool,
|
||||
}: {
|
||||
schema: string;
|
||||
tableName: string;
|
||||
pool: Pool;
|
||||
}) {
|
||||
if (tableWhiteList.includes(`${schema}.${tableName}`)) {
|
||||
let command = `SELECT * from ${schema}.${tableName}`;
|
||||
if (!allTableNames.includes(tableName)) {
|
||||
throw Error(
|
||||
`Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
);
|
||||
}
|
||||
let command = `SELECT * from ${tableName}`;
|
||||
let response = await runPgCommand({ command, pool });
|
||||
let results = response.results;
|
||||
return results;
|
||||
} else {
|
||||
throw Error(
|
||||
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function pgRead({
|
||||
schema,
|
||||
tableName,
|
||||
}: {
|
||||
schema: string;
|
||||
tableName: string;
|
||||
}) {
|
||||
return await pgReadWithPool({ schema, tableName, pool: readWritePool });
|
||||
export async function pgRead({ tableName }: { tableName: string }) {
|
||||
return await pgReadWithPool({ tableName, pool: readWritePool });
|
||||
}
|
||||
|
||||
export async function pgReadWithReadCredentials({
|
||||
schema,
|
||||
tableName,
|
||||
}: {
|
||||
schema: string;
|
||||
tableName: string;
|
||||
}) {
|
||||
// currently does not work.
|
||||
/* return await pgReadWithPool({
|
||||
schema,
|
||||
tableName,
|
||||
pool: readOnlyPool,
|
||||
});
|
||||
*/
|
||||
return await pgReadWithPool({ schema, tableName, pool: readWritePool });
|
||||
return await pgReadWithPool({ tableName, pool: readWritePool });
|
||||
}
|
||||
|
||||
export async function pgGetByIds({
|
||||
ids,
|
||||
schema,
|
||||
table,
|
||||
}: {
|
||||
ids: string[];
|
||||
schema: string;
|
||||
table: string;
|
||||
}) {
|
||||
let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3)
|
||||
let command = `SELECT * from ${schema}.${table} where id in ${idstring}`;
|
||||
let command = `SELECT * from ${table} where id in ${idstring}`;
|
||||
// see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn
|
||||
let response = await runPgCommand({ command, pool: readWritePool });
|
||||
let results = response.results;
|
||||
|
|
@ -409,23 +292,21 @@ export async function pgGetByIds({
|
|||
|
||||
export async function pgBulkInsert({
|
||||
data,
|
||||
schema,
|
||||
tableName,
|
||||
client,
|
||||
}: {
|
||||
data: Forecast[];
|
||||
schema: string;
|
||||
tableName: string;
|
||||
client: PoolClient;
|
||||
}) {
|
||||
if (!tableWhiteList.includes(`${schema}.${tableName}`)) {
|
||||
if (!forecastTableNames.includes(tableName)) {
|
||||
throw Error(
|
||||
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
`Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
);
|
||||
}
|
||||
|
||||
const generateQuery = (rows: number) => {
|
||||
let text = `INSERT INTO ${schema}.${tableName} VALUES`;
|
||||
let text = `INSERT INTO ${tableName} VALUES`;
|
||||
const cols = 10;
|
||||
const parts: string[] = [];
|
||||
for (let r = 0; r < rows; r++) {
|
||||
|
|
@ -478,9 +359,8 @@ export async function pgBulkInsert({
|
|||
}
|
||||
}
|
||||
|
||||
export async function pgInsertIntoDashboard({ datum, schema, tableName }) {
|
||||
if (tableWhiteList.includes(`${schema}.${tableName}`)) {
|
||||
let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7)`;
|
||||
export async function pgInsertIntoDashboard({ datum }) {
|
||||
let text = `INSERT INTO dashboards VALUES($1, $2, $3, $4, $5, $6, $7)`;
|
||||
let timestamp = datum.timestamp || new Date().toISOString();
|
||||
timestamp = timestamp.slice(0, 19).replace("T", " ");
|
||||
let values = [
|
||||
|
|
@ -503,11 +383,6 @@ export async function pgInsertIntoDashboard({ datum, schema, tableName }) {
|
|||
}
|
||||
// console.log(result)
|
||||
return result;
|
||||
} else {
|
||||
throw Error(
|
||||
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
);
|
||||
}
|
||||
}
|
||||
/* For reference
|
||||
id text,
|
||||
|
|
@ -532,16 +407,21 @@ pgInsertIntoDashboard({
|
|||
],
|
||||
creator: "Nuño Sempere",
|
||||
},
|
||||
schema: "latest",
|
||||
tableName: "dashboards",
|
||||
});
|
||||
*/
|
||||
export async function pgUpsert({ contents, schema, tableName }) {
|
||||
if (!tableWhiteList.includes(`${schema}.${tableName}`)) {
|
||||
console.log("tableWhiteList:");
|
||||
console.log(tableWhiteList);
|
||||
export async function pgUpsert({
|
||||
contents,
|
||||
tableName,
|
||||
replacePlatform,
|
||||
}: {
|
||||
contents: Forecast[];
|
||||
tableName: string;
|
||||
replacePlatform?: string;
|
||||
}) {
|
||||
if (!forecastTableNames.includes(tableName)) {
|
||||
throw Error(
|
||||
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
`Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -549,27 +429,22 @@ export async function pgUpsert({ contents, schema, tableName }) {
|
|||
const client = await readWritePool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
if (schema === "latest") {
|
||||
client.query(`DELETE FROM latest.${tableName}`);
|
||||
if (replacePlatform) {
|
||||
await client.query(`DELETE FROM ${tableName} WHERE platform = $1`, [
|
||||
replacePlatform,
|
||||
]);
|
||||
}
|
||||
console.log(
|
||||
`Upserting ${contents.length} rows into postgres table ${schema}.${tableName}.`
|
||||
);
|
||||
console.log(
|
||||
`Expected to take ${Number((contents.length * 831.183) / 4422).toFixed(
|
||||
2
|
||||
)} seconds or ${Number((contents.length * 13.85305) / 4422).toFixed(
|
||||
2
|
||||
)} minutes`
|
||||
`Upserting ${contents.length} rows into postgres table ${tableName}.`
|
||||
);
|
||||
|
||||
await pgBulkInsert({ data: contents, schema, tableName, client });
|
||||
await pgBulkInsert({ data: contents, tableName, client });
|
||||
console.log(
|
||||
`Inserted ${
|
||||
contents.length
|
||||
} rows with approximate cummulative size ${roughSizeOfObject(
|
||||
contents
|
||||
)} MB into ${schema}.${tableName}.`
|
||||
)} MB into ${tableName}.`
|
||||
);
|
||||
|
||||
console.log("Sample: ");
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
import {
|
||||
databaseReadWithReadCredentials,
|
||||
databaseUpsert,
|
||||
} from "../../database/database-wrapper";
|
||||
import { pgReadWithReadCredentials, pgUpsert } from "../../database/pg-wrapper";
|
||||
|
||||
export async function updateHistory() {
|
||||
let latest = await databaseReadWithReadCredentials({ group: "combined" });
|
||||
await databaseUpsert({
|
||||
let latest = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
await pgUpsert({
|
||||
contents: latest,
|
||||
group: "history",
|
||||
tableName: "history",
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import { pgInitialize } from "../database/pg-wrapper";
|
||||
import { doEverything } from "../flow/doEverything";
|
||||
import { updateHistory } from "../flow/history/updateHistory";
|
||||
import { mergeEverything } from "../flow/mergeEverything";
|
||||
import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData";
|
||||
import { rebuildFrontpage } from "../frontpage";
|
||||
import { platforms, processPlatform } from "../platforms";
|
||||
|
|
@ -20,13 +19,6 @@ export const jobs: Job[] = [
|
|||
message: `Download predictions from ${platform.name}`,
|
||||
run: () => processPlatform(platform),
|
||||
})),
|
||||
{
|
||||
name: "merge",
|
||||
message:
|
||||
"Merge tables into one big table (and push the result to a pg database)",
|
||||
run: mergeEverything,
|
||||
separate: true,
|
||||
},
|
||||
{
|
||||
name: "algolia",
|
||||
message: 'Rebuild algolia database ("index")',
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
import { databaseRead, databaseUpsert } from "../database/database-wrapper";
|
||||
import { platforms } from "../platforms";
|
||||
|
||||
/* Merge everything */
|
||||
|
||||
export async function mergeEverythingInner() {
|
||||
let merged = [];
|
||||
for (let platform of platforms) {
|
||||
const platformName = platform.name;
|
||||
let json = await databaseRead({ group: platformName });
|
||||
console.log(`${platformName} has ${json.length} questions\n`);
|
||||
merged = merged.concat(json);
|
||||
}
|
||||
let mergedprocessed = merged.map((element) => ({
|
||||
...element,
|
||||
optionsstringforsearch: element.options
|
||||
.map((option) => option.name)
|
||||
.join(", "),
|
||||
}));
|
||||
console.log(`In total, there are ${mergedprocessed.length} questions`);
|
||||
return mergedprocessed;
|
||||
}
|
||||
|
||||
export async function mergeEverything() {
|
||||
let merged = await mergeEverythingInner();
|
||||
await databaseUpsert({ contents: merged, group: "combined" });
|
||||
console.log("Done");
|
||||
}
|
||||
|
|
@ -1,51 +1,35 @@
|
|||
import { pgRead, readWritePool } from "./database/pg-wrapper";
|
||||
import { Forecast } from "./platforms";
|
||||
|
||||
export async function getFrontpageRaw() {
|
||||
export async function getFrontpage(): Promise<Forecast[]> {
|
||||
const client = await readWritePool.connect();
|
||||
const res = await client.query(
|
||||
"SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1"
|
||||
"SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
|
||||
);
|
||||
if (!res.rows.length) return [];
|
||||
console.log(res.rows[0].frontpage_sliced);
|
||||
return res.rows[0].frontpage_sliced;
|
||||
}
|
||||
|
||||
export async function getFrontpageFullRaw() {
|
||||
export async function getFrontpageFull(): Promise<Forecast[]> {
|
||||
const client = await readWritePool.connect();
|
||||
const res = await client.query(
|
||||
"SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1"
|
||||
"SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
|
||||
);
|
||||
if (!res.rows.length) return [];
|
||||
console.log(res.rows[0]);
|
||||
return res.rows[0].frontpage_full;
|
||||
}
|
||||
|
||||
export async function getFrontpage() {
|
||||
let frontPageForecastsCompatibleWithFuse = [];
|
||||
try {
|
||||
let data = await getFrontpageRaw();
|
||||
frontPageForecastsCompatibleWithFuse = data.map((result) => ({
|
||||
item: result,
|
||||
score: 0,
|
||||
}));
|
||||
return frontPageForecastsCompatibleWithFuse;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
} finally {
|
||||
return frontPageForecastsCompatibleWithFuse;
|
||||
}
|
||||
}
|
||||
|
||||
export async function rebuildFrontpage() {
|
||||
const frontpageFull = await pgRead({
|
||||
schema: "latest",
|
||||
tableName: "combined",
|
||||
tableName: "questions",
|
||||
});
|
||||
|
||||
const client = await readWritePool.connect();
|
||||
const frontpageSliced = (
|
||||
await client.query(`
|
||||
SELECT * FROM latest.combined
|
||||
SELECT * FROM questions
|
||||
WHERE
|
||||
(qualityindicators->>'stars')::int >= 3
|
||||
AND description != ''
|
||||
|
|
@ -56,7 +40,7 @@ export async function rebuildFrontpage() {
|
|||
|
||||
const start = Date.now();
|
||||
await client.query(
|
||||
"INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
|
||||
"INSERT INTO frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
|
||||
[JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ import "dotenv/config";
|
|||
|
||||
import fs from "fs";
|
||||
|
||||
import { databaseReadWithReadCredentials } from "../database/database-wrapper";
|
||||
import { pgReadWithReadCredentials } from "../database/pg-wrapper";
|
||||
|
||||
let main = async () => {
|
||||
let json = await databaseReadWithReadCredentials({ group: "combined" });
|
||||
let json = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
let string = JSON.stringify(json, null, 2);
|
||||
let filename = "metaforecasts.json";
|
||||
fs.writeFileSync(filename, string);
|
||||
|
|
|
|||
92
src/backend/manual/noSchemaMigrate.ts
Normal file
92
src/backend/manual/noSchemaMigrate.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { readWritePool } from "../database/pg-wrapper";
|
||||
|
||||
const migrate = async () => {
|
||||
const client = await readWritePool.connect();
|
||||
|
||||
const execQuery = async (q: string) => {
|
||||
console.log(q);
|
||||
await client.query(q);
|
||||
};
|
||||
|
||||
const platformTitleToName = {
|
||||
Betfair: "betfair",
|
||||
FantasySCOTUS: "fantasyscotus",
|
||||
Foretold: "foretold",
|
||||
"GiveWell/OpenPhilanthropy": "givewellopenphil",
|
||||
"Good Judgment": "goodjudgement",
|
||||
"Good Judgment Open": "goodjudgmentopen",
|
||||
Infer: "infer",
|
||||
Kalshi: "kalshi",
|
||||
"Manifold Markets": "manifold",
|
||||
Metaculus: "metaculus",
|
||||
"Peter Wildeford": "wildeford",
|
||||
PolyMarket: "polymarket",
|
||||
PredictIt: "predictit",
|
||||
Rootclaim: "rootclaim",
|
||||
Smarkets: "smarkets",
|
||||
"X-risk estimates": "xrisk",
|
||||
};
|
||||
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const copyTable = async (from: string, to: string) => {
|
||||
await execQuery(`DROP TABLE IF EXISTS ${to}`);
|
||||
await execQuery(`CREATE TABLE ${to} (LIKE ${from} INCLUDING ALL)`);
|
||||
await execQuery(`INSERT INTO ${to} SELECT * FROM ${from}`);
|
||||
};
|
||||
|
||||
await copyTable("latest.dashboards", "dashboards");
|
||||
await copyTable("latest.combined", "questions");
|
||||
await copyTable("latest.frontpage", "frontpage");
|
||||
await copyTable("history.h2022", "history");
|
||||
|
||||
for (const [title, name] of Object.entries(platformTitleToName)) {
|
||||
console.log(`Updating ${title} -> ${name}`);
|
||||
for (const table of ["questions", "history"]) {
|
||||
await client.query(
|
||||
`UPDATE ${table} SET platform=$1 WHERE platform=$2`,
|
||||
[name, title]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Fixing GJOpen ids in questions and history");
|
||||
for (const table of ["questions", "history"]) {
|
||||
await client.query(
|
||||
`UPDATE ${table} SET id=REPLACE(id, 'goodjudmentopen-', 'goodjudgmentopen-') WHERE id LIKE 'goodjudmentopen-%'`
|
||||
);
|
||||
}
|
||||
|
||||
const fixId = (id: string) =>
|
||||
id.replace("goodjudmentopen-", "goodjudgmentopen-");
|
||||
|
||||
console.log(
|
||||
"Please rebuild frontpage manually - current version includes invalid GJOpen and xrisk ids"
|
||||
);
|
||||
|
||||
const updateDashboards = async () => {
|
||||
const res = await client.query("SELECT id, contents FROM dashboards");
|
||||
for (const row of res.rows) {
|
||||
let { id, contents } = row;
|
||||
contents = contents.map(fixId);
|
||||
await client.query(
|
||||
"UPDATE dashboards SET contents = $1 WHERE id = $2",
|
||||
[JSON.stringify(contents), id]
|
||||
);
|
||||
}
|
||||
};
|
||||
console.log("Updating dashboards");
|
||||
await updateDashboards();
|
||||
|
||||
await client.query("COMMIT");
|
||||
} catch (e) {
|
||||
await client.query("ROLLBACK");
|
||||
throw e;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
};
|
||||
|
||||
migrate();
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = "https://example.com/";
|
||||
const platformName = "example";
|
||||
const endpoint = "https://example.com/";
|
||||
|
||||
/* Support functions */
|
||||
|
||||
|
|
@ -23,7 +24,7 @@ async function fetchData() {
|
|||
|
||||
async function processPredictions(predictions) {
|
||||
let results = await predictions.map((prediction) => {
|
||||
let id = `platform-${prediction.id}`;
|
||||
let id = `${platformName}-${prediction.id}`;
|
||||
let probability = prediction.probability;
|
||||
let options = [
|
||||
{
|
||||
|
|
@ -40,12 +41,12 @@ async function processPredictions(predictions) {
|
|||
let result = {
|
||||
title: prediction.title,
|
||||
url: `https://example.com`,
|
||||
platform: "Example",
|
||||
platform: platformName,
|
||||
description: prediction.description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Example", {
|
||||
stars: calculateStars(platformName, {
|
||||
/* some: somex, factors: factors */
|
||||
}),
|
||||
other: prediction.otherx,
|
||||
|
|
@ -59,12 +60,13 @@ async function processPredictions(predictions) {
|
|||
|
||||
/* Body */
|
||||
|
||||
export async function example() {
|
||||
export const example: Platform = {
|
||||
name: platformName,
|
||||
label: "Example platform",
|
||||
color: "#ff0000",
|
||||
async fetcher() {
|
||||
let data = await fetchData();
|
||||
let results = await processPredictions(data); // somehow needed
|
||||
// console.log(results)
|
||||
// let string = JSON.stringify(results, null, 2)
|
||||
await databaseUpsert({ contents: results, group: "example" });
|
||||
console.log("Done");
|
||||
}
|
||||
//example()
|
||||
return results;
|
||||
},
|
||||
};
|
||||
|
|
@ -5,6 +5,8 @@ import https from "https";
|
|||
import { calculateStars } from "../utils/stars";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
const platformName = "betfair";
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
|
||||
|
||||
|
|
@ -82,7 +84,7 @@ async function processPredictions(data) {
|
|||
/* if(Math.floor(Math.random() * 10) % 20 ==0){
|
||||
console.log(JSON.stringify(prediction, null, 4))
|
||||
} */
|
||||
let id = `betfair-${prediction.marketId}`;
|
||||
let id = `${platformName}-${prediction.marketId}`;
|
||||
let normalizationFactor = prediction.options
|
||||
.filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
|
||||
.map((option) => option.lastPriceTraded)
|
||||
|
|
@ -121,12 +123,14 @@ async function processPredictions(data) {
|
|||
id: id,
|
||||
title: title,
|
||||
url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
|
||||
platform: "Betfair",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Betfair", { volume: prediction.totalMatched }),
|
||||
stars: calculateStars(platformName, {
|
||||
volume: prediction.totalMatched,
|
||||
}),
|
||||
volume: prediction.totalMatched,
|
||||
},
|
||||
};
|
||||
|
|
@ -136,7 +140,9 @@ async function processPredictions(data) {
|
|||
}
|
||||
|
||||
export const betfair: Platform = {
|
||||
name: "betfair",
|
||||
name: platformName,
|
||||
label: "Betfair",
|
||||
color: "#3d674a",
|
||||
async fetcher() {
|
||||
const data = await fetchPredictions();
|
||||
const results = await processPredictions(data); // somehow needed
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
const platformName = "fantasyscotus";
|
||||
|
||||
/* Definitions */
|
||||
let unixtime = new Date().getTime();
|
||||
|
|
@ -65,19 +67,19 @@ async function processData(data) {
|
|||
let historicalPercentageCorrect = data.stats.pcnt_correct;
|
||||
let historicalProbabilityCorrect =
|
||||
Number(historicalPercentageCorrect.replace("%", "")) / 100;
|
||||
let results = [];
|
||||
let results: Forecast[] = [];
|
||||
for (let event of events) {
|
||||
if (event.accuracy == "") {
|
||||
let id = `fantasyscotus-${event.id}`;
|
||||
let id = `${platformName}-${event.id}`;
|
||||
// if the thing hasn't already resolved
|
||||
let predictionData = await getPredictionsData(event.docket_url);
|
||||
let pAffirm = predictionData.proportionAffirm;
|
||||
//let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
|
||||
let eventObject = {
|
||||
let eventObject: Forecast = {
|
||||
id: id,
|
||||
title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
|
||||
url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,
|
||||
platform: "FantasySCOTUS",
|
||||
platform: platformName,
|
||||
description: `${(pAffirm * 100).toFixed(2)}% (${
|
||||
predictionData.numAffirm
|
||||
} out of ${
|
||||
|
|
@ -100,7 +102,7 @@ async function processData(data) {
|
|||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number(predictionData.numForecasts),
|
||||
stars: calculateStars("FantasySCOTUS", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
};
|
||||
results.push(eventObject);
|
||||
|
|
@ -112,7 +114,9 @@ async function processData(data) {
|
|||
|
||||
/* Body */
|
||||
export const fantasyscotus: Platform = {
|
||||
name: "fantasyscotus",
|
||||
name: platformName,
|
||||
label: "FantasySCOTUS",
|
||||
color: "#231149",
|
||||
async fetcher() {
|
||||
let rawData = await fetchData();
|
||||
let results = await processData(rawData);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
|
||||
const platformName = "foretold";
|
||||
|
||||
let graphQLendpoint = "https://api.foretold.io/graphql";
|
||||
let highQualityCommunities = [
|
||||
"0104d8e8-07e4-464b-8b32-74ef22b49f21",
|
||||
|
|
@ -54,7 +57,9 @@ async function fetchAllCommunityQuestions(communityId) {
|
|||
}
|
||||
|
||||
export const foretold: Platform = {
|
||||
name: "foretold",
|
||||
name: platformName,
|
||||
label: "Foretold",
|
||||
color: "#62520b",
|
||||
async fetcher() {
|
||||
let results = [];
|
||||
for (let community of highQualityCommunities) {
|
||||
|
|
@ -62,7 +67,7 @@ export const foretold: Platform = {
|
|||
questions = questions.map((question) => question.node);
|
||||
questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
|
||||
questions.forEach((question) => {
|
||||
let id = `foretold-${question.id}`;
|
||||
let id = `${platformName}-${question.id}`;
|
||||
let options = [];
|
||||
if (question.valueType == "PERCENTAGE") {
|
||||
let probability = question.previousAggregate.value.percentage;
|
||||
|
|
@ -83,13 +88,13 @@ export const foretold: Platform = {
|
|||
id: id,
|
||||
title: question.name,
|
||||
url: `https://www.foretold.io/c/${community}/m/${question.id}`,
|
||||
platform: "Foretold",
|
||||
platform: platformName,
|
||||
description: "",
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Math.floor(Number(question.measurementCount) / 2),
|
||||
stars: calculateStars("Foretold", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
/*liquidity: liquidity.toFixed(2),
|
||||
tradevolume: tradevolume.toFixed(2),
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
import axios from "axios";
|
||||
import fs from "fs";
|
||||
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
|
||||
const platformName = "givewellopenphil";
|
||||
|
||||
/* Support functions */
|
||||
async function fetchPage(url: string) {
|
||||
let response = await axios({
|
||||
|
|
@ -49,24 +50,26 @@ async function main1() {
|
|||
let result = {
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "GiveWell",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("GiveWell/OpenPhilanthropy", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
}; // Note: This requires some processing afterwards
|
||||
// console.log(result)
|
||||
results.push(result);
|
||||
}
|
||||
await databaseUpsert({
|
||||
contents: results,
|
||||
group: "givewell-questions-unprocessed",
|
||||
});
|
||||
// await databaseUpsert({
|
||||
// contents: results,
|
||||
// group: "givewell-questions-unprocessed",
|
||||
// });
|
||||
}
|
||||
|
||||
export const givewellopenphil: Platform = {
|
||||
name: "givewellopenphil",
|
||||
name: platformName,
|
||||
label: "GiveWell/OpenPhilanthropy",
|
||||
color: "#32407e",
|
||||
async fetcher() {
|
||||
// main1()
|
||||
return; // not necessary to refill the DB every time
|
||||
|
|
@ -76,6 +79,7 @@ export const givewellopenphil: Platform = {
|
|||
const data = JSON.parse(rawdata);
|
||||
const dataWithDate = data.map((datum: any) => ({
|
||||
...datum,
|
||||
platform: platformName,
|
||||
timestamp: "2021-02-23",
|
||||
}));
|
||||
return dataWithDate;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "goodjudgment";
|
||||
let endpoint = "https://goodjudgment.io/superforecasts/";
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
|
|
@ -15,7 +16,9 @@ String.prototype.replaceAll = function replaceAll(search, replace) {
|
|||
|
||||
/* Body */
|
||||
export const goodjudgment: Platform = {
|
||||
name: "goodjudgment",
|
||||
name: platformName,
|
||||
label: "Good Judgment",
|
||||
color: "#7d4f1b",
|
||||
async fetcher() {
|
||||
// Proxy fuckery
|
||||
let proxy;
|
||||
|
|
@ -64,7 +67,7 @@ export const goodjudgment: Platform = {
|
|||
let title = table[0]["0"].split("\t\t\t").splice(3)[0];
|
||||
if (title != undefined) {
|
||||
title = title.replaceAll("</a>", "");
|
||||
let id = `goodjudgment-${hash(title)}`;
|
||||
let id = `${platformName}-${hash(title)}`;
|
||||
let description = table
|
||||
.filter((row) => row["0"].includes("BACKGROUND:"))
|
||||
.map((row) => row["0"])
|
||||
|
|
@ -101,12 +104,12 @@ export const goodjudgment: Platform = {
|
|||
id: id,
|
||||
title: title,
|
||||
url: endpoint,
|
||||
platform: "Good Judgment",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Good Judgment", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
extra: {
|
||||
superforecastercommentary: analysis || "",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import toMarkdown from "../utils/toMarkdown";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "goodjudgmentopen";
|
||||
|
||||
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
|
||||
let annoyingPromptUrls = [
|
||||
"https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
|
||||
|
|
@ -185,12 +187,12 @@ async function goodjudgmentopen_inner(cookie) {
|
|||
}
|
||||
let questionNumRegex = new RegExp("questions/([0-9]+)");
|
||||
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
|
||||
let id = `goodjudmentopen-${questionNum}`;
|
||||
let id = `${platformName}-${questionNum}`;
|
||||
let question = {
|
||||
id: id,
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "Good Judgment Open",
|
||||
platform: platformName,
|
||||
...moreinfo,
|
||||
};
|
||||
if (j % 30 == 0 || DEBUG_MODE == "on") {
|
||||
|
|
@ -236,8 +238,10 @@ async function goodjudgmentopen_inner(cookie) {
|
|||
return results;
|
||||
}
|
||||
|
||||
export const goodjudmentopen: Platform = {
|
||||
name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen`
|
||||
export const goodjudgmentopen: Platform = {
|
||||
name: platformName,
|
||||
label: "Good Judgment Open",
|
||||
color: "#002455",
|
||||
async fetcher() {
|
||||
let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
|
||||
return await applyIfSecretExists(cookie, goodjudgmentopen_inner);
|
||||
|
|
@ -1,13 +1,13 @@
|
|||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { pgUpsert } from "../database/pg-wrapper";
|
||||
import { betfair } from "./betfair";
|
||||
import { fantasyscotus } from "./fantasyscotus";
|
||||
import { foretold } from "./foretold";
|
||||
import { givewellopenphil } from "./givewellopenphil";
|
||||
import { goodjudgment } from "./goodjudgment";
|
||||
import { goodjudmentopen } from "./goodjudmentopen";
|
||||
import { goodjudgmentopen } from "./goodjudgmentopen";
|
||||
import { infer } from "./infer";
|
||||
import { kalshi } from "./kalshi";
|
||||
import { manifoldmarkets } from "./manifoldmarkets";
|
||||
import { manifold } from "./manifold";
|
||||
import { metaculus } from "./metaculus";
|
||||
import { polymarket } from "./polymarket";
|
||||
import { predictit } from "./predictit";
|
||||
|
|
@ -67,7 +67,9 @@ export interface Forecast {
|
|||
export type PlatformFetcher = () => Promise<Forecast[] | null>;
|
||||
|
||||
export interface Platform {
|
||||
name: string;
|
||||
name: string; // short name for ids and `platform` db column, e.g. "xrisk"
|
||||
label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates"
|
||||
color: string; // used on frontend
|
||||
fetcher?: PlatformFetcher;
|
||||
}
|
||||
|
||||
|
|
@ -92,10 +94,10 @@ export const platforms: Platform[] = [
|
|||
foretold,
|
||||
givewellopenphil,
|
||||
goodjudgment,
|
||||
goodjudmentopen,
|
||||
goodjudgmentopen,
|
||||
infer,
|
||||
kalshi,
|
||||
manifoldmarkets,
|
||||
manifold,
|
||||
metaculus,
|
||||
polymarket,
|
||||
predictit,
|
||||
|
|
@ -112,7 +114,11 @@ export const processPlatform = async (platform: Platform) => {
|
|||
}
|
||||
let results = await platform.fetcher();
|
||||
if (results && results.length) {
|
||||
await databaseUpsert({ contents: results, group: platform.name });
|
||||
await pgUpsert({
|
||||
contents: results,
|
||||
tableName: "questions",
|
||||
replacePlatform: platform.name,
|
||||
});
|
||||
console.log("Done");
|
||||
} else {
|
||||
console.log(`Platform ${platform.name} didn't return any results`);
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@ import axios from "axios";
|
|||
import { Tabletojson } from "tabletojson";
|
||||
|
||||
import { applyIfSecretExists } from "../utils/getSecrets";
|
||||
import { measureTime } from "../utils/measureTime";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "infer";
|
||||
let htmlEndPoint = "https://www.infer-pub.com/questions";
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
|
|
@ -145,7 +147,7 @@ async function fetchStats(questionUrl, cookie) {
|
|||
qualityindicators: {
|
||||
numforecasts: Number(numforecasts),
|
||||
numforecasters: Number(numforecasters),
|
||||
stars: calculateStars("Infer", { numforecasts }),
|
||||
stars: calculateStars(platformName, { numforecasts }),
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -179,11 +181,12 @@ function sleep(ms) {
|
|||
|
||||
/* Body */
|
||||
|
||||
async function infer_inner(cookie) {
|
||||
async function infer_inner(cookie: string) {
|
||||
let i = 1;
|
||||
let response = await fetchPage(i, cookie);
|
||||
let results: Forecast[] = [];
|
||||
let init = Date.now();
|
||||
|
||||
await measureTime(async () => {
|
||||
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
|
||||
while (!isEnd(response) && isSignedIn(response)) {
|
||||
let htmlLines = response.split("\n");
|
||||
|
|
@ -194,18 +197,11 @@ async function infer_inner(cookie) {
|
|||
// console.log(questionHrefs)
|
||||
|
||||
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
|
||||
//console.log(response)
|
||||
console.log("questionHrefs: ");
|
||||
console.log(questionHrefs);
|
||||
}
|
||||
|
||||
//console.log("")
|
||||
//console.log("")
|
||||
//console.log(h4elements)
|
||||
|
||||
for (let questionHref of questionHrefs) {
|
||||
//console.log(h4element)
|
||||
|
||||
let elementSplit = questionHref.split('"><span>');
|
||||
let url = elementSplit[0].split('<a href="')[1];
|
||||
let title = elementSplit[1]
|
||||
|
|
@ -218,12 +214,12 @@ async function infer_inner(cookie) {
|
|||
let moreinfo = await fetchStats(url, cookie);
|
||||
let questionNumRegex = new RegExp("questions/([0-9]+)");
|
||||
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
|
||||
let id = `infer-${questionNum}`;
|
||||
let question = {
|
||||
let id = `${platformName}-${questionNum}`;
|
||||
let question: Forecast = {
|
||||
id: id,
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "Infer",
|
||||
platform: platformName,
|
||||
...moreinfo,
|
||||
};
|
||||
if (
|
||||
|
|
@ -247,7 +243,6 @@ async function infer_inner(cookie) {
|
|||
}
|
||||
|
||||
i++;
|
||||
//i=Number(i)+1
|
||||
|
||||
console.log(
|
||||
"Sleeping for ~5secs so as to not be as noticeable to the infer servers"
|
||||
|
|
@ -263,12 +258,7 @@ async function infer_inner(cookie) {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
let end = Date.now();
|
||||
let difference = end - init;
|
||||
console.log(
|
||||
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
|
||||
);
|
||||
});
|
||||
|
||||
if (results.length === 0) {
|
||||
console.log("Not updating results, as process was not signed in");
|
||||
|
|
@ -278,7 +268,9 @@ async function infer_inner(cookie) {
|
|||
}
|
||||
|
||||
export const infer: Platform = {
|
||||
name: "infer",
|
||||
name: platformName,
|
||||
label: "Infer",
|
||||
color: "#223900",
|
||||
async fetcher() {
|
||||
let cookie = process.env.INFER_COOKIE;
|
||||
return await applyIfSecretExists(cookie, infer_inner);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "kalshi";
|
||||
let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
|
||||
|
||||
async function fetchAllMarkets() {
|
||||
|
|
@ -34,17 +35,17 @@ async function processMarkets(markets) {
|
|||
type: "PROBABILITY",
|
||||
},
|
||||
];
|
||||
let id = `kalshi-${market.id}`;
|
||||
let id = `${platformName}-${market.id}`;
|
||||
let result = {
|
||||
id: id,
|
||||
title: market.title.replaceAll("*", ""),
|
||||
url: `https://kalshi.com/markets/${market.ticker_name}`,
|
||||
platform: "Kalshi",
|
||||
platform: platformName,
|
||||
description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Kalshi", {
|
||||
stars: calculateStars(platformName, {
|
||||
shares_volume: market.volume,
|
||||
interest: market.open_interest,
|
||||
}),
|
||||
|
|
@ -70,7 +71,9 @@ async function processMarkets(markets) {
|
|||
}
|
||||
|
||||
export const kalshi: Platform = {
|
||||
name: "kalshi",
|
||||
name: platformName,
|
||||
label: "Kalshi",
|
||||
color: "#615691",
|
||||
fetcher: async function () {
|
||||
let markets = await fetchAllMarkets();
|
||||
return await processMarkets(markets);
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "manifold";
|
||||
let endpoint = "https://manifold.markets/api/v0/markets";
|
||||
// See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5
|
||||
|
||||
|
|
@ -22,7 +23,7 @@ async function fetchData() {
|
|||
return response;
|
||||
}
|
||||
|
||||
function showStatistics(results) {
|
||||
function showStatistics(results: Forecast[]) {
|
||||
console.log(`Num unresolved markets: ${results.length}`);
|
||||
let sum = (arr) => arr.reduce((tally, a) => tally + a, 0);
|
||||
let num2StarsOrMore = results.filter(
|
||||
|
|
@ -43,8 +44,8 @@ function showStatistics(results) {
|
|||
}
|
||||
|
||||
async function processPredictions(predictions) {
|
||||
let results = await predictions.map((prediction) => {
|
||||
let id = `manifold-${prediction.id}`;
|
||||
let results: Forecast[] = await predictions.map((prediction) => {
|
||||
let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name
|
||||
let probability = prediction.probability;
|
||||
let options = [
|
||||
{
|
||||
|
|
@ -58,16 +59,16 @@ async function processPredictions(predictions) {
|
|||
type: "PROBABILITY",
|
||||
},
|
||||
];
|
||||
let result = {
|
||||
const result: Forecast = {
|
||||
id: id,
|
||||
title: prediction.question,
|
||||
url: prediction.url,
|
||||
platform: "Manifold Markets",
|
||||
platform: platformName,
|
||||
description: prediction.description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Manifold Markets", {
|
||||
stars: calculateStars(platformName, {
|
||||
volume7Days: prediction.volume7Days,
|
||||
volume24Hours: prediction.volume24Hours,
|
||||
pool: prediction.pool,
|
||||
|
|
@ -83,13 +84,17 @@ async function processPredictions(predictions) {
|
|||
};
|
||||
return result;
|
||||
});
|
||||
let unresolvedResults = results.filter((result) => !result.extra.isResolved);
|
||||
// console.log(unresolvedResults);
|
||||
return unresolvedResults; //resultsProcessed
|
||||
|
||||
const unresolvedResults = results.filter(
|
||||
(result) => !result.extra.isResolved
|
||||
);
|
||||
return unresolvedResults;
|
||||
}
|
||||
|
||||
export const manifoldmarkets: Platform = {
|
||||
name: "manifoldmarkets",
|
||||
export const manifold: Platform = {
|
||||
name: platformName,
|
||||
label: "Manifold Markets",
|
||||
color: "#793466",
|
||||
async fetcher() {
|
||||
let data = await fetchData();
|
||||
let results = await processPredictions(data); // somehow needed
|
||||
|
|
@ -6,6 +6,7 @@ import toMarkdown from "../utils/toMarkdown";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "metaculus";
|
||||
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
|
||||
let now = new Date().toISOString();
|
||||
let DEBUG_MODE = "off";
|
||||
|
|
@ -49,7 +50,7 @@ async function fetchMetaculusQuestions(next) {
|
|||
return data;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
|
|
@ -94,7 +95,9 @@ async function fetchMetaculusQuestionDescription(slug) {
|
|||
}
|
||||
|
||||
export const metaculus: Platform = {
|
||||
name: "metaculus",
|
||||
name: platformName,
|
||||
label: "Metaculus",
|
||||
color: "#006669",
|
||||
async fetcher() {
|
||||
// let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
|
||||
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
|
||||
|
|
@ -144,18 +147,18 @@ export const metaculus: Platform = {
|
|||
},
|
||||
];
|
||||
}
|
||||
let id = `metaculus-${result.id}`;
|
||||
let id = `${platformName}-${result.id}`;
|
||||
let interestingInfo = {
|
||||
id: id,
|
||||
title: result.title,
|
||||
url: "https://www.metaculus.com" + result.page_url,
|
||||
platform: "Metaculus",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number(result.number_of_predictions),
|
||||
stars: calculateStars("Metaculus", {
|
||||
stars: calculateStars(platformName, {
|
||||
numforecasts: result.number_of_predictions,
|
||||
}),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "polymarket";
|
||||
let graphQLendpoint =
|
||||
"https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
|
||||
let units = 10 ** 6;
|
||||
|
|
@ -63,7 +64,9 @@ async function fetchIndividualContractData(marketMakerAddress) {
|
|||
}
|
||||
|
||||
export const polymarket: Platform = {
|
||||
name: "polymarket",
|
||||
name: platformName,
|
||||
label: "PolyMarket",
|
||||
color: "#00314e",
|
||||
async fetcher() {
|
||||
let results: Forecast[] = [];
|
||||
let webpageEndpointData = await fetchAllContractInfo();
|
||||
|
|
@ -79,7 +82,7 @@ export const polymarket: Platform = {
|
|||
);
|
||||
if (moreMarketAnswer.length > 0) {
|
||||
let moreMarketInfo = moreMarketAnswer[0];
|
||||
let id = `polymarket-${addressLowerCase.slice(0, 10)}`;
|
||||
let id = `${platformName}-${addressLowerCase.slice(0, 10)}`;
|
||||
// console.log(id);
|
||||
let numforecasts = Number(moreMarketInfo.tradesQuantity);
|
||||
let tradevolume =
|
||||
|
|
@ -103,7 +106,7 @@ export const polymarket: Platform = {
|
|||
id: id,
|
||||
title: marketInfo.question,
|
||||
url: "https://polymarket.com/market/" + marketInfo.slug,
|
||||
platform: "PolyMarket",
|
||||
platform: platformName,
|
||||
description: marketInfo.description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
|
|
@ -111,7 +114,7 @@ export const polymarket: Platform = {
|
|||
numforecasts: numforecasts.toFixed(0),
|
||||
liquidity: liquidity.toFixed(2),
|
||||
tradevolume: tradevolume.toFixed(2),
|
||||
stars: calculateStars("Polymarket", {
|
||||
stars: calculateStars(platformName, {
|
||||
liquidity,
|
||||
option: options[0],
|
||||
volume: tradevolume,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import { Platform } from "./";
|
||||
|
||||
const platformName = "predictit";
|
||||
|
||||
/* Support functions */
|
||||
async function fetchmarkets() {
|
||||
let response = await axios({
|
||||
|
|
@ -39,7 +40,9 @@ function sleep(ms: number) {
|
|||
|
||||
/* Body */
|
||||
export const predictit: Platform = {
|
||||
name: "predictit",
|
||||
name: platformName,
|
||||
label: "PredictIt",
|
||||
color: "#460c00",
|
||||
async fetcher() {
|
||||
let markets = await fetchmarkets();
|
||||
let marketVolumes = await fetchmarketvolumes();
|
||||
|
|
@ -53,7 +56,7 @@ export const predictit: Platform = {
|
|||
let results = [];
|
||||
for (let market of markets) {
|
||||
// console.log(market.name)
|
||||
let id = `predictit-${market.id}`;
|
||||
let id = `${platformName}-${market.id}`;
|
||||
let isbinary = market.contracts.length == 1;
|
||||
await sleep(3000 * (1 + Math.random()));
|
||||
let descriptionraw = await fetchmarketrules(market.id);
|
||||
|
|
@ -97,12 +100,12 @@ export const predictit: Platform = {
|
|||
id: id,
|
||||
title: market["name"],
|
||||
url: market.url,
|
||||
platform: "PredictIt",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("PredictIt", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
shares_volume: shares_volume,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import { Platform } from "./";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "rootclaim";
|
||||
let jsonEndpoint =
|
||||
"https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
|
||||
|
||||
|
|
@ -24,12 +24,14 @@ async function fetchAllRootclaims() {
|
|||
}
|
||||
|
||||
export const rootclaim: Platform = {
|
||||
name: "rootclaim",
|
||||
name: platformName,
|
||||
label: "Rootclaim",
|
||||
color: "#0d1624",
|
||||
async fetcher() {
|
||||
let claims = await fetchAllRootclaims();
|
||||
let results = [];
|
||||
let results: Forecast[] = [];
|
||||
for (let claim of claims) {
|
||||
let id = `rootclaim-${claim.slug.toLowerCase()}`;
|
||||
let id = `${platformName}-${claim.slug.toLowerCase()}`;
|
||||
let options = [];
|
||||
for (let scenario of claim.scenarios) {
|
||||
//console.log(scenario)
|
||||
|
|
@ -42,17 +44,17 @@ export const rootclaim: Platform = {
|
|||
});
|
||||
}
|
||||
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
|
||||
let obj = {
|
||||
let obj: Forecast = {
|
||||
id: id,
|
||||
title: toMarkdown(claim.question).replace("\n", ""),
|
||||
url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
|
||||
platform: "Rootclaim",
|
||||
platform: platformName,
|
||||
description: toMarkdown(claim.background).replace("'", "'"),
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: 1,
|
||||
stars: calculateStars("Rootclaim", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
};
|
||||
results.push(obj);
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "smarkets";
|
||||
let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
|
||||
let VERBOSE = false;
|
||||
let empty = () => 0;
|
||||
|
||||
/* Support functions */
|
||||
|
||||
async function fetchEvents(url) {
|
||||
|
|
@ -60,7 +61,9 @@ async function fetchPrices(marketid) {
|
|||
}
|
||||
|
||||
export const smarkets: Platform = {
|
||||
name: "smarkets",
|
||||
name: platformName,
|
||||
label: "Smarkets",
|
||||
color: "#6f5b41",
|
||||
async fetcher() {
|
||||
let htmlPath =
|
||||
"?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
|
||||
|
|
@ -93,7 +96,7 @@ export const smarkets: Platform = {
|
|||
for (let market of markets) {
|
||||
VERBOSE ? console.log("================") : empty();
|
||||
VERBOSE ? console.log("Market: ", market) : empty();
|
||||
let id = `smarkets-${market.id}`;
|
||||
let id = `${platformName}-${market.id}`;
|
||||
let name = market.name;
|
||||
|
||||
let contracts = await fetchContracts(market.id);
|
||||
|
|
@ -156,16 +159,16 @@ export const smarkets: Platform = {
|
|||
name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`)
|
||||
}
|
||||
*/
|
||||
let result = {
|
||||
let result: Forecast = {
|
||||
id: id,
|
||||
title: name,
|
||||
url: "https://smarkets.com/event/" + market.event_id + market.slug,
|
||||
platform: "Smarkets",
|
||||
platform: platformName,
|
||||
description: market.description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Smarkets", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
};
|
||||
VERBOSE ? console.log(result) : empty();
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "wildeford";
|
||||
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
|
||||
const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`;
|
||||
// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0
|
||||
|
|
@ -73,7 +74,7 @@ async function processPredictions(predictions) {
|
|||
);
|
||||
let results = currentPredictions.map((prediction) => {
|
||||
let title = prediction["Prediction"].replace(" [update]", "");
|
||||
let id = `wildeford-${hash(title)}`;
|
||||
let id = `${platformName}-${hash(title)}`;
|
||||
let probability = Number(prediction["Odds"].replace("%", "")) / 100;
|
||||
let options = [
|
||||
{
|
||||
|
|
@ -91,14 +92,14 @@ async function processPredictions(predictions) {
|
|||
id: id,
|
||||
title: title,
|
||||
url: prediction["url"],
|
||||
platform: "Peter Wildeford",
|
||||
platform: platformName,
|
||||
description: prediction["Notes"] || "",
|
||||
options: options,
|
||||
timestamp: new Date(
|
||||
Date.parse(prediction["Prediction Date"] + "Z")
|
||||
).toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Peter Wildeford", null),
|
||||
stars: calculateStars(platformName, null),
|
||||
},
|
||||
};
|
||||
return result;
|
||||
|
|
@ -120,7 +121,9 @@ export async function wildeford_inner(google_api_key) {
|
|||
}
|
||||
|
||||
export const wildeford: Platform = {
|
||||
name: "wildeford",
|
||||
name: platformName,
|
||||
label: "Peter Wildeford",
|
||||
color: "#984158",
|
||||
async fetcher() {
|
||||
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
|
||||
return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);
|
||||
|
|
|
|||
|
|
@ -1,15 +1,25 @@
|
|||
import fs from "fs";
|
||||
|
||||
import { hash } from "../utils/hash";
|
||||
import { Platform } from "./";
|
||||
|
||||
const platformName = "xrisk";
|
||||
|
||||
export const xrisk: Platform = {
|
||||
name: "xrisk",
|
||||
label: "X-risk estimates",
|
||||
color: "#272600",
|
||||
async fetcher() {
|
||||
return; // not necessary to refill the DB every time
|
||||
// return; // not necessary to refill the DB every time
|
||||
let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {
|
||||
encoding: "utf-8",
|
||||
});
|
||||
const results = JSON.parse(fileRaw);
|
||||
let results = JSON.parse(fileRaw);
|
||||
results = results.map((item) => ({
|
||||
...item,
|
||||
id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
|
||||
platform: platformName,
|
||||
}));
|
||||
return results;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,34 +1,13 @@
|
|||
import algoliasearch from "algoliasearch";
|
||||
|
||||
import { databaseReadWithReadCredentials } from "../database/database-wrapper";
|
||||
import { mergeEverythingInner } from "../flow/mergeEverything";
|
||||
import { pgReadWithReadCredentials } from "../database/pg-wrapper";
|
||||
import { platforms } from "../platforms";
|
||||
|
||||
let cookie = process.env.ALGOLIA_MASTER_API_KEY;
|
||||
const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
|
||||
const client = algoliasearch(algoliaAppId, cookie);
|
||||
const index = client.initIndex("metaforecast");
|
||||
|
||||
export async function rebuildAlgoliaDatabaseTheHardWay() {
|
||||
console.log("Doing this the hard way");
|
||||
let records = await mergeEverythingInner();
|
||||
records = records.map((record, index: number) => ({
|
||||
...record,
|
||||
has_numforecasts: record.numforecasts ? true : false,
|
||||
objectID: index,
|
||||
}));
|
||||
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
|
||||
|
||||
if (index.exists()) {
|
||||
console.log("Index exists");
|
||||
index
|
||||
.replaceAllObjects(records, { safe: true })
|
||||
.catch((error) => console.log(error));
|
||||
console.log(
|
||||
`Pushed ${records.length} records. Algolia will update asynchronously`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let getoptionsstringforsearch = (record: any) => {
|
||||
let result = "";
|
||||
if (!!record.options && record.options.length > 0) {
|
||||
|
|
@ -41,12 +20,17 @@ let getoptionsstringforsearch = (record: any) => {
|
|||
};
|
||||
|
||||
export async function rebuildAlgoliaDatabaseTheEasyWay() {
|
||||
let records: any[] = await databaseReadWithReadCredentials({
|
||||
group: "combined",
|
||||
let records: any[] = await pgReadWithReadCredentials({
|
||||
tableName: "questions",
|
||||
});
|
||||
|
||||
const platformNameToLabel = Object.fromEntries(
|
||||
platforms.map((platform) => [platform.name, platform.label])
|
||||
);
|
||||
|
||||
records = records.map((record, index: number) => ({
|
||||
...record,
|
||||
platformLabel: platformNameToLabel[record.platform] || record.platform,
|
||||
has_numforecasts: record.numforecasts ? true : false,
|
||||
objectID: index,
|
||||
optionsstringforsearch: getoptionsstringforsearch(record),
|
||||
|
|
@ -62,4 +46,4 @@ export async function rebuildAlgoliaDatabaseTheEasyWay() {
|
|||
}
|
||||
}
|
||||
|
||||
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay
|
||||
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
|
||||
import { databaseReadWithReadCredentials } from "../../database/database-wrapper";
|
||||
import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
|
||||
|
||||
/* Definitions */
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ let main = async () => {
|
|||
"PredictIt",
|
||||
"Rootclaim",
|
||||
];
|
||||
let json = await databaseReadWithReadCredentials({ group: "combined" });
|
||||
let json = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
console.log(json.length);
|
||||
//let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
|
||||
//console.log(uniquePlatforms)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
|
||||
import { databaseReadWithReadCredentials } from "../../database/database-wrapper";
|
||||
import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
|
||||
|
||||
/* Definitions */
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ let shuffleArray = (array) => {
|
|||
|
||||
let main = async () => {
|
||||
let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim']
|
||||
let json = await databaseReadWithReadCredentials({ group: "combined" });
|
||||
let json = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
console.log(json.length);
|
||||
//let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
|
||||
//console.log(uniquePlatforms)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
export async function applyIfSecretExists<T>(
|
||||
cookie,
|
||||
fun: (...args: any[]) => T
|
||||
cookie: string,
|
||||
fun: (cookie: string) => T
|
||||
) {
|
||||
if (cookie) {
|
||||
return await fun(cookie);
|
||||
} else if (!cookie) {
|
||||
} else {
|
||||
console.log(
|
||||
`Cannot proceed with ${fun.name} because cookie does not exist`
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
|
||||
import { databaseReadWithReadCredentials } from "../../database/database-wrapper";
|
||||
import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
|
||||
|
||||
/* Definitions */
|
||||
let locationData = "./data/";
|
||||
|
|
@ -9,7 +9,7 @@ let locationData = "./data/";
|
|||
/* Body */
|
||||
// let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src
|
||||
async function main() {
|
||||
let data = await databaseReadWithReadCredentials({ group: "combined" }); //JSON.parse(rawdata)
|
||||
let data = await pgReadWithReadCredentials({ tableName: "questions" }); //JSON.parse(rawdata)
|
||||
let processDescription = (description) => {
|
||||
if (description == null || description == undefined || description == "") {
|
||||
return "";
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ function calculateStarsGiveWellOpenPhil(data) {
|
|||
return starsInteger;
|
||||
}
|
||||
|
||||
function calculateStarsGoodJudment(data) {
|
||||
function calculateStarsGoodJudgment(data) {
|
||||
let nuno = (data) => 4;
|
||||
let eli = (data) => 4;
|
||||
let misha = (data) => 3.5;
|
||||
|
|
@ -114,7 +114,7 @@ function calculateStarsGoodJudment(data) {
|
|||
return starsInteger;
|
||||
}
|
||||
|
||||
function calculateStarsGoodJudmentOpen(data) {
|
||||
function calculateStarsGoodJudgmentOpen(data) {
|
||||
let nuno = (data) => (data.numforecasts > 100 ? 3 : 2);
|
||||
let eli = (data) => 3;
|
||||
let misha = (data) =>
|
||||
|
|
@ -173,7 +173,7 @@ function calculateStarsLadbrokes(data) {
|
|||
return starsInteger;
|
||||
}
|
||||
|
||||
function calculateStarsManifoldMarkets(data) {
|
||||
function calculateStarsManifold(data) {
|
||||
let nuno = (data) =>
|
||||
data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100)
|
||||
? 2
|
||||
|
|
@ -268,15 +268,56 @@ function calculateStarsWilliamHill(data) {
|
|||
return starsInteger;
|
||||
}
|
||||
|
||||
export function calculateStars(platform, data) {
|
||||
export function calculateStars(platform: string, data) {
|
||||
let stars = 2;
|
||||
switch (platform) {
|
||||
case "betfair":
|
||||
stars = calculateStarsBetfair(data);
|
||||
break;
|
||||
case "infer":
|
||||
stars = calculateStarsInfer(data);
|
||||
break;
|
||||
case "foretold":
|
||||
stars = calculateStarsForetold(data);
|
||||
break;
|
||||
case "givewellopenphil":
|
||||
stars = calculateStarsGiveWellOpenPhil(data);
|
||||
break;
|
||||
case "goodjudgment":
|
||||
stars = calculateStarsGoodJudgment(data);
|
||||
break;
|
||||
case "goodjudgmentopen":
|
||||
stars = calculateStarsGoodJudgmentOpen(data);
|
||||
break;
|
||||
case "kalshi":
|
||||
stars = calculateStarsKalshi(data);
|
||||
break;
|
||||
case "manifold":
|
||||
stars = calculateStarsManifold(data);
|
||||
break;
|
||||
case "metaculus":
|
||||
stars = calculateStarsMetaculus(data);
|
||||
break;
|
||||
case "polymarket":
|
||||
stars = calculateStarsPolymarket(data);
|
||||
break;
|
||||
case "predictit":
|
||||
stars = calculateStarsPredictIt(data);
|
||||
break;
|
||||
case "rootclaim":
|
||||
stars = calculateStarsRootclaim(data);
|
||||
break;
|
||||
case "smarkets":
|
||||
stars = calculateStarsSmarkets(data);
|
||||
break;
|
||||
case "wildeford":
|
||||
stars = calculateStarsWildeford(data);
|
||||
break;
|
||||
|
||||
// deprecated
|
||||
case "AstralCodexTen":
|
||||
stars = calculateStarsAstralCodexTen(data);
|
||||
break;
|
||||
case "Betfair":
|
||||
stars = calculateStarsBetfair(data);
|
||||
break;
|
||||
case "CoupCast":
|
||||
stars = calculateStarsCoupCast(data);
|
||||
break;
|
||||
|
|
@ -289,54 +330,15 @@ export function calculateStars(platform, data) {
|
|||
case "Estimize":
|
||||
stars = calculateStarsEstimize(data);
|
||||
break;
|
||||
case "Foretold":
|
||||
stars = calculateStarsForetold(data);
|
||||
break;
|
||||
case "GiveWell/OpenPhilanthropy":
|
||||
stars = calculateStarsGiveWellOpenPhil(data);
|
||||
break;
|
||||
case "Good Judgment":
|
||||
stars = calculateStarsGoodJudment(data);
|
||||
break;
|
||||
case "Good Judgment Open":
|
||||
stars = calculateStarsGoodJudmentOpen(data);
|
||||
break;
|
||||
case "Hypermind":
|
||||
stars = calculateStarsHypermind(data);
|
||||
break;
|
||||
case "Infer":
|
||||
stars = calculateStarsInfer(data);
|
||||
break;
|
||||
case "Kalshi":
|
||||
stars = calculateStarsKalshi(data);
|
||||
break;
|
||||
case "Ladbrokes":
|
||||
stars = calculateStarsLadbrokes(data);
|
||||
break;
|
||||
case "Manifold Markets":
|
||||
stars = calculateStarsManifoldMarkets(data);
|
||||
break;
|
||||
case "Metaculus":
|
||||
stars = calculateStarsMetaculus(data);
|
||||
break;
|
||||
case "Omen":
|
||||
stars = calculateStarsOmen(data);
|
||||
break;
|
||||
case "Polymarket":
|
||||
stars = calculateStarsPolymarket(data);
|
||||
break;
|
||||
case "PredictIt":
|
||||
stars = calculateStarsPredictIt(data);
|
||||
break;
|
||||
case "Rootclaim":
|
||||
stars = calculateStarsRootclaim(data);
|
||||
break;
|
||||
case "Smarkets":
|
||||
stars = calculateStarsSmarkets(data);
|
||||
break;
|
||||
case "Peter Wildeford":
|
||||
stars = calculateStarsWildeford(data);
|
||||
break;
|
||||
case "WilliamHill":
|
||||
stars = calculateStarsWilliamHill(data);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { NextApiRequest, NextApiResponse } from "next/types";
|
||||
|
||||
import { getFrontpageFullRaw } from "../../backend/frontpage";
|
||||
import { getFrontpageFull } from "../../backend/frontpage";
|
||||
|
||||
export default async function handler(
|
||||
req: NextApiRequest,
|
||||
res: NextApiResponse
|
||||
) {
|
||||
let frontpageFull = await getFrontpageFullRaw();
|
||||
let frontpageFull = await getFrontpageFull();
|
||||
console.log(frontpageFull.map((element) => element.title).slice(0, 5));
|
||||
console.log("...");
|
||||
res.status(200).json(frontpageFull);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import crypto from "crypto";
|
||||
import { NextApiRequest, NextApiResponse } from "next/types";
|
||||
|
||||
import { pgInsertIntoDashboard } from "../../backend/database/pg-wrapper";
|
||||
import { hash } from "../../backend/utils/hash";
|
||||
|
||||
export default async function handler(
|
||||
req: NextApiRequest,
|
||||
|
|
@ -14,8 +14,6 @@ export default async function handler(
|
|||
|
||||
let body = req.body;
|
||||
console.log(body);
|
||||
const hash = (s: string) =>
|
||||
crypto.createHash("sha256").update(s).digest("hex").slice(0, 10);
|
||||
try {
|
||||
let id = hash(JSON.stringify(body.ids));
|
||||
let pgResponse = await pgInsertIntoDashboard({
|
||||
|
|
@ -27,8 +25,6 @@ export default async function handler(
|
|||
creator: body.creator || "",
|
||||
extra: [],
|
||||
},
|
||||
schema: "latest",
|
||||
tableName: "dashboards",
|
||||
});
|
||||
res.status(200).send({
|
||||
dashboardId: id,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ export default async function handler(
|
|||
console.log(id);
|
||||
let dashboardItemArray = await pgGetByIds({
|
||||
ids: [id],
|
||||
schema: "latest",
|
||||
table: "dashboards",
|
||||
});
|
||||
if (!!dashboardItemArray && dashboardItemArray.length > 0) {
|
||||
|
|
@ -24,8 +23,7 @@ export default async function handler(
|
|||
console.log(dashboardItem);
|
||||
let dashboardContents = await pgGetByIds({
|
||||
ids: dashboardItem.contents,
|
||||
schema: "latest",
|
||||
table: "combined",
|
||||
table: "questions",
|
||||
});
|
||||
res.status(200).send({
|
||||
dashboardContents,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { NextApiRequest, NextApiResponse } from "next/types";
|
||||
|
||||
import { getFrontpageRaw } from "../../backend/frontpage";
|
||||
import { getFrontpage } from "../../backend/frontpage";
|
||||
|
||||
export default async function handler(
|
||||
req: NextApiRequest,
|
||||
res: NextApiResponse
|
||||
) {
|
||||
let frontpageElements = await getFrontpageRaw();
|
||||
let frontpageElements = await getFrontpage();
|
||||
console.log(frontpageElements.map((element) => element.title).slice(0, 5));
|
||||
console.log("...");
|
||||
res.status(200).json(frontpageElements);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ export default async function handler(
|
|||
req: NextApiRequest,
|
||||
res: NextApiResponse
|
||||
) {
|
||||
let allQuestions = await pgRead({ schema: "latest", tableName: "combined" });
|
||||
let allQuestions = await pgRead({ tableName: "questions" });
|
||||
console.log(allQuestions.map((element) => element.title).slice(0, 5));
|
||||
console.log("...");
|
||||
res.status(200).json(allQuestions);
|
||||
|
|
|
|||
|
|
@ -8,17 +8,11 @@ import CommonDisplay from "../web/search/CommonDisplay";
|
|||
|
||||
export { getServerSideProps } from "../web/search/anySearchPage";
|
||||
|
||||
const CapturePage: NextPage<Props> = ({
|
||||
defaultResults,
|
||||
initialResults,
|
||||
initialQueryParameters,
|
||||
}) => {
|
||||
const CapturePage: NextPage<Props> = (props) => {
|
||||
return (
|
||||
<Layout page={"capture"}>
|
||||
<CommonDisplay
|
||||
defaultResults={defaultResults}
|
||||
initialResults={initialResults}
|
||||
initialQueryParameters={initialQueryParameters}
|
||||
{...props}
|
||||
hasSearchbar={true}
|
||||
hasCapture={true}
|
||||
hasAdvancedOptions={false}
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ export default function Home({
|
|||
let isGraubardEasterEgg = (name) => (name == "Clay Graubard" ? true : false);
|
||||
|
||||
return (
|
||||
<Layout key="index" page={"dashboard"}>
|
||||
<Layout page="dashboard">
|
||||
{/* Display forecasts */}
|
||||
<div className="mt-7 mb-7">
|
||||
<h1
|
||||
|
|
|
|||
|
|
@ -8,17 +8,11 @@ import CommonDisplay from "../web/search/CommonDisplay";
|
|||
|
||||
export { getServerSideProps } from "../web/search/anySearchPage";
|
||||
|
||||
const IndexPage: NextPage<Props> = ({
|
||||
defaultResults,
|
||||
initialResults,
|
||||
initialQueryParameters,
|
||||
}) => {
|
||||
const IndexPage: NextPage<Props> = (props) => {
|
||||
return (
|
||||
<Layout page={"search"}>
|
||||
<CommonDisplay
|
||||
defaultResults={defaultResults}
|
||||
initialResults={initialResults}
|
||||
initialQueryParameters={initialQueryParameters}
|
||||
{...props}
|
||||
hasSearchbar={true}
|
||||
hasCapture={false}
|
||||
hasAdvancedOptions={true}
|
||||
|
|
|
|||
|
|
@ -1,32 +1,33 @@
|
|||
/* Imports */
|
||||
|
||||
import { GetServerSideProps, NextPage } from "next";
|
||||
import React from "react";
|
||||
|
||||
import { displayForecast } from "../web/display/displayForecasts";
|
||||
import { platformsWithLabels } from "../web/platforms";
|
||||
import { platforms } from "../backend/platforms";
|
||||
import { DisplayForecast } from "../web/display/displayForecasts";
|
||||
import { FrontendForecast } from "../web/platforms";
|
||||
import searchAccordingToQueryData from "../web/worker/searchAccordingToQueryData";
|
||||
|
||||
/* Helper functions */
|
||||
interface Props {
|
||||
results: FrontendForecast[];
|
||||
}
|
||||
|
||||
export async function getServerSideProps(context) {
|
||||
export const getServerSideProps: GetServerSideProps<Props> = async (
|
||||
context
|
||||
) => {
|
||||
let urlQuery = context.query; // this is an object, not a string which I have to parse!!
|
||||
|
||||
let initialQueryParameters = {
|
||||
query: "",
|
||||
starsThreshold: 2,
|
||||
forecastsThreshold: 0,
|
||||
forecastingPlatforms: platformsWithLabels, // weird key value format,
|
||||
forecastingPlatforms: platforms.map((platform) => platform.name),
|
||||
...urlQuery,
|
||||
};
|
||||
|
||||
let results;
|
||||
switch (initialQueryParameters.query != "") {
|
||||
case true:
|
||||
results = await searchAccordingToQueryData(initialQueryParameters);
|
||||
break;
|
||||
default:
|
||||
results = [];
|
||||
break;
|
||||
let results: FrontendForecast[] = [];
|
||||
if (initialQueryParameters.query != "") {
|
||||
results = await searchAccordingToQueryData(initialQueryParameters, 1);
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
@ -34,35 +35,32 @@ export async function getServerSideProps(context) {
|
|||
results: results,
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
/* Body */
|
||||
export default function Home({ results }) {
|
||||
/* Final return */
|
||||
let result = results ? results[0] : null;
|
||||
const SecretEmbedPage: NextPage<Props> = ({ results }) => {
|
||||
let result = results.length ? results[0] : null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="mb-4 mt-8 flex flex-row justify-center items-center ">
|
||||
<div className="mb-4 mt-8 flex flex-row justify-center items-center">
|
||||
<div className="w-6/12 place-self-center">
|
||||
<div>
|
||||
<div id="secretEmbed">
|
||||
{result
|
||||
? displayForecast({
|
||||
...result.item,
|
||||
score: result.score,
|
||||
showTimeStamp: true,
|
||||
expandFooterToFullWidth: true,
|
||||
})
|
||||
: null}
|
||||
{result ? (
|
||||
<DisplayForecast
|
||||
forecast={result}
|
||||
showTimeStamp={true}
|
||||
expandFooterToFullWidth={true}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
<br></br>
|
||||
<div id="secretObject">
|
||||
{result ? JSON.stringify(result.item, null, 4) : null}
|
||||
{result ? JSON.stringify(result, null, 4) : null}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export default SecretEmbedPage;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import React, { useState } from "react";
|
|||
let exampleInput = `{
|
||||
"title": "Random example",
|
||||
"description": "Just a random description of a random example",
|
||||
"ids": [ "metaculus-372", "goodjudmentopen-2244", "metaculus-7550", "kalshi-09d060ee-b184-4167-b86b-d773e56b4162", "wildeford-5d1a04e1a8", "metaculus-2817" ],
|
||||
"ids": [ "metaculus-372", "goodjudgmentopen-2244", "metaculus-7550", "kalshi-09d060ee-b184-4167-b86b-d773e56b4162", "wildeford-5d1a04e1a8", "metaculus-2817" ],
|
||||
"creator": "Peter Parker"
|
||||
}`;
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import React from "react";
|
|||
import { FaRegClipboard } from "react-icons/fa";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
|
||||
import { FrontendForecast } from "../platforms";
|
||||
|
||||
/* Definitions */
|
||||
|
||||
/* Support functions */
|
||||
|
|
@ -350,9 +352,15 @@ let checkIfDisplayTimeStampAtBottom = (qualityIndicators) => {
|
|||
}
|
||||
};
|
||||
|
||||
let getCurrencySymbolIfNeeded = ({ indicator, platform }) => {
|
||||
let getCurrencySymbolIfNeeded = ({
|
||||
indicator,
|
||||
platform,
|
||||
}: {
|
||||
indicator: any;
|
||||
platform: string;
|
||||
}) => {
|
||||
let indicatorsWhichNeedCurrencySymbol = ["Volume", "Interest", "Liquidity"];
|
||||
let dollarPlatforms = ["PredictIt", "Kalshi", "PolyMarket"];
|
||||
let dollarPlatforms = ["predictit", "kalshi", "polymarket"];
|
||||
if (indicatorsWhichNeedCurrencySymbol.includes(indicator)) {
|
||||
if (dollarPlatforms.includes(platform)) {
|
||||
return "$";
|
||||
|
|
@ -461,7 +469,13 @@ let showFirstQualityIndicator = ({
|
|||
}
|
||||
};
|
||||
|
||||
let displayQualityIndicators = ({
|
||||
const displayQualityIndicators: React.FC<{
|
||||
numforecasts: number;
|
||||
timestamp: number;
|
||||
showTimeStamp: boolean;
|
||||
qualityindicators: any;
|
||||
platform: string; // id string - e.g. "goodjudgment", not "Good Judgment"
|
||||
}> = ({
|
||||
numforecasts,
|
||||
timestamp,
|
||||
showTimeStamp,
|
||||
|
|
@ -504,6 +518,7 @@ let displayQualityIndicators = ({
|
|||
let forecastFooter = ({
|
||||
stars,
|
||||
platform,
|
||||
platformLabel,
|
||||
numforecasts,
|
||||
qualityindicators,
|
||||
timestamp,
|
||||
|
|
@ -532,7 +547,7 @@ let forecastFooter = ({
|
|||
expandFooterToFullWidth ? "place-self-center" : "self-center"
|
||||
} col-span-1 font-bold ${debuggingWithBackground ? "bg-red-100" : ""}`}
|
||||
>
|
||||
{platform
|
||||
{platformLabel
|
||||
.replace("Good Judgment Open", "GJOpen")
|
||||
.replace("OpenPhilanthropy", "OpenPhil")
|
||||
.replace("AstralCodexTen", "ACX")
|
||||
|
|
@ -559,22 +574,30 @@ let forecastFooter = ({
|
|||
|
||||
/* Body */
|
||||
|
||||
export function displayForecast({
|
||||
interface SingleProps {
|
||||
forecast: FrontendForecast;
|
||||
showTimeStamp: boolean;
|
||||
expandFooterToFullWidth: boolean;
|
||||
showIdToggle?: boolean;
|
||||
}
|
||||
|
||||
export const DisplayForecast: React.FC<SingleProps> = ({
|
||||
forecast: {
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
platform,
|
||||
author,
|
||||
platformLabel,
|
||||
description,
|
||||
options,
|
||||
qualityindicators,
|
||||
timestamp,
|
||||
visualization,
|
||||
score,
|
||||
},
|
||||
showTimeStamp,
|
||||
expandFooterToFullWidth,
|
||||
showIdToggle,
|
||||
}) {
|
||||
}) => {
|
||||
// const [isJustCopiedSignalVisible, setIsJustCopiedSignalVisible] = useState(false)
|
||||
const isJustCopiedSignalVisible = false;
|
||||
|
||||
|
|
@ -588,7 +611,7 @@ export function displayForecast({
|
|||
<div className="flex-grow">
|
||||
<div
|
||||
className={`text-gray-800 ${opacityFromScore(
|
||||
score
|
||||
0
|
||||
)} text-lg mb-2 font-medium justify-self-start`}
|
||||
>
|
||||
<div
|
||||
|
|
@ -641,7 +664,7 @@ export function displayForecast({
|
|||
? "flex"
|
||||
: "hidden"
|
||||
} ${opacityFromScore(
|
||||
score
|
||||
0
|
||||
)} row-end-2 col-start-2 col-end-2 row-start-1 row-end-1 col-span-1 items-center justify-center text-gray-600 ml-3 mr-2 `}
|
||||
>
|
||||
<svg className="mt-1" height="10" width="16">
|
||||
|
|
@ -657,7 +680,7 @@ export function displayForecast({
|
|||
{(options.length != 2 ||
|
||||
(options[0].name != "Yes" && options[0].name != "No")) && (
|
||||
<>
|
||||
<div className={`mb-2 mt-2 ${opacityFromScore(score)}`}>
|
||||
<div className={`mb-2 mt-2 ${opacityFromScore(0)}`}>
|
||||
{formatForecastOptions(options)}
|
||||
</div>
|
||||
<div
|
||||
|
|
@ -667,7 +690,7 @@ export function displayForecast({
|
|||
? "flex"
|
||||
: "hidden"
|
||||
} ${opacityFromScore(
|
||||
score
|
||||
0
|
||||
)} col-start-2 col-end-2 row-start-1 row-end-1 text-gray-600 mt-3 mb-3`}
|
||||
>
|
||||
<svg className="ml-6 mr-1 mt-2" height="10" width="16">
|
||||
|
|
@ -680,13 +703,13 @@ export function displayForecast({
|
|||
</>
|
||||
)}
|
||||
|
||||
{platform !== "Guesstimate" && options.length < 3 && (
|
||||
<div className={`text-gray-500 ${opacityFromScore(score)} mt-4`}>
|
||||
{platform !== "guesstimate" && options.length < 3 && (
|
||||
<div className={`text-gray-500 ${opacityFromScore(0)} mt-4`}>
|
||||
{displayMarkdown(description)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{platform === "Guesstimate" && (
|
||||
{platform === "guesstimate" && (
|
||||
<img
|
||||
className="rounded-sm mb-1"
|
||||
src={visualization}
|
||||
|
|
@ -705,10 +728,11 @@ export function displayForecast({
|
|||
</svg>
|
||||
{`Last updated: ${timestamp ? timestamp.slice(0, 10) : "unknown"}`}
|
||||
</div>
|
||||
<div className={`${opacityFromScore(score)} w-full`}>
|
||||
<div className={`${opacityFromScore(0)} w-full`}>
|
||||
{forecastFooter({
|
||||
stars: qualityindicators.stars,
|
||||
platform: author || platform,
|
||||
platform: platform,
|
||||
platformLabel: platformLabel || platform, // author || platformLabel,
|
||||
numforecasts: qualityindicators.numforecasts,
|
||||
qualityindicators,
|
||||
timestamp,
|
||||
|
|
@ -718,30 +742,39 @@ export function displayForecast({
|
|||
</div>
|
||||
</a>
|
||||
);
|
||||
};
|
||||
|
||||
interface Props {
|
||||
results: FrontendForecast[];
|
||||
numDisplay: number;
|
||||
showIdToggle: boolean;
|
||||
}
|
||||
|
||||
export default function displayForecasts({
|
||||
const DisplayForecasts: React.FC<Props> = ({
|
||||
results,
|
||||
numDisplay,
|
||||
showIdToggle,
|
||||
}) {
|
||||
return !!results && !!results.slice ? (
|
||||
results.slice(0, numDisplay).map((fuseSearchResult) => {
|
||||
}) => {
|
||||
if (!results) {
|
||||
return <></>;
|
||||
}
|
||||
return (
|
||||
<>
|
||||
{results.slice(0, numDisplay).map((result) => (
|
||||
/*let displayWithMetaculusCapture =
|
||||
fuseSearchResult.item.platform == "Metaculus"
|
||||
? metaculusEmbed(fuseSearchResult.item)
|
||||
: displayForecast({ ...fuseSearchResult.item });
|
||||
*/
|
||||
let display = displayForecast({
|
||||
...fuseSearchResult.item,
|
||||
score: fuseSearchResult.score,
|
||||
showTimeStamp: false,
|
||||
expandFooterToFullWidth: false,
|
||||
showIdToggle,
|
||||
});
|
||||
return display;
|
||||
})
|
||||
) : (
|
||||
<></>
|
||||
<DisplayForecast
|
||||
forecast={result}
|
||||
showTimeStamp={false}
|
||||
expandFooterToFullWidth={false}
|
||||
showIdToggle={showIdToggle}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export default DisplayForecasts;
|
||||
|
|
|
|||
|
|
@ -2,20 +2,20 @@ import domtoimage from "dom-to-image"; // https://github.com/tsayen/dom-to-image
|
|||
import { useEffect, useRef, useState } from "react";
|
||||
import { CopyToClipboard } from "react-copy-to-clipboard";
|
||||
|
||||
import { FrontendForecast } from "../platforms";
|
||||
import { uploadToImgur } from "../worker/uploadToImgur";
|
||||
import { displayForecast } from "./displayForecasts";
|
||||
import { DisplayForecast } from "./displayForecasts";
|
||||
|
||||
function displayOneForecastInner(result, containerRef) {
|
||||
function displayOneForecastInner(result: FrontendForecast, containerRef) {
|
||||
return (
|
||||
<div ref={containerRef}>
|
||||
{result
|
||||
? displayForecast({
|
||||
...result.item,
|
||||
score: result.score,
|
||||
showTimeStamp: true,
|
||||
expandFooterToFullWidth: true,
|
||||
})
|
||||
: null}
|
||||
{result ? (
|
||||
<DisplayForecast
|
||||
forecast={result}
|
||||
showTimeStamp={true}
|
||||
expandFooterToFullWidth={true}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -170,7 +170,7 @@ let generateMetaculusSource = (result, hasDisplayBeenCaptured) => {
|
|||
};
|
||||
|
||||
interface Props {
|
||||
result: any;
|
||||
result: FrontendForecast;
|
||||
}
|
||||
|
||||
const DisplayOneForecast: React.FC<Props> = ({ result }) => {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import chroma from "chroma-js";
|
|||
import React from "react";
|
||||
import Select from "react-select";
|
||||
|
||||
import { platformsWithLabels } from "../platforms";
|
||||
import { PlatformConfig } from "../platforms";
|
||||
|
||||
const colourStyles = {
|
||||
control: (styles) => ({ ...styles, backgroundColor: "white" }),
|
||||
|
|
@ -59,17 +59,48 @@ const colourStyles = {
|
|||
}),
|
||||
};
|
||||
|
||||
export default function MultiSelectPlatform({ onChange, value }) {
|
||||
interface Props {
|
||||
onChange: (platforms: string[]) => void;
|
||||
value: string[];
|
||||
platformsConfig: PlatformConfig[];
|
||||
}
|
||||
|
||||
export const MultiSelectPlatform: React.FC<Props> = ({
|
||||
onChange,
|
||||
value,
|
||||
platformsConfig,
|
||||
}) => {
|
||||
type Option = {
|
||||
value: string;
|
||||
label: string;
|
||||
color: string;
|
||||
};
|
||||
|
||||
const options: Option[] = platformsConfig.map((platform) => ({
|
||||
value: platform.name,
|
||||
label: platform.label,
|
||||
color: platform.color,
|
||||
}));
|
||||
|
||||
const id2option: { [k: string]: Option } = {};
|
||||
for (const option of options) id2option[option.value] = option;
|
||||
|
||||
const selectValue = value.map((v) => id2option[v]).filter((v) => v);
|
||||
|
||||
const onSelectChange = (newValue: Option[]) => {
|
||||
onChange(newValue.map((o) => o.value));
|
||||
};
|
||||
|
||||
return (
|
||||
<Select
|
||||
defaultValue={platformsWithLabels}
|
||||
defaultValue={options}
|
||||
isMulti
|
||||
className="basic-multi-select w-full text-gray-700"
|
||||
onChange={onChange}
|
||||
onChange={onSelectChange}
|
||||
closeMenuOnSelect={false}
|
||||
options={platformsWithLabels}
|
||||
value={value}
|
||||
options={options}
|
||||
value={selectValue}
|
||||
styles={colourStyles}
|
||||
/>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,61 +1,12 @@
|
|||
export const distinctColors = [
|
||||
"#3d674a",
|
||||
"#231149",
|
||||
"#62520b",
|
||||
"#32407e",
|
||||
"#7d4f1b",
|
||||
"#002455",
|
||||
"#223900",
|
||||
"#615691",
|
||||
"#003419",
|
||||
"#793466",
|
||||
"#006669",
|
||||
"#984158",
|
||||
"#00314e",
|
||||
"#460c00",
|
||||
"#0d1624",
|
||||
"#6f5b41",
|
||||
"#240d23",
|
||||
"#272600",
|
||||
"#755469",
|
||||
"#2a323d",
|
||||
];
|
||||
import { Forecast } from "../backend/platforms";
|
||||
|
||||
// https://medialab.github.io/iwanthue/ fancy light background
|
||||
export const platformNames = [
|
||||
"Betfair",
|
||||
"FantasySCOTUS",
|
||||
"Foretold",
|
||||
"GiveWell/OpenPhilanthropy",
|
||||
"Good Judgment",
|
||||
"Good Judgment Open",
|
||||
"Guesstimate",
|
||||
"Infer",
|
||||
"Kalshi",
|
||||
"Manifold Markets",
|
||||
"Metaculus",
|
||||
"Peter Wildeford",
|
||||
"PolyMarket",
|
||||
"PredictIt",
|
||||
"Rootclaim",
|
||||
"Smarkets",
|
||||
"X-risk estimates",
|
||||
];
|
||||
|
||||
export interface PlatformWithLabel {
|
||||
value: string;
|
||||
export interface PlatformConfig {
|
||||
name: string;
|
||||
label: string;
|
||||
color: string;
|
||||
}
|
||||
|
||||
export const platformsWithLabels: PlatformWithLabel[] = platformNames.map(
|
||||
(name, i) => ({
|
||||
value: name,
|
||||
label: name,
|
||||
color: distinctColors[i],
|
||||
})
|
||||
);
|
||||
|
||||
export const platforms = platformsWithLabels;
|
||||
|
||||
// Deprecated: AstralCodexTen, CoupCast, CSET-foretell, Estimize, Elicit, Hypermind, Omen, WilliamHill
|
||||
export type FrontendForecast = Forecast & {
|
||||
platformLabel: string;
|
||||
visualization?: any;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,54 +1,13 @@
|
|||
import { useRouter } from "next/router";
|
||||
import React, {
|
||||
DependencyList,
|
||||
EffectCallback,
|
||||
Fragment,
|
||||
useEffect,
|
||||
useState,
|
||||
} from "react";
|
||||
import React, { DependencyList, EffectCallback, Fragment, useEffect, useState } from "react";
|
||||
|
||||
import ButtonsForStars from "../display/buttonsForStars";
|
||||
import Form from "../display/form";
|
||||
import MultiSelectPlatform from "../display/multiSelectPlatforms";
|
||||
import { MultiSelectPlatform } from "../display/multiSelectPlatforms";
|
||||
import { SliderElement } from "../display/slider";
|
||||
import { platformsWithLabels, PlatformWithLabel } from "../platforms";
|
||||
import { FrontendForecast } from "../platforms";
|
||||
import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
|
||||
|
||||
interface QueryParametersWithoutNum {
|
||||
query: string;
|
||||
starsThreshold: number;
|
||||
forecastsThreshold: number;
|
||||
forecastingPlatforms: PlatformWithLabel[];
|
||||
}
|
||||
|
||||
export interface QueryParameters extends QueryParametersWithoutNum {
|
||||
numDisplay: number;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
defaultResults: any;
|
||||
initialResults: any;
|
||||
initialQueryParameters: QueryParameters;
|
||||
hasSearchbar: boolean;
|
||||
hasCapture: boolean;
|
||||
hasAdvancedOptions: boolean;
|
||||
placeholder: string;
|
||||
displaySeeMoreHint: boolean;
|
||||
displayForecastsWrapper: (opts: {
|
||||
results: any;
|
||||
numDisplay: number;
|
||||
whichResultToDisplayAndCapture: number;
|
||||
showIdToggle: boolean;
|
||||
}) => React.ReactNode;
|
||||
}
|
||||
|
||||
export const defaultQueryParameters: QueryParametersWithoutNum = {
|
||||
query: "",
|
||||
starsThreshold: 2,
|
||||
forecastsThreshold: 0,
|
||||
forecastingPlatforms: platformsWithLabels, // weird key value format,
|
||||
};
|
||||
export const defaultNumDisplay = 21;
|
||||
import { Props as AnySearchPageProps, QueryParameters } from "./anySearchPage";
|
||||
|
||||
const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => {
|
||||
const initial = React.useRef(true);
|
||||
|
|
@ -61,11 +20,29 @@ const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => {
|
|||
}, deps);
|
||||
};
|
||||
|
||||
interface Props extends AnySearchPageProps {
|
||||
hasSearchbar: boolean;
|
||||
hasCapture: boolean;
|
||||
hasAdvancedOptions: boolean;
|
||||
placeholder: string;
|
||||
displaySeeMoreHint: boolean;
|
||||
displayForecastsWrapper: (opts: {
|
||||
results: FrontendForecast[];
|
||||
numDisplay: number;
|
||||
whichResultToDisplayAndCapture: number;
|
||||
showIdToggle: boolean;
|
||||
}) => React.ReactNode;
|
||||
}
|
||||
|
||||
/* Body */
|
||||
const CommonDisplay: React.FC<Props> = ({
|
||||
defaultResults,
|
||||
initialResults,
|
||||
initialQueryParameters,
|
||||
defaultQueryParameters,
|
||||
initialNumDisplay,
|
||||
defaultNumDisplay,
|
||||
platformsConfig,
|
||||
hasSearchbar,
|
||||
hasCapture,
|
||||
hasAdvancedOptions,
|
||||
|
|
@ -76,13 +53,12 @@ const CommonDisplay: React.FC<Props> = ({
|
|||
const router = useRouter();
|
||||
/* States */
|
||||
|
||||
const [queryParameters, setQueryParameters] =
|
||||
useState<QueryParametersWithoutNum>(initialQueryParameters);
|
||||
|
||||
const [numDisplay, setNumDisplay] = useState(
|
||||
initialQueryParameters.numDisplay ?? defaultNumDisplay
|
||||
const [queryParameters, setQueryParameters] = useState<QueryParameters>(
|
||||
initialQueryParameters
|
||||
);
|
||||
|
||||
const [numDisplay, setNumDisplay] = useState(initialNumDisplay);
|
||||
|
||||
// used to distinguish numDisplay updates which force search and don't force search, see effects below
|
||||
const [forceSearch, setForceSearch] = useState(0);
|
||||
|
||||
|
|
@ -100,25 +76,25 @@ const CommonDisplay: React.FC<Props> = ({
|
|||
numDisplay,
|
||||
};
|
||||
|
||||
let filterManually = (queryData: QueryParameters, results) => {
|
||||
let filterManually = (
|
||||
queryData: QueryParameters,
|
||||
results: FrontendForecast[]
|
||||
) => {
|
||||
if (
|
||||
queryData.forecastingPlatforms &&
|
||||
queryData.forecastingPlatforms.length > 0
|
||||
) {
|
||||
let forecastingPlatforms = queryData.forecastingPlatforms.map(
|
||||
(platformObj) => platformObj.value
|
||||
);
|
||||
results = results.filter((result) =>
|
||||
forecastingPlatforms.includes(result.item.platform)
|
||||
queryData.forecastingPlatforms.includes(result.platform)
|
||||
);
|
||||
}
|
||||
if (queryData.starsThreshold === 4) {
|
||||
results = results.filter(
|
||||
(result) => result.item.qualityindicators.stars >= 4
|
||||
(result) => result.qualityindicators.stars >= 4
|
||||
);
|
||||
}
|
||||
if (queryData.forecastsThreshold) {
|
||||
// results = results.filter(result => (result.qualityindicators && result.item.qualityindicators.numforecasts > forecastsThreshold))
|
||||
// results = results.filter(result => (result.qualityindicators && result.qualityindicators.numforecasts > forecastsThreshold))
|
||||
}
|
||||
return results;
|
||||
};
|
||||
|
|
@ -128,17 +104,13 @@ const CommonDisplay: React.FC<Props> = ({
|
|||
|
||||
let results = queryIsEmpty
|
||||
? filterManually(queryData, defaultResults)
|
||||
: await searchAccordingToQueryData(queryData);
|
||||
: await searchAccordingToQueryData(queryData, numDisplay);
|
||||
|
||||
setResults(results);
|
||||
}
|
||||
|
||||
// I don't want the function which display forecasts (displayForecasts) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home.
|
||||
let getInfoToDisplayForecastsFunction = ({
|
||||
results,
|
||||
whichResultToDisplayAndCapture,
|
||||
showIdToggle,
|
||||
}) => {
|
||||
let getInfoToDisplayForecastsFunction = () => {
|
||||
let numDisplayRounded =
|
||||
numDisplay % 3 != 0
|
||||
? numDisplay + (3 - (Math.round(numDisplay) % 3))
|
||||
|
|
@ -154,7 +126,7 @@ const CommonDisplay: React.FC<Props> = ({
|
|||
const updateRoute = () => {
|
||||
const stringify = (key: string, value: any) => {
|
||||
if (key === "forecastingPlatforms") {
|
||||
return value.map((x) => x.value).join("|");
|
||||
return value.join("|");
|
||||
} else {
|
||||
return String(value);
|
||||
}
|
||||
|
|
@ -330,6 +302,7 @@ const CommonDisplay: React.FC<Props> = ({
|
|||
</div>
|
||||
<div className="flex col-span-3 items-center justify-center">
|
||||
<MultiSelectPlatform
|
||||
platformsConfig={platformsConfig}
|
||||
value={queryParameters.forecastingPlatforms}
|
||||
onChange={onChangeSelectedPlatforms}
|
||||
/>
|
||||
|
|
@ -344,13 +317,7 @@ const CommonDisplay: React.FC<Props> = ({
|
|||
</div>
|
||||
) : null}
|
||||
|
||||
<div>
|
||||
{getInfoToDisplayForecastsFunction({
|
||||
results,
|
||||
whichResultToDisplayAndCapture,
|
||||
showIdToggle,
|
||||
})}
|
||||
</div>
|
||||
<div>{getInfoToDisplayForecastsFunction()}</div>
|
||||
|
||||
{displaySeeMoreHint &&
|
||||
(!results || (results.length != 0 && numDisplay < results.length)) ? (
|
||||
|
|
|
|||
|
|
@ -1,46 +1,103 @@
|
|||
import { GetServerSideProps } from "next";
|
||||
|
||||
import { getFrontpage } from "../../backend/frontpage";
|
||||
import { platforms } from "../../backend/platforms";
|
||||
import { FrontendForecast, PlatformConfig } from "../platforms";
|
||||
import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
|
||||
import {
|
||||
defaultNumDisplay,
|
||||
defaultQueryParameters,
|
||||
QueryParameters,
|
||||
} from "./CommonDisplay";
|
||||
|
||||
/* Common code for / and /capture */
|
||||
|
||||
export interface QueryParameters {
|
||||
query: string;
|
||||
starsThreshold: number;
|
||||
forecastsThreshold: number;
|
||||
forecastingPlatforms: string[]; // platform names
|
||||
}
|
||||
|
||||
export interface Props {
|
||||
defaultResults: any;
|
||||
initialResults: any;
|
||||
defaultResults: FrontendForecast[];
|
||||
initialResults: FrontendForecast[];
|
||||
initialQueryParameters: QueryParameters;
|
||||
defaultQueryParameters: QueryParameters;
|
||||
initialNumDisplay: number;
|
||||
defaultNumDisplay: number;
|
||||
platformsConfig: PlatformConfig[];
|
||||
}
|
||||
|
||||
export const getServerSideProps: GetServerSideProps<Props> = async (
|
||||
context
|
||||
) => {
|
||||
let urlQuery = context.query;
|
||||
const urlQuery = context.query;
|
||||
|
||||
let initialQueryParameters: QueryParameters = {
|
||||
...defaultQueryParameters,
|
||||
numDisplay: defaultNumDisplay,
|
||||
...urlQuery, // FIXME - parse numerical fields
|
||||
const platformsConfig = platforms.map((platform) => ({
|
||||
name: platform.name,
|
||||
label: platform.label,
|
||||
color: platform.color,
|
||||
}));
|
||||
platformsConfig.push({
|
||||
name: "guesstimate",
|
||||
label: "Guesstimate",
|
||||
color: "223900",
|
||||
});
|
||||
|
||||
const defaultQueryParameters: QueryParameters = {
|
||||
query: "",
|
||||
starsThreshold: 2,
|
||||
forecastsThreshold: 0,
|
||||
forecastingPlatforms: platforms.map((platform) => platform.name),
|
||||
};
|
||||
|
||||
let defaultResults = await getFrontpage();
|
||||
const initialQueryParameters: QueryParameters = {
|
||||
...defaultQueryParameters,
|
||||
};
|
||||
if (urlQuery.query) {
|
||||
initialQueryParameters.query = String(urlQuery.query);
|
||||
}
|
||||
if (urlQuery.starsThreshold) {
|
||||
initialQueryParameters.starsThreshold = Number(urlQuery.starsThreshold);
|
||||
}
|
||||
if (urlQuery.forecastsThreshold !== undefined) {
|
||||
initialQueryParameters.forecastsThreshold = Number(
|
||||
urlQuery.forecastsThreshold
|
||||
);
|
||||
}
|
||||
if (urlQuery.forecastingPlatforms !== undefined) {
|
||||
initialQueryParameters.forecastingPlatforms = String(
|
||||
urlQuery.forecastingPlatforms
|
||||
).split("|");
|
||||
}
|
||||
|
||||
const platformNameToLabel = Object.fromEntries(
|
||||
platforms.map((platform) => [platform.name, platform.label])
|
||||
);
|
||||
|
||||
const defaultNumDisplay = 21;
|
||||
const initialNumDisplay = Number(urlQuery.numDisplay) || defaultNumDisplay;
|
||||
|
||||
const defaultResults = (await getFrontpage()).map((result) => ({
|
||||
...result,
|
||||
platformLabel: platformNameToLabel[result.platform] || result.platform,
|
||||
}));
|
||||
|
||||
const initialResults =
|
||||
!!initialQueryParameters &&
|
||||
initialQueryParameters.query != "" &&
|
||||
initialQueryParameters.query != undefined
|
||||
? await searchAccordingToQueryData(initialQueryParameters)
|
||||
? await searchAccordingToQueryData(
|
||||
initialQueryParameters,
|
||||
initialNumDisplay
|
||||
)
|
||||
: defaultResults;
|
||||
|
||||
return {
|
||||
props: {
|
||||
initialQueryParameters,
|
||||
defaultQueryParameters,
|
||||
initialNumDisplay,
|
||||
defaultNumDisplay,
|
||||
initialResults,
|
||||
defaultResults,
|
||||
platformsConfig,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import axios from "axios";
|
|||
|
||||
export async function getDashboardForecastsByDashboardId({ dashboardId }) {
|
||||
console.log("getDashboardForecastsByDashboardId: ");
|
||||
let dashboardForecastCompatibleWithFuse = [];
|
||||
let dashboardContents = [];
|
||||
let dashboardItem = null;
|
||||
try {
|
||||
let { data } = await axios({
|
||||
|
|
@ -13,22 +13,13 @@ export async function getDashboardForecastsByDashboardId({ dashboardId }) {
|
|||
},
|
||||
});
|
||||
console.log(data);
|
||||
let dashboardContents = data.dashboardContents;
|
||||
dashboardContents = data.dashboardContents;
|
||||
dashboardItem = data.dashboardItem;
|
||||
// let { dashboardContents, dashboardItem } = data
|
||||
if (!!dashboardContents && !!dashboardContents.map) {
|
||||
dashboardForecastCompatibleWithFuse = dashboardContents.map((result) => ({
|
||||
item: result,
|
||||
score: 0,
|
||||
}));
|
||||
} else {
|
||||
console.log("Error in getDashboardForecastsByDashboardId");
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
} finally {
|
||||
return {
|
||||
dashboardForecasts: dashboardForecastCompatibleWithFuse,
|
||||
dashboardForecasts: dashboardContents,
|
||||
dashboardItem,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,36 +1,35 @@
|
|||
import { FrontendForecast } from "../platforms";
|
||||
import { QueryParameters } from "../search/anySearchPage";
|
||||
import searchGuesstimate from "./searchGuesstimate";
|
||||
import searchWithAlgolia from "./searchWithAlgolia";
|
||||
|
||||
export default async function searchAccordingToQueryData(queryData) {
|
||||
let results = [];
|
||||
export default async function searchAccordingToQueryData(
|
||||
queryData: QueryParameters,
|
||||
limit: number
|
||||
): Promise<FrontendForecast[]> {
|
||||
let results: FrontendForecast[] = [];
|
||||
|
||||
try {
|
||||
// defs
|
||||
let query = queryData.query == undefined ? "" : queryData.query;
|
||||
if (query == "") return -1;
|
||||
if (query == "") return [];
|
||||
let forecastsThreshold = queryData.forecastsThreshold;
|
||||
let starsThreshold = queryData.starsThreshold;
|
||||
let forecastingPlatforms = queryData.forecastingPlatforms.map(
|
||||
(x) => x.value
|
||||
);
|
||||
let platformsIncludeGuesstimate =
|
||||
forecastingPlatforms.includes("Guesstimate") && starsThreshold <= 1;
|
||||
queryData.forecastingPlatforms.includes("guesstimate") &&
|
||||
starsThreshold <= 1;
|
||||
|
||||
// preparation
|
||||
let unawaitedAlgoliaResponse = searchWithAlgolia({
|
||||
queryString: query,
|
||||
hitsPerPage: queryData.numDisplay + 50,
|
||||
hitsPerPage: limit + 50,
|
||||
starsThreshold,
|
||||
filterByPlatforms: forecastingPlatforms,
|
||||
filterByPlatforms: queryData.forecastingPlatforms,
|
||||
forecastsThreshold,
|
||||
});
|
||||
|
||||
// consider the guesstimate and the non-guesstimate cases separately.
|
||||
switch (platformsIncludeGuesstimate) {
|
||||
case false:
|
||||
results = await unawaitedAlgoliaResponse;
|
||||
break;
|
||||
case true:
|
||||
if (platformsIncludeGuesstimate) {
|
||||
let responses = await Promise.all([
|
||||
unawaitedAlgoliaResponse,
|
||||
searchGuesstimate(query),
|
||||
|
|
@ -39,18 +38,11 @@ export default async function searchAccordingToQueryData(queryData) {
|
|||
let responsesGuesstimate = responses[1];
|
||||
results = [...responsesNotGuesstimate, ...responsesGuesstimate];
|
||||
//results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
|
||||
break;
|
||||
default:
|
||||
return -1;
|
||||
} else {
|
||||
results = await unawaitedAlgoliaResponse;
|
||||
}
|
||||
// Maintain compatibility with fuse
|
||||
let makeCompatibleWithFuse = (results) =>
|
||||
results.map((result, index) => ({
|
||||
item: result,
|
||||
score: 0, // 0.4 - 0.4 / (index + 1),
|
||||
}));
|
||||
|
||||
results = makeCompatibleWithFuse(results);
|
||||
return results;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { FrontendForecast } from "../platforms";
|
||||
|
||||
/* Definitions */
|
||||
let urlEndPoint =
|
||||
"https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c";
|
||||
|
||||
/* Body */
|
||||
|
||||
export default function searchGuesstimate(query) {
|
||||
let response = axios({
|
||||
export default async function searchGuesstimate(
|
||||
query
|
||||
): Promise<FrontendForecast[]> {
|
||||
let response = await axios({
|
||||
url: urlEndPoint,
|
||||
// credentials: "omit",
|
||||
headers: {
|
||||
|
|
@ -24,18 +28,21 @@ export default function searchGuesstimate(query) {
|
|||
"%20"
|
||||
)}&hitsPerPage=20&page=0&getRankingInfo=true\"}`,
|
||||
method: "POST",
|
||||
})
|
||||
.then((res) => res.data.hits)
|
||||
.then((models) =>
|
||||
models.map((model, index) => {
|
||||
});
|
||||
|
||||
const models: any[] = response.data.hits;
|
||||
const mappedModels: FrontendForecast[] = models.map((model, index) => {
|
||||
let description = model.description
|
||||
? model.description.replace(/\n/g, " ").replace(/ /g, " ")
|
||||
: "";
|
||||
let stars = description.length > 250 ? 2 : 1;
|
||||
return {
|
||||
id: `guesstimate-${model.id}`,
|
||||
title: model.name,
|
||||
url: `https://www.getguesstimate.com/models/${model.id}`,
|
||||
platform: "Guesstimate",
|
||||
timestamp: model.created_at, // TODO - check that format matches
|
||||
platform: "guesstimate",
|
||||
platformLabel: "Guesstimate",
|
||||
description: description,
|
||||
options: [],
|
||||
qualityindicators: {
|
||||
|
|
@ -46,26 +53,20 @@ export default function searchGuesstimate(query) {
|
|||
visualization: model.big_screenshot,
|
||||
ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
|
||||
};
|
||||
})
|
||||
)
|
||||
.then((models) => {
|
||||
});
|
||||
|
||||
// filter for duplicates. Surprisingly common.
|
||||
let uniqueTitles = [];
|
||||
let uniqueModels = [];
|
||||
for (let model of models) {
|
||||
if (
|
||||
!uniqueTitles.includes(model.title) &&
|
||||
!model.title.includes("copy")
|
||||
) {
|
||||
let uniqueModels: FrontendForecast[] = [];
|
||||
for (let model of mappedModels) {
|
||||
if (!uniqueTitles.includes(model.title) && !model.title.includes("copy")) {
|
||||
uniqueModels.push(model);
|
||||
uniqueTitles.push(model.title);
|
||||
}
|
||||
}
|
||||
return uniqueModels;
|
||||
});
|
||||
|
||||
console.log(response);
|
||||
return response; // This is a promise. Usable with either async/await (a mess in React) or with .then(guesstimateModels => doSomething(guesstimateModels))
|
||||
console.log(uniqueModels);
|
||||
return uniqueModels;
|
||||
}
|
||||
|
||||
// searchGuesstimate("COVID-19").then(guesstimateModels => console.log(guesstimateModels))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import algoliasearch from "algoliasearch";
|
||||
|
||||
import { FrontendForecast } from "../platforms";
|
||||
|
||||
const client = algoliasearch(
|
||||
process.env.NEXT_PUBLIC_ALGOLIA_APP_ID,
|
||||
process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_KEY
|
||||
|
|
@ -76,13 +78,12 @@ let noExactMatch = (queryString, result) => {
|
|||
// only query string
|
||||
export default async function searchWithAlgolia({
|
||||
queryString,
|
||||
hitsPerPage,
|
||||
hitsPerPage = 5,
|
||||
starsThreshold,
|
||||
filterByPlatforms,
|
||||
forecastsThreshold,
|
||||
}) {
|
||||
hitsPerPage = hitsPerPage || 5;
|
||||
let response = await index.search(queryString, {
|
||||
}): Promise<FrontendForecast[]> {
|
||||
let response = await index.search<FrontendForecast>(queryString, {
|
||||
hitsPerPage,
|
||||
filters: buildFilter({
|
||||
starsThreshold,
|
||||
|
|
@ -92,12 +93,7 @@ export default async function searchWithAlgolia({
|
|||
//facetFilters: buildFacetFilter({filterByPlatforms}),
|
||||
getRankingInfo: true,
|
||||
});
|
||||
let results: any[] = response.hits;
|
||||
console.log(
|
||||
"searchWithAlgolia.js/searchWithAlgolia/queryString",
|
||||
queryString
|
||||
);
|
||||
console.log("searchWithAlgolia.js/searchWithAlgolia/results", results);
|
||||
let results: FrontendForecast[] = response.hits;
|
||||
|
||||
let recursionError = ["metaforecast", "metaforecasts", "metaforecasting"];
|
||||
if (
|
||||
|
|
@ -106,9 +102,11 @@ export default async function searchWithAlgolia({
|
|||
) {
|
||||
results = [
|
||||
{
|
||||
id: "not-found",
|
||||
title: "No search results match your query",
|
||||
url: "https://metaforecast.org",
|
||||
platform: "Metaforecast",
|
||||
platform: "metaforecast",
|
||||
platformLabel: "Metaforecast",
|
||||
description: "Maybe try a broader query?",
|
||||
options: [
|
||||
{
|
||||
|
|
@ -128,17 +126,19 @@ export default async function searchWithAlgolia({
|
|||
numforecasters: 1,
|
||||
stars: 5,
|
||||
},
|
||||
noExactSearchResults: true,
|
||||
optionsstringforsearch: "Yes, No",
|
||||
has_numforecasts: true,
|
||||
// noExactSearchResults: true,
|
||||
// optionsstringforsearch: "Yes, No",
|
||||
// has_numforecasts: true,
|
||||
},
|
||||
];
|
||||
} else if (recursionError.includes(queryString.toLowerCase())) {
|
||||
results = [
|
||||
{
|
||||
id: "recursion-error",
|
||||
title: `Did you mean: ${queryString}?`,
|
||||
url: "https://metaforecast.org/recursion?bypassEasterEgg=true",
|
||||
platform: "Metaforecast",
|
||||
platform: "metaforecast",
|
||||
platformLabel: "Metaforecast",
|
||||
description:
|
||||
"Fatal error: Too much recursion. Click to proceed anyways",
|
||||
options: [
|
||||
|
|
@ -159,9 +159,9 @@ export default async function searchWithAlgolia({
|
|||
numforecasters: 1,
|
||||
stars: 5,
|
||||
},
|
||||
noExactSearchResults: true,
|
||||
optionsstringforsearch: "Yes, No",
|
||||
has_numforecasts: true,
|
||||
// noExactSearchResults: true,
|
||||
// optionsstringforsearch: "Yes, No",
|
||||
// has_numforecasts: true,
|
||||
},
|
||||
...results,
|
||||
];
|
||||
|
|
@ -171,9 +171,11 @@ export default async function searchWithAlgolia({
|
|||
noExactMatch(queryString, results[0])
|
||||
) {
|
||||
results.unshift({
|
||||
id: "not-found-2",
|
||||
title: "No search results appear to match your query",
|
||||
url: "https://metaforecast.org",
|
||||
platform: "Metaforecast",
|
||||
platform: "metaforecast",
|
||||
platformLabel: "Metaforecast",
|
||||
description: "Maybe try a broader query? That said, we could be wrong.",
|
||||
options: [
|
||||
{
|
||||
|
|
@ -193,12 +195,12 @@ export default async function searchWithAlgolia({
|
|||
numforecasters: 1,
|
||||
stars: 1,
|
||||
},
|
||||
noExactSearchResults: true,
|
||||
optionsstringforsearch: "Yes, No",
|
||||
has_numforecasts: true,
|
||||
// noExactSearchResults: true,
|
||||
// optionsstringforsearch: "Yes, No",
|
||||
// has_numforecasts: true,
|
||||
});
|
||||
} else {
|
||||
results[0].noExactSearchResults = false;
|
||||
// results[0].noExactSearchResults = false;
|
||||
}
|
||||
|
||||
return results;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@
|
|||
"incremental": true,
|
||||
"moduleResolution": "node",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true
|
||||
"isolatedModules": true,
|
||||
"allowJs": true
|
||||
},
|
||||
"include": [
|
||||
"next-env.d.ts",
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user