quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #42 from QURIresearch/no-schemas

Browse Source 
Should be fine. Tables in `public` schema are ready.

Fixed secretEmbed and dashboards, fixed small bugs in platforms code (I'm starting to dislike the fact that platform fetchers have to populate `platform` field and set `id` to `${platform}-${shortId}`, it seems error-prone; maybe platforms should remove an object with `id: shortId` and without `platform`, and then an external code would populate it as needed... but then we'd need a different type for "fetcher output result", and I'm still hesitant).

Steps for the next half-hour:
- [ ] move tables to new DB instance in the US (copied with pg_dump/pg_restore)
- [ ] merge code, deploy on heroku/netlify
- [ ] switch to the new db
- [ ] update algolia index
- [ ] check that everything works
- [ ] drop schemas (from new DB, not a problem even in case of significant issues since we have backups and the old DB)
This commit is contained in:
Vyacheslav Matyukhin 2022-04-02 03:12:36 +03:00 committed by GitHub
commit 5ccbb7fabc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 905 additions and 1002 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -26,12 +26,6 @@ npm-debug.log*
yarn-debug.log* yarn-debug.log*
yarn-error.log* yarn-error.log*
# local env files
.env.local
.env.development.local
.env.test.local
.env.production.local
# vercel # vercel
.vercel .vercel
@ -41,5 +35,4 @@ package-lock.json ## use yarn.lock instead
# Local Netlify folder # Local Netlify folder
.netlify .netlify
/.env /.env*
/.env.production

55
src/backend/database/database-wrapper.ts
View File

@ -1,55 +0,0 @@
import { pgRead, pgReadWithReadCredentials, pgUpsert } from "./pg-wrapper";
const dateUpToYear = () => new Date().toISOString().slice(0, 4);
const dateUpToMonth = () =>
new Date().toISOString().slice(0, 7).replace("-", "_");
export async function databaseUpsert({ contents, group }) {
// No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
// (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){
switch (group) {
case "combined":
await pgUpsert({ contents, schema: "latest", tableName: "combined" });
break;
case "history":
await pgUpsert({
contents,
schema: "history",
tableName: `h${dateUpToYear()}`,
});
await pgUpsert({
contents,
schema: "history",
tableName: `h${dateUpToMonth()}`,
});
break;
default:
await pgUpsert({ contents, schema: "latest", tableName: group });
}
}
const readWithReader = async (
group: string,
reader: (opts: { schema: string; tableName: string }) => Promise<any>
) => {
const schema = group === "history" ? "history" : "latest";
const tableName = group === "history" ? `h${dateUpToMonth()}` : group;
const response = await reader({
schema,
tableName,
});
console.log("Postgres: ");
console.log(response.slice(0, 2));
console.log("");
return response;
};
export async function databaseRead({ group }) {
return await readWithReader(group, pgRead);
}
export async function databaseReadWithReadCredentials({ group }) {
return await readWithReader(group, pgReadWithReadCredentials);
}

331
src/backend/database/pg-wrapper.ts
View File

@ -1,37 +1,13 @@
import { Pool, PoolClient } from "pg"; import { Pool, PoolClient } from "pg";
import { Forecast, platforms } from "../platforms"; import { Forecast } from "../platforms";
import { hash } from "../utils/hash"; import { hash } from "../utils/hash";
import { measureTime } from "../utils/measureTime"; import { measureTime } from "../utils/measureTime";
import { roughSizeOfObject } from "../utils/roughSize"; import { roughSizeOfObject } from "../utils/roughSize";
// Definitions const forecastTableNames = ["questions", "history"];
const schemas = ["latest", "history"];
const year = Number(new Date().toISOString().slice(0, 4)); const allTableNames = [...forecastTableNames, "dashboards", "frontpage"];
const allowed_years = [year, year + 1].map((year) => `h${year}`); // tables can't begin with number
const allowed_months = [...Array(12).keys()]
.map((x) => x + 1)
.map((x) => (String(x).length == 1 ? `0${x}` : x));
const allowed_year_month_histories = [].concat(
...allowed_years.map((year) =>
allowed_months.map((month) => `${year}_${month}`)
)
); // h2022_01
const tableNamesWhitelistLatest = [
"combined",
...platforms.map((platform) => platform.name),
];
const tableNamesWhiteListHistory = [
...allowed_years,
...allowed_year_month_histories,
];
const createFullName = (schemaName, namesArray) =>
namesArray.map((name) => `${schemaName}.${name}`);
const tableWhiteList = [
...createFullName("latest", tableNamesWhitelistLatest),
...createFullName("history", tableNamesWhiteListHistory),
"latest.dashboards",
];
/* Postgres database connection code */ /* Postgres database connection code */
const databaseURL = process.env.DIGITALOCEAN_POSTGRES; const databaseURL = process.env.DIGITALOCEAN_POSTGRES;
@ -80,12 +56,11 @@ export const runPgCommand = async ({
}; };
// Initialize // Initialize
let dropTable = (schema: string, table: string) => let dropTable = (table: string) => `DROP TABLE IF EXISTS ${table}`;
`DROP TABLE IF EXISTS ${schema}.${table}`; let createIndex = (table: string) =>
let createIndex = (schema: string, table: string) => `CREATE INDEX ${table}_id_index ON ${table} (id);`;
`CREATE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`; let createUniqueIndex = (table: string) =>
let createUniqueIndex = (schema: string, table: string) => `CREATE UNIQUE INDEX ${table}_id_index ON ${table} (id);`;
`CREATE UNIQUE INDEX ${schema}_${table}_id_index ON ${schema}.${table} (id);`;
async function pgInitializeScaffolding() { async function pgInitializeScaffolding() {
async function setPermissionsForPublicUser() { async function setPermissionsForPublicUser() {
@ -97,42 +72,20 @@ async function pgInitializeScaffolding() {
await runPgCommand({ command, pool: readWritePool }); await runPgCommand({ command, pool: readWritePool });
} }
let buildGrantSelectForSchema = (schema: string) => await runPgCommand({
`GRANT SELECT ON ALL TABLES IN SCHEMA ${schema} TO public_read_only_user`; command:
for (let schema of schemas) { "GRANT SELECT ON ALL TABLES IN SCHEMA public TO public_read_only_user",
await runPgCommand({ pool: readWritePool,
command: buildGrantSelectForSchema(schema), });
pool: readWritePool,
});
}
let alterDefaultPrivilegesForSchema = (schema: string) => await runPgCommand({
`ALTER DEFAULT PRIVILEGES IN SCHEMA ${schema} GRANT SELECT ON TABLES TO public_read_only_user`; command:
for (let schema of schemas) { "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO public_read_only_user",
await runPgCommand({ pool: readWritePool,
command: alterDefaultPrivilegesForSchema(schema), });
pool: readWritePool,
});
}
} }
let YOLO = false; let YOLO = false;
if (YOLO) { if (YOLO) {
console.log("Create schemas");
for (let schema of schemas) {
await runPgCommand({
command: `CREATE SCHEMA IF NOT EXISTS ${schema}`,
pool: readWritePool,
});
}
console.log("");
console.log("Set search path");
await runPgCommand({
command: `SET search_path TO ${schemas.join(",")},public;`,
pool: readWritePool,
});
console.log("");
console.log("Set public user permissions"); console.log("Set public user permissions");
await setPermissionsForPublicUser(); await setPermissionsForPublicUser();
console.log(""); console.log("");
@ -143,10 +96,7 @@ async function pgInitializeScaffolding() {
} }
} }
let buildMetaforecastTable = ( let buildMetaforecastTable = (table: string) => `CREATE TABLE ${table} (
schema: string,
table: string
) => `CREATE TABLE ${schema}.${table} (
id text, id text,
title text, title text,
url text, url text,
@ -159,45 +109,34 @@ let buildMetaforecastTable = (
extra json extra json
);`; );`;
async function pgInitializeLatest() { async function pgInitializeQuestions() {
let YOLO = false; let YOLO = false;
if (YOLO) { if (YOLO) {
console.log("Create tables & their indexes"); console.log("Create tables & their indexes");
let schema = "latest"; const table = "questions";
for (let table of tableNamesWhitelistLatest) { await runPgCommand({
await runPgCommand({ command: dropTable(table),
command: dropTable(schema, table), pool: readWritePool,
pool: readWritePool, });
}); await runPgCommand({
await runPgCommand({ command: buildMetaforecastTable(table),
command: buildMetaforecastTable(schema, table), pool: readWritePool,
pool: readWritePool, });
}); await runPgCommand({
/* command: createUniqueIndex(table),
if (schema == "history") { pool: readWritePool,
await runPgCommand({ });
command: createIndex(schema, table),
pool: readWritePool,
});
} else {
*/
await runPgCommand({
command: createUniqueIndex(schema, table),
pool: readWritePool,
});
//}
}
console.log(""); console.log("");
} else { } else {
console.log( console.log(
"pgInitializeLatest: This command is dangerous, set YOLO to true in the code to invoke it" "pgInitializeQuestions: This command is dangerous, set YOLO to true in the code to invoke it"
); );
} }
} }
async function pgInitializeDashboards() { async function pgInitializeDashboards() {
let buildDashboard = () => let buildDashboard = () =>
`CREATE TABLE latest.dashboards ( `CREATE TABLE dashboards (
id text, id text,
title text, title text,
description text, description text,
@ -208,23 +147,10 @@ async function pgInitializeDashboards() {
);`; );`;
let YOLO = false; let YOLO = false;
if (YOLO) { if (YOLO) {
await runPgCommand({
command: `CREATE SCHEMA IF NOT EXISTS history;`,
pool: readWritePool,
});
console.log("");
console.log("Set search path");
await runPgCommand({
command: `SET search_path TO ${schemas.join(",")},public;`,
pool: readWritePool,
});
console.log("");
console.log("Create dashboard table and its index"); console.log("Create dashboard table and its index");
await runPgCommand({ await runPgCommand({
command: dropTable("latest", "dashboards"), command: dropTable("dashboards"),
pool: readWritePool, pool: readWritePool,
}); });
@ -234,7 +160,7 @@ async function pgInitializeDashboards() {
}); });
await runPgCommand({ await runPgCommand({
command: createUniqueIndex("latest", "dashboards"), command: createUniqueIndex("dashboards"),
pool: readWritePool, pool: readWritePool,
}); });
console.log(""); console.log("");
@ -245,10 +171,7 @@ async function pgInitializeDashboards() {
} }
} }
let buildHistoryTable = ( let buildHistoryTable = (table: string) => `CREATE TABLE ${table} (
schema: string,
table: string
) => `CREATE TABLE ${schema}.${table} (
id text, id text,
title text, title text,
url text, url text,
@ -263,45 +186,19 @@ let buildHistoryTable = (
export async function pgInitializeHistories() { export async function pgInitializeHistories() {
let YOLO = false; let YOLO = false;
if (YOLO) { if (YOLO) {
console.log("Drop all previous history tables (Danger!)"); console.log("Create history table & index");
await runPgCommand({ await runPgCommand({
command: `DROP SCHEMA history CASCADE;`, command: dropTable("history"),
pool: readWritePool, pool: readWritePool,
}); });
console.log("");
console.log("Create schemas");
for (let schema of schemas) {
await runPgCommand({
command: `CREATE SCHEMA IF NOT EXISTS ${schema}`,
pool: readWritePool,
});
}
console.log("");
console.log("Set search path");
await runPgCommand({ await runPgCommand({
command: `SET search_path TO ${schemas.join(",")},public;`, command: buildHistoryTable("history"),
pool: readWritePool,
});
await runPgCommand({
command: createIndex("history"), // Not unique!!
pool: readWritePool, pool: readWritePool,
}); });
console.log("");
console.log("Create tables & their indexes");
let schema = "history";
for (let table of tableNamesWhiteListHistory) {
await runPgCommand({
command: dropTable(schema, table),
pool: readWritePool,
});
await runPgCommand({
command: buildHistoryTable(schema, table),
pool: readWritePool,
});
await runPgCommand({
command: createIndex(schema, table), // Not unique!!
pool: readWritePool,
});
}
console.log(""); console.log("");
} else { } else {
console.log( console.log(
@ -314,11 +211,11 @@ async function pgInitializeFrontpage() {
let YOLO = false; let YOLO = false;
if (YOLO) { if (YOLO) {
await runPgCommand({ await runPgCommand({
command: dropTable("latest", "frontpage"), command: dropTable("frontpage"),
pool: readWritePool, pool: readWritePool,
}); });
await runPgCommand({ await runPgCommand({
command: `CREATE TABLE latest.frontpage ( command: `CREATE TABLE frontpage (
id serial primary key, id serial primary key,
frontpage_full jsonb, frontpage_full jsonb,
frontpage_sliced jsonb frontpage_sliced jsonb
@ -334,7 +231,7 @@ async function pgInitializeFrontpage() {
export async function pgInitialize() { export async function pgInitialize() {
await pgInitializeScaffolding(); await pgInitializeScaffolding();
await pgInitializeLatest(); await pgInitializeQuestions();
await pgInitializeHistories(); await pgInitializeHistories();
await pgInitializeDashboards(); await pgInitializeDashboards();
await pgInitializeFrontpage(); await pgInitializeFrontpage();
@ -342,64 +239,50 @@ export async function pgInitialize() {
// Read // Read
async function pgReadWithPool({ async function pgReadWithPool({
schema,
tableName, tableName,
pool, pool,
}: { }: {
schema: string;
tableName: string; tableName: string;
pool: Pool; pool: Pool;
}) { }) {
if (tableWhiteList.includes(`${schema}.${tableName}`)) { if (!allTableNames.includes(tableName)) {
let command = `SELECT * from ${schema}.${tableName}`;
let response = await runPgCommand({ command, pool });
let results = response.results;
return results;
} else {
throw Error( throw Error(
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections` `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
); );
} }
let command = `SELECT * from ${tableName}`;
let response = await runPgCommand({ command, pool });
let results = response.results;
return results;
} }
export async function pgRead({ export async function pgRead({ tableName }: { tableName: string }) {
schema, return await pgReadWithPool({ tableName, pool: readWritePool });
tableName,
}: {
schema: string;
tableName: string;
}) {
return await pgReadWithPool({ schema, tableName, pool: readWritePool });
} }
export async function pgReadWithReadCredentials({ export async function pgReadWithReadCredentials({
schema,
tableName, tableName,
}: { }: {
schema: string;
tableName: string; tableName: string;
}) { }) {
// currently does not work. // currently does not work.
/* return await pgReadWithPool({ /* return await pgReadWithPool({
schema,
tableName, tableName,
pool: readOnlyPool, pool: readOnlyPool,
}); });
*/ */
return await pgReadWithPool({ schema, tableName, pool: readWritePool }); return await pgReadWithPool({ tableName, pool: readWritePool });
} }
export async function pgGetByIds({ export async function pgGetByIds({
ids, ids,
schema,
table, table,
}: { }: {
ids: string[]; ids: string[];
schema: string;
table: string; table: string;
}) { }) {
let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3) let idstring = `( ${ids.map((id: string) => `'${id}'`).join(", ")} )`; // (1, 2, 3)
let command = `SELECT * from ${schema}.${table} where id in ${idstring}`; let command = `SELECT * from ${table} where id in ${idstring}`;
// see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn // see: https://stackoverflow.com/questions/5803472/sql-where-id-in-id1-id2-idn
let response = await runPgCommand({ command, pool: readWritePool }); let response = await runPgCommand({ command, pool: readWritePool });
let results = response.results; let results = response.results;
@ -409,23 +292,21 @@ export async function pgGetByIds({
export async function pgBulkInsert({ export async function pgBulkInsert({
data, data,
schema,
tableName, tableName,
client, client,
}: { }: {
data: Forecast[]; data: Forecast[];
schema: string;
tableName: string; tableName: string;
client: PoolClient; client: PoolClient;
}) { }) {
if (!tableWhiteList.includes(`${schema}.${tableName}`)) { if (!forecastTableNames.includes(tableName)) {
throw Error( throw Error(
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections` `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
); );
} }
const generateQuery = (rows: number) => { const generateQuery = (rows: number) => {
let text = `INSERT INTO ${schema}.${tableName} VALUES`; let text = `INSERT INTO ${tableName} VALUES`;
const cols = 10; const cols = 10;
const parts: string[] = []; const parts: string[] = [];
for (let r = 0; r < rows; r++) { for (let r = 0; r < rows; r++) {
@ -478,36 +359,30 @@ export async function pgBulkInsert({
} }
} }
export async function pgInsertIntoDashboard({ datum, schema, tableName }) { export async function pgInsertIntoDashboard({ datum }) {
if (tableWhiteList.includes(`${schema}.${tableName}`)) { let text = `INSERT INTO dashboards VALUES($1, $2, $3, $4, $5, $6, $7)`;
let text = `INSERT INTO ${schema}.${tableName} VALUES($1, $2, $3, $4, $5, $6, $7)`; let timestamp = datum.timestamp || new Date().toISOString();
let timestamp = datum.timestamp || new Date().toISOString(); timestamp = timestamp.slice(0, 19).replace("T", " ");
timestamp = timestamp.slice(0, 19).replace("T", " "); let values = [
let values = [ hash(JSON.stringify(datum.contents)),
hash(JSON.stringify(datum.contents)), datum.title || "",
datum.title || "", datum.description || "",
datum.description || "", JSON.stringify(datum.contents || []),
JSON.stringify(datum.contents || []), timestamp, // fixed
timestamp, // fixed datum.creator || "",
datum.creator || "", JSON.stringify(datum.extra || []),
JSON.stringify(datum.extra || []), ];
]; const client = await readWritePool.connect();
const client = await readWritePool.connect(); let result;
let result; try {
try { result = await client.query(text, values);
result = await client.query(text, values); } catch (error) {
} catch (error) { console.log(error);
console.log(error); } finally {
} finally { client.release();
client.release();
}
// console.log(result)
return result;
} else {
throw Error(
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections`
);
} }
// console.log(result)
return result;
} }
/* For reference /* For reference
id text, id text,
@ -532,16 +407,21 @@ pgInsertIntoDashboard({
], ],
creator: "Nuño Sempere", creator: "Nuño Sempere",
}, },
schema: "latest",
tableName: "dashboards", tableName: "dashboards",
}); });
*/ */
export async function pgUpsert({ contents, schema, tableName }) { export async function pgUpsert({
if (!tableWhiteList.includes(`${schema}.${tableName}`)) { contents,
console.log("tableWhiteList:"); tableName,
console.log(tableWhiteList); replacePlatform,
}: {
contents: Forecast[];
tableName: string;
replacePlatform?: string;
}) {
if (!forecastTableNames.includes(tableName)) {
throw Error( throw Error(
`Table ${schema}.${tableName} not in whitelist; stopping to avoid tricky sql injections` `Table ${tableName} not in whitelist; stopping to avoid tricky sql injections`
); );
} }
@ -549,27 +429,22 @@ export async function pgUpsert({ contents, schema, tableName }) {
const client = await readWritePool.connect(); const client = await readWritePool.connect();
try { try {
await client.query("BEGIN"); await client.query("BEGIN");
if (schema === "latest") { if (replacePlatform) {
client.query(`DELETE FROM latest.${tableName}`); await client.query(`DELETE FROM ${tableName} WHERE platform = $1`, [
replacePlatform,
]);
} }
console.log( console.log(
`Upserting ${contents.length} rows into postgres table ${schema}.${tableName}.` `Upserting ${contents.length} rows into postgres table ${tableName}.`
);
console.log(
`Expected to take ${Number((contents.length * 831.183) / 4422).toFixed(
2
)} seconds or ${Number((contents.length * 13.85305) / 4422).toFixed(
2
)} minutes`
); );
await pgBulkInsert({ data: contents, schema, tableName, client }); await pgBulkInsert({ data: contents, tableName, client });
console.log( console.log(
`Inserted ${ `Inserted ${
contents.length contents.length
} rows with approximate cummulative size ${roughSizeOfObject( } rows with approximate cummulative size ${roughSizeOfObject(
contents contents
)} MB into ${schema}.${tableName}.` )} MB into ${tableName}.`
); );
console.log("Sample: "); console.log("Sample: ");

11
src/backend/flow/history/updateHistory.ts
View File

@ -1,12 +1,9 @@
import { import { pgReadWithReadCredentials, pgUpsert } from "../../database/pg-wrapper";
databaseReadWithReadCredentials,
databaseUpsert,
} from "../../database/database-wrapper";
export async function updateHistory() { export async function updateHistory() {
let latest = await databaseReadWithReadCredentials({ group: "combined" }); let latest = await pgReadWithReadCredentials({ tableName: "questions" });
await databaseUpsert({ await pgUpsert({
contents: latest, contents: latest,
group: "history", tableName: "history",
}); });
} }

8
src/backend/flow/jobs.ts
View File

@ -1,7 +1,6 @@
import { pgInitialize } from "../database/pg-wrapper"; import { pgInitialize } from "../database/pg-wrapper";
import { doEverything } from "../flow/doEverything"; import { doEverything } from "../flow/doEverything";
import { updateHistory } from "../flow/history/updateHistory"; import { updateHistory } from "../flow/history/updateHistory";
import { mergeEverything } from "../flow/mergeEverything";
import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData"; import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData";
import { rebuildFrontpage } from "../frontpage"; import { rebuildFrontpage } from "../frontpage";
import { platforms, processPlatform } from "../platforms"; import { platforms, processPlatform } from "../platforms";
@ -20,13 +19,6 @@ export const jobs: Job[] = [
message: `Download predictions from ${platform.name}`, message: `Download predictions from ${platform.name}`,
run: () => processPlatform(platform), run: () => processPlatform(platform),
})), })),
{
name: "merge",
message:
"Merge tables into one big table (and push the result to a pg database)",
run: mergeEverything,
separate: true,
},
{ {
name: "algolia", name: "algolia",
message: 'Rebuild algolia database ("index")', message: 'Rebuild algolia database ("index")',

28
src/backend/flow/mergeEverything.ts
View File

@ -1,28 +0,0 @@
import { databaseRead, databaseUpsert } from "../database/database-wrapper";
import { platforms } from "../platforms";
/* Merge everything */
export async function mergeEverythingInner() {
let merged = [];
for (let platform of platforms) {
const platformName = platform.name;
let json = await databaseRead({ group: platformName });
console.log(`${platformName} has ${json.length} questions\n`);
merged = merged.concat(json);
}
let mergedprocessed = merged.map((element) => ({
...element,
optionsstringforsearch: element.options
.map((option) => option.name)
.join(", "),
}));
console.log(`In total, there are ${mergedprocessed.length} questions`);
return mergedprocessed;
}
export async function mergeEverything() {
let merged = await mergeEverythingInner();
await databaseUpsert({ contents: merged, group: "combined" });
console.log("Done");
}

32
src/backend/frontpage.ts
View File

@ -1,51 +1,35 @@
import { pgRead, readWritePool } from "./database/pg-wrapper"; import { pgRead, readWritePool } from "./database/pg-wrapper";
import { Forecast } from "./platforms";
export async function getFrontpageRaw() { export async function getFrontpage(): Promise<Forecast[]> {
const client = await readWritePool.connect(); const client = await readWritePool.connect();
const res = await client.query( const res = await client.query(
"SELECT frontpage_sliced FROM latest.frontpage ORDER BY id DESC LIMIT 1" "SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
); );
if (!res.rows.length) return []; if (!res.rows.length) return [];
console.log(res.rows[0].frontpage_sliced); console.log(res.rows[0].frontpage_sliced);
return res.rows[0].frontpage_sliced; return res.rows[0].frontpage_sliced;
} }
export async function getFrontpageFullRaw() { export async function getFrontpageFull(): Promise<Forecast[]> {
const client = await readWritePool.connect(); const client = await readWritePool.connect();
const res = await client.query( const res = await client.query(
"SELECT frontpage_full FROM latest.frontpage ORDER BY id DESC LIMIT 1" "SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
); );
if (!res.rows.length) return []; if (!res.rows.length) return [];
console.log(res.rows[0]); console.log(res.rows[0]);
return res.rows[0].frontpage_full; return res.rows[0].frontpage_full;
} }
export async function getFrontpage() {
let frontPageForecastsCompatibleWithFuse = [];
try {
let data = await getFrontpageRaw();
frontPageForecastsCompatibleWithFuse = data.map((result) => ({
item: result,
score: 0,
}));
return frontPageForecastsCompatibleWithFuse;
} catch (error) {
console.log(error);
} finally {
return frontPageForecastsCompatibleWithFuse;
}
}
export async function rebuildFrontpage() { export async function rebuildFrontpage() {
const frontpageFull = await pgRead({ const frontpageFull = await pgRead({
schema: "latest", tableName: "questions",
tableName: "combined",
}); });
const client = await readWritePool.connect(); const client = await readWritePool.connect();
const frontpageSliced = ( const frontpageSliced = (
await client.query(` await client.query(`
SELECT * FROM latest.combined SELECT * FROM questions
WHERE WHERE
(qualityindicators->>'stars')::int >= 3 (qualityindicators->>'stars')::int >= 3
AND description != '' AND description != ''
@ -56,7 +40,7 @@ export async function rebuildFrontpage() {
const start = Date.now(); const start = Date.now();
await client.query( await client.query(
"INSERT INTO latest.frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)", "INSERT INTO frontpage(frontpage_full, frontpage_sliced) VALUES($1, $2)",
[JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)] [JSON.stringify(frontpageFull), JSON.stringify(frontpageSliced)]
); );

4
src/backend/manual/manualDownload.ts
View File

@ -2,10 +2,10 @@ import "dotenv/config";
import fs from "fs"; import fs from "fs";
import { databaseReadWithReadCredentials } from "../database/database-wrapper"; import { pgReadWithReadCredentials } from "../database/pg-wrapper";
let main = async () => { let main = async () => {
let json = await databaseReadWithReadCredentials({ group: "combined" }); let json = await pgReadWithReadCredentials({ tableName: "questions" });
let string = JSON.stringify(json, null, 2); let string = JSON.stringify(json, null, 2);
let filename = "metaforecasts.json"; let filename = "metaforecasts.json";
fs.writeFileSync(filename, string); fs.writeFileSync(filename, string);

92
src/backend/manual/noSchemaMigrate.ts Normal file
View File

@ -0,0 +1,92 @@
import "dotenv/config";
import { readWritePool } from "../database/pg-wrapper";
const migrate = async () => {
const client = await readWritePool.connect();
const execQuery = async (q: string) => {
console.log(q);
await client.query(q);
};
const platformTitleToName = {
Betfair: "betfair",
FantasySCOTUS: "fantasyscotus",
Foretold: "foretold",
"GiveWell/OpenPhilanthropy": "givewellopenphil",
"Good Judgment": "goodjudgement",
"Good Judgment Open": "goodjudgmentopen",
Infer: "infer",
Kalshi: "kalshi",
"Manifold Markets": "manifold",
Metaculus: "metaculus",
"Peter Wildeford": "wildeford",
PolyMarket: "polymarket",
PredictIt: "predictit",
Rootclaim: "rootclaim",
Smarkets: "smarkets",
"X-risk estimates": "xrisk",
};
try {
await client.query("BEGIN");
const copyTable = async (from: string, to: string) => {
await execQuery(`DROP TABLE IF EXISTS ${to}`);
await execQuery(`CREATE TABLE ${to} (LIKE ${from} INCLUDING ALL)`);
await execQuery(`INSERT INTO ${to} SELECT * FROM ${from}`);
};
await copyTable("latest.dashboards", "dashboards");
await copyTable("latest.combined", "questions");
await copyTable("latest.frontpage", "frontpage");
await copyTable("history.h2022", "history");
for (const [title, name] of Object.entries(platformTitleToName)) {
console.log(`Updating ${title} -> ${name}`);
for (const table of ["questions", "history"]) {
await client.query(
`UPDATE ${table} SET platform=$1 WHERE platform=$2`,
[name, title]
);
}
}
console.log("Fixing GJOpen ids in questions and history");
for (const table of ["questions", "history"]) {
await client.query(
`UPDATE ${table} SET id=REPLACE(id, 'goodjudmentopen-', 'goodjudgmentopen-') WHERE id LIKE 'goodjudmentopen-%'`
);
}
const fixId = (id: string) =>
id.replace("goodjudmentopen-", "goodjudgmentopen-");
console.log(
"Please rebuild frontpage manually - current version includes invalid GJOpen and xrisk ids"
);
const updateDashboards = async () => {
const res = await client.query("SELECT id, contents FROM dashboards");
for (const row of res.rows) {
let { id, contents } = row;
contents = contents.map(fixId);
await client.query(
"UPDATE dashboards SET contents = $1 WHERE id = $2",
[JSON.stringify(contents), id]
);
}
};
console.log("Updating dashboards");
await updateDashboards();
await client.query("COMMIT");
} catch (e) {
await client.query("ROLLBACK");
throw e;
} finally {
client.release();
}
};
migrate();

30
src/backend/platforms/example-fetch.ts → src/backend/platforms/_example.ts
View File

@ -1,11 +1,12 @@
/* Imports */ /* Imports */
import axios from "axios"; import axios from "axios";
import { databaseUpsert } from "../database/database-wrapper";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import { Platform } from "./";
/* Definitions */ /* Definitions */
let endpoint = "https://example.com/"; const platformName = "example";
const endpoint = "https://example.com/";
/* Support functions */ /* Support functions */
@ -23,7 +24,7 @@ async function fetchData() {
async function processPredictions(predictions) { async function processPredictions(predictions) {
let results = await predictions.map((prediction) => { let results = await predictions.map((prediction) => {
let id = `platform-${prediction.id}`; let id = `${platformName}-${prediction.id}`;
let probability = prediction.probability; let probability = prediction.probability;
let options = [ let options = [
{ {
@ -40,12 +41,12 @@ async function processPredictions(predictions) {
let result = { let result = {
title: prediction.title, title: prediction.title,
url: `https://example.com`, url: `https://example.com`,
platform: "Example", platform: platformName,
description: prediction.description, description: prediction.description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Example", { stars: calculateStars(platformName, {
/* some: somex, factors: factors */ /* some: somex, factors: factors */
}), }),
other: prediction.otherx, other: prediction.otherx,
@ -59,12 +60,13 @@ async function processPredictions(predictions) {
/* Body */ /* Body */
export async function example() { export const example: Platform = {
let data = await fetchData(); name: platformName,
let results = await processPredictions(data); // somehow needed label: "Example platform",
// console.log(results) color: "#ff0000",
// let string = JSON.stringify(results, null, 2) async fetcher() {
await databaseUpsert({ contents: results, group: "example" }); let data = await fetchData();
console.log("Done"); let results = await processPredictions(data); // somehow needed
} return results;
//example() },
};

14
src/backend/platforms/betfair.ts
View File

@ -5,6 +5,8 @@ import https from "https";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import { Forecast, Platform } from "./"; import { Forecast, Platform } from "./";
const platformName = "betfair";
/* Definitions */ /* Definitions */
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
@ -82,7 +84,7 @@ async function processPredictions(data) {
/* if(Math.floor(Math.random() * 10) % 20 ==0){ /* if(Math.floor(Math.random() * 10) % 20 ==0){
console.log(JSON.stringify(prediction, null, 4)) console.log(JSON.stringify(prediction, null, 4))
} */ } */
let id = `betfair-${prediction.marketId}`; let id = `${platformName}-${prediction.marketId}`;
let normalizationFactor = prediction.options let normalizationFactor = prediction.options
.filter((option) => option.status == "ACTIVE" && option.totalMatched > 0) .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
.map((option) => option.lastPriceTraded) .map((option) => option.lastPriceTraded)
@ -121,12 +123,14 @@ async function processPredictions(data) {
id: id, id: id,
title: title, title: title,
url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`, url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
platform: "Betfair", platform: platformName,
description: description, description: description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Betfair", { volume: prediction.totalMatched }), stars: calculateStars(platformName, {
volume: prediction.totalMatched,
}),
volume: prediction.totalMatched, volume: prediction.totalMatched,
}, },
}; };
@ -136,7 +140,9 @@ async function processPredictions(data) {
} }
export const betfair: Platform = { export const betfair: Platform = {
name: "betfair", name: platformName,
label: "Betfair",
color: "#3d674a",
async fetcher() { async fetcher() {
const data = await fetchPredictions(); const data = await fetchPredictions();
const results = await processPredictions(data); // somehow needed const results = await processPredictions(data); // somehow needed

18
src/backend/platforms/fantasyscotus.ts
View File

@ -2,7 +2,9 @@
import axios from "axios"; import axios from "axios";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Forecast, Platform } from "./";
const platformName = "fantasyscotus";
/* Definitions */ /* Definitions */
let unixtime = new Date().getTime(); let unixtime = new Date().getTime();
@ -65,19 +67,19 @@ async function processData(data) {
let historicalPercentageCorrect = data.stats.pcnt_correct; let historicalPercentageCorrect = data.stats.pcnt_correct;
let historicalProbabilityCorrect = let historicalProbabilityCorrect =
Number(historicalPercentageCorrect.replace("%", "")) / 100; Number(historicalPercentageCorrect.replace("%", "")) / 100;
let results = []; let results: Forecast[] = [];
for (let event of events) { for (let event of events) {
if (event.accuracy == "") { if (event.accuracy == "") {
let id = `fantasyscotus-${event.id}`; let id = `${platformName}-${event.id}`;
// if the thing hasn't already resolved // if the thing hasn't already resolved
let predictionData = await getPredictionsData(event.docket_url); let predictionData = await getPredictionsData(event.docket_url);
let pAffirm = predictionData.proportionAffirm; let pAffirm = predictionData.proportionAffirm;
//let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect //let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
let eventObject = { let eventObject: Forecast = {
id: id, id: id,
title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`, title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
url: `https://fantasyscotus.net/user-predictions${event.docket_url}`, url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,
platform: "FantasySCOTUS", platform: platformName,
description: `${(pAffirm * 100).toFixed(2)}% (${ description: `${(pAffirm * 100).toFixed(2)}% (${
predictionData.numAffirm predictionData.numAffirm
} out of ${ } out of ${
@ -100,7 +102,7 @@ async function processData(data) {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
numforecasts: Number(predictionData.numForecasts), numforecasts: Number(predictionData.numForecasts),
stars: calculateStars("FantasySCOTUS", {}), stars: calculateStars(platformName, {}),
}, },
}; };
results.push(eventObject); results.push(eventObject);
@ -112,7 +114,9 @@ async function processData(data) {
/* Body */ /* Body */
export const fantasyscotus: Platform = { export const fantasyscotus: Platform = {
name: "fantasyscotus", name: platformName,
label: "FantasySCOTUS",
color: "#231149",
async fetcher() { async fetcher() {
let rawData = await fetchData(); let rawData = await fetchData();
let results = await processData(rawData); let results = await processData(rawData);

13
src/backend/platforms/foretold.ts
View File

@ -5,6 +5,9 @@ import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "foretold";
let graphQLendpoint = "https://api.foretold.io/graphql"; let graphQLendpoint = "https://api.foretold.io/graphql";
let highQualityCommunities = [ let highQualityCommunities = [
"0104d8e8-07e4-464b-8b32-74ef22b49f21", "0104d8e8-07e4-464b-8b32-74ef22b49f21",
@ -54,7 +57,9 @@ async function fetchAllCommunityQuestions(communityId) {
} }
export const foretold: Platform = { export const foretold: Platform = {
name: "foretold", name: platformName,
label: "Foretold",
color: "#62520b",
async fetcher() { async fetcher() {
let results = []; let results = [];
for (let community of highQualityCommunities) { for (let community of highQualityCommunities) {
@ -62,7 +67,7 @@ export const foretold: Platform = {
questions = questions.map((question) => question.node); questions = questions.map((question) => question.node);
questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
questions.forEach((question) => { questions.forEach((question) => {
let id = `foretold-${question.id}`; let id = `${platformName}-${question.id}`;
let options = []; let options = [];
if (question.valueType == "PERCENTAGE") { if (question.valueType == "PERCENTAGE") {
let probability = question.previousAggregate.value.percentage; let probability = question.previousAggregate.value.percentage;
@ -83,13 +88,13 @@ export const foretold: Platform = {
id: id, id: id,
title: question.name, title: question.name,
url: `https://www.foretold.io/c/${community}/m/${question.id}`, url: `https://www.foretold.io/c/${community}/m/${question.id}`,
platform: "Foretold", platform: platformName,
description: "", description: "",
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
numforecasts: Math.floor(Number(question.measurementCount) / 2), numforecasts: Math.floor(Number(question.measurementCount) / 2),
stars: calculateStars("Foretold", {}), stars: calculateStars(platformName, {}),
}, },
/*liquidity: liquidity.toFixed(2), /*liquidity: liquidity.toFixed(2),
tradevolume: tradevolume.toFixed(2), tradevolume: tradevolume.toFixed(2),

20
src/backend/platforms/givewellopenphil.ts
View File

@ -2,10 +2,11 @@
import axios from "axios"; import axios from "axios";
import fs from "fs"; import fs from "fs";
import { databaseUpsert } from "../database/database-wrapper";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Platform } from "./";
const platformName = "givewellopenphil";
/* Support functions */ /* Support functions */
async function fetchPage(url: string) { async function fetchPage(url: string) {
let response = await axios({ let response = await axios({
@ -49,24 +50,26 @@ async function main1() {
let result = { let result = {
title: title, title: title,
url: url, url: url,
platform: "GiveWell", platform: platformName,
description: description, description: description,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("GiveWell/OpenPhilanthropy", {}), stars: calculateStars(platformName, {}),
}, },
}; // Note: This requires some processing afterwards }; // Note: This requires some processing afterwards
// console.log(result) // console.log(result)
results.push(result); results.push(result);
} }
await databaseUpsert({ // await databaseUpsert({
contents: results, // contents: results,
group: "givewell-questions-unprocessed", // group: "givewell-questions-unprocessed",
}); // });
} }
export const givewellopenphil: Platform = { export const givewellopenphil: Platform = {
name: "givewellopenphil", name: platformName,
label: "GiveWell/OpenPhilanthropy",
color: "#32407e",
async fetcher() { async fetcher() {
// main1() // main1()
return; // not necessary to refill the DB every time return; // not necessary to refill the DB every time
@ -76,6 +79,7 @@ export const givewellopenphil: Platform = {
const data = JSON.parse(rawdata); const data = JSON.parse(rawdata);
const dataWithDate = data.map((datum: any) => ({ const dataWithDate = data.map((datum: any) => ({
...datum, ...datum,
platform: platformName,
timestamp: "2021-02-23", timestamp: "2021-02-23",
})); }));
return dataWithDate; return dataWithDate;

11
src/backend/platforms/goodjudgment.ts
View File

@ -8,6 +8,7 @@ import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "goodjudgment";
let endpoint = "https://goodjudgment.io/superforecasts/"; let endpoint = "https://goodjudgment.io/superforecasts/";
String.prototype.replaceAll = function replaceAll(search, replace) { String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace); return this.split(search).join(replace);
@ -15,7 +16,9 @@ String.prototype.replaceAll = function replaceAll(search, replace) {
/* Body */ /* Body */
export const goodjudgment: Platform = { export const goodjudgment: Platform = {
name: "goodjudgment", name: platformName,
label: "Good Judgment",
color: "#7d4f1b",
async fetcher() { async fetcher() {
// Proxy fuckery // Proxy fuckery
let proxy; let proxy;
@ -64,7 +67,7 @@ export const goodjudgment: Platform = {
let title = table[0]["0"].split("\t\t\t").splice(3)[0]; let title = table[0]["0"].split("\t\t\t").splice(3)[0];
if (title != undefined) { if (title != undefined) {
title = title.replaceAll("</a>", ""); title = title.replaceAll("</a>", "");
let id = `goodjudgment-${hash(title)}`; let id = `${platformName}-${hash(title)}`;
let description = table let description = table
.filter((row) => row["0"].includes("BACKGROUND:")) .filter((row) => row["0"].includes("BACKGROUND:"))
.map((row) => row["0"]) .map((row) => row["0"])
@ -101,12 +104,12 @@ export const goodjudgment: Platform = {
id: id, id: id,
title: title, title: title,
url: endpoint, url: endpoint,
platform: "Good Judgment", platform: platformName,
description: description, description: description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Good Judgment", {}), stars: calculateStars(platformName, {}),
}, },
extra: { extra: {
superforecastercommentary: analysis || "", superforecastercommentary: analysis || "",

12
src/backend/platforms/goodjudmentopen.ts → src/backend/platforms/goodjudgmentopen.ts
View File

@ -8,6 +8,8 @@ import toMarkdown from "../utils/toMarkdown";
import { Platform } from "./"; import { Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "goodjudgmentopen";
let htmlEndPoint = "https://www.gjopen.com/questions?page="; let htmlEndPoint = "https://www.gjopen.com/questions?page=";
let annoyingPromptUrls = [ let annoyingPromptUrls = [
"https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
@ -185,12 +187,12 @@ async function goodjudgmentopen_inner(cookie) {
} }
let questionNumRegex = new RegExp("questions/([0-9]+)"); let questionNumRegex = new RegExp("questions/([0-9]+)");
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
let id = `goodjudmentopen-${questionNum}`; let id = `${platformName}-${questionNum}`;
let question = { let question = {
id: id, id: id,
title: title, title: title,
url: url, url: url,
platform: "Good Judgment Open", platform: platformName,
...moreinfo, ...moreinfo,
}; };
if (j % 30 == 0 || DEBUG_MODE == "on") { if (j % 30 == 0 || DEBUG_MODE == "on") {
@ -236,8 +238,10 @@ async function goodjudgmentopen_inner(cookie) {
return results; return results;
} }
export const goodjudmentopen: Platform = { export const goodjudgmentopen: Platform = {
name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen` name: platformName,
label: "Good Judgment Open",
color: "#002455",
async fetcher() { async fetcher() {
let cookie = process.env.GOODJUDGMENTOPENCOOKIE; let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
return await applyIfSecretExists(cookie, goodjudgmentopen_inner); return await applyIfSecretExists(cookie, goodjudgmentopen_inner);

20
src/backend/platforms/index.ts
View File

@ -1,13 +1,13 @@
import { databaseUpsert } from "../database/database-wrapper"; import { pgUpsert } from "../database/pg-wrapper";
import { betfair } from "./betfair"; import { betfair } from "./betfair";
import { fantasyscotus } from "./fantasyscotus"; import { fantasyscotus } from "./fantasyscotus";
import { foretold } from "./foretold"; import { foretold } from "./foretold";
import { givewellopenphil } from "./givewellopenphil"; import { givewellopenphil } from "./givewellopenphil";
import { goodjudgment } from "./goodjudgment"; import { goodjudgment } from "./goodjudgment";
import { goodjudmentopen } from "./goodjudmentopen"; import { goodjudgmentopen } from "./goodjudgmentopen";
import { infer } from "./infer"; import { infer } from "./infer";
import { kalshi } from "./kalshi"; import { kalshi } from "./kalshi";
import { manifoldmarkets } from "./manifoldmarkets"; import { manifold } from "./manifold";
import { metaculus } from "./metaculus"; import { metaculus } from "./metaculus";
import { polymarket } from "./polymarket"; import { polymarket } from "./polymarket";
import { predictit } from "./predictit"; import { predictit } from "./predictit";
@ -67,7 +67,9 @@ export interface Forecast {
export type PlatformFetcher = () => Promise<Forecast[] | null>; export type PlatformFetcher = () => Promise<Forecast[] | null>;
export interface Platform { export interface Platform {
name: string; name: string; // short name for ids and `platform` db column, e.g. "xrisk"
label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates"
color: string; // used on frontend
fetcher?: PlatformFetcher; fetcher?: PlatformFetcher;
} }
@ -92,10 +94,10 @@ export const platforms: Platform[] = [
foretold, foretold,
givewellopenphil, givewellopenphil,
goodjudgment, goodjudgment,
goodjudmentopen, goodjudgmentopen,
infer, infer,
kalshi, kalshi,
manifoldmarkets, manifold,
metaculus, metaculus,
polymarket, polymarket,
predictit, predictit,
@ -112,7 +114,11 @@ export const processPlatform = async (platform: Platform) => {
} }
let results = await platform.fetcher(); let results = await platform.fetcher();
if (results && results.length) { if (results && results.length) {
await databaseUpsert({ contents: results, group: platform.name }); await pgUpsert({
contents: results,
tableName: "questions",
replacePlatform: platform.name,
});
console.log("Done"); console.log("Done");
} else { } else {
console.log(`Platform ${platform.name} didn't return any results`); console.log(`Platform ${platform.name} didn't return any results`);

144
src/backend/platforms/infer.ts
View File

@ -3,11 +3,13 @@ import axios from "axios";
import { Tabletojson } from "tabletojson"; import { Tabletojson } from "tabletojson";
import { applyIfSecretExists } from "../utils/getSecrets"; import { applyIfSecretExists } from "../utils/getSecrets";
import { measureTime } from "../utils/measureTime";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown"; import toMarkdown from "../utils/toMarkdown";
import { Forecast, Platform } from "./"; import { Forecast, Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "infer";
let htmlEndPoint = "https://www.infer-pub.com/questions"; let htmlEndPoint = "https://www.infer-pub.com/questions";
String.prototype.replaceAll = function replaceAll(search, replace) { String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace); return this.split(search).join(replace);
@ -145,7 +147,7 @@ async function fetchStats(questionUrl, cookie) {
qualityindicators: { qualityindicators: {
numforecasts: Number(numforecasts), numforecasts: Number(numforecasts),
numforecasters: Number(numforecasters), numforecasters: Number(numforecasters),
stars: calculateStars("Infer", { numforecasts }), stars: calculateStars(platformName, { numforecasts }),
}, },
}; };
@ -179,96 +181,84 @@ function sleep(ms) {
/* Body */ /* Body */
async function infer_inner(cookie) { async function infer_inner(cookie: string) {
let i = 1; let i = 1;
let response = await fetchPage(i, cookie); let response = await fetchPage(i, cookie);
let results: Forecast[] = []; let results: Forecast[] = [];
let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n");
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.infer-pub.com/questions/")
);
// console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { await measureTime(async () => {
//console.log(response) // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
console.log("questionHrefs: "); while (!isEnd(response) && isSignedIn(response)) {
console.log(questionHrefs); let htmlLines = response.split("\n");
} // let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.infer-pub.com/questions/")
);
// console.log(questionHrefs)
//console.log("") if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log("") console.log("questionHrefs: ");
//console.log(h4elements) console.log(questionHrefs);
}
for (let questionHref of questionHrefs) { for (let questionHref of questionHrefs) {
//console.log(h4element) let elementSplit = questionHref.split('"><span>');
let url = elementSplit[0].split('<a href="')[1];
let title = elementSplit[1]
.replace("</h4>", "")
.replace("</h5>", "")
.replace("</span></a>", "");
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
let elementSplit = questionHref.split('"><span>'); try {
let url = elementSplit[0].split('<a href="')[1]; let moreinfo = await fetchStats(url, cookie);
let title = elementSplit[1] let questionNumRegex = new RegExp("questions/([0-9]+)");
.replace("</h4>", "") let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
.replace("</h5>", "") let id = `${platformName}-${questionNum}`;
.replace("</span></a>", ""); let question: Forecast = {
id: id,
title: title,
url: url,
platform: platformName,
...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(`Page #${i}`);
console.log(question);
}
results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url);
console.log(question);
}
} catch (error) {
console.log(error);
console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
}
}
i++;
console.log(
"Sleeping for ~5secs so as to not be as noticeable to the infer servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try { try {
let moreinfo = await fetchStats(url, cookie); response = await fetchPage(i, cookie);
let questionNumRegex = new RegExp("questions/([0-9]+)");
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
let id = `infer-${questionNum}`;
let question = {
id: id,
title: title,
url: url,
platform: "Infer",
...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(`Page #${i}`);
console.log(question);
}
results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url);
console.log(question);
}
} catch (error) { } catch (error) {
console.log(error); console.log(error);
console.log( console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json` `The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
); );
} }
} }
});
i++;
//i=Number(i)+1
console.log(
"Sleeping for ~5secs so as to not be as noticeable to the infer servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
response = await fetchPage(i, cookie);
} catch (error) {
console.log(error);
console.log(
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
);
}
}
let end = Date.now();
let difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
if (results.length === 0) { if (results.length === 0) {
console.log("Not updating results, as process was not signed in"); console.log("Not updating results, as process was not signed in");
@ -278,7 +268,9 @@ async function infer_inner(cookie) {
} }
export const infer: Platform = { export const infer: Platform = {
name: "infer", name: platformName,
label: "Infer",
color: "#223900",
async fetcher() { async fetcher() {
let cookie = process.env.INFER_COOKIE; let cookie = process.env.INFER_COOKIE;
return await applyIfSecretExists(cookie, infer_inner); return await applyIfSecretExists(cookie, infer_inner);

11
src/backend/platforms/kalshi.ts
View File

@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "kalshi";
let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
async function fetchAllMarkets() { async function fetchAllMarkets() {
@ -34,17 +35,17 @@ async function processMarkets(markets) {
type: "PROBABILITY", type: "PROBABILITY",
}, },
]; ];
let id = `kalshi-${market.id}`; let id = `${platformName}-${market.id}`;
let result = { let result = {
id: id, id: id,
title: market.title.replaceAll("*", ""), title: market.title.replaceAll("*", ""),
url: `https://kalshi.com/markets/${market.ticker_name}`, url: `https://kalshi.com/markets/${market.ticker_name}`,
platform: "Kalshi", platform: platformName,
description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`, description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Kalshi", { stars: calculateStars(platformName, {
shares_volume: market.volume, shares_volume: market.volume,
interest: market.open_interest, interest: market.open_interest,
}), }),
@ -70,7 +71,9 @@ async function processMarkets(markets) {
} }
export const kalshi: Platform = { export const kalshi: Platform = {
name: "kalshi", name: platformName,
label: "Kalshi",
color: "#615691",
fetcher: async function () { fetcher: async function () {
let markets = await fetchAllMarkets(); let markets = await fetchAllMarkets();
return await processMarkets(markets); return await processMarkets(markets);

29
src/backend/platforms/manifoldmarkets.ts → src/backend/platforms/manifold.ts
View File

@ -2,9 +2,10 @@
import axios from "axios"; import axios from "axios";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Forecast, Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "manifold";
let endpoint = "https://manifold.markets/api/v0/markets"; let endpoint = "https://manifold.markets/api/v0/markets";
// See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5 // See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5
@ -22,7 +23,7 @@ async function fetchData() {
return response; return response;
} }
function showStatistics(results) { function showStatistics(results: Forecast[]) {
console.log(`Num unresolved markets: ${results.length}`); console.log(`Num unresolved markets: ${results.length}`);
let sum = (arr) => arr.reduce((tally, a) => tally + a, 0); let sum = (arr) => arr.reduce((tally, a) => tally + a, 0);
let num2StarsOrMore = results.filter( let num2StarsOrMore = results.filter(
@ -43,8 +44,8 @@ function showStatistics(results) {
} }
async function processPredictions(predictions) { async function processPredictions(predictions) {
let results = await predictions.map((prediction) => { let results: Forecast[] = await predictions.map((prediction) => {
let id = `manifold-${prediction.id}`; let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name
let probability = prediction.probability; let probability = prediction.probability;
let options = [ let options = [
{ {
@ -58,16 +59,16 @@ async function processPredictions(predictions) {
type: "PROBABILITY", type: "PROBABILITY",
}, },
]; ];
let result = { const result: Forecast = {
id: id, id: id,
title: prediction.question, title: prediction.question,
url: prediction.url, url: prediction.url,
platform: "Manifold Markets", platform: platformName,
description: prediction.description, description: prediction.description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Manifold Markets", { stars: calculateStars(platformName, {
volume7Days: prediction.volume7Days, volume7Days: prediction.volume7Days,
volume24Hours: prediction.volume24Hours, volume24Hours: prediction.volume24Hours,
pool: prediction.pool, pool: prediction.pool,
@ -83,13 +84,17 @@ async function processPredictions(predictions) {
}; };
return result; return result;
}); });
let unresolvedResults = results.filter((result) => !result.extra.isResolved);
// console.log(unresolvedResults); const unresolvedResults = results.filter(
return unresolvedResults; //resultsProcessed (result) => !result.extra.isResolved
);
return unresolvedResults;
} }
export const manifoldmarkets: Platform = { export const manifold: Platform = {
name: "manifoldmarkets", name: platformName,
label: "Manifold Markets",
color: "#793466",
async fetcher() { async fetcher() {
let data = await fetchData(); let data = await fetchData();
let results = await processPredictions(data); // somehow needed let results = await processPredictions(data); // somehow needed

13
src/backend/platforms/metaculus.ts
View File

@ -6,6 +6,7 @@ import toMarkdown from "../utils/toMarkdown";
import { Platform } from "./"; import { Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "metaculus";
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
let now = new Date().toISOString(); let now = new Date().toISOString();
let DEBUG_MODE = "off"; let DEBUG_MODE = "off";
@ -49,7 +50,7 @@ async function fetchMetaculusQuestions(next) {
return data; return data;
} }
function sleep(ms) { function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
@ -94,7 +95,9 @@ async function fetchMetaculusQuestionDescription(slug) {
} }
export const metaculus: Platform = { export const metaculus: Platform = {
name: "metaculus", name: platformName,
label: "Metaculus",
color: "#006669",
async fetcher() { async fetcher() {
// let metaculusQuestionsInit = await fetchMetaculusQuestions(1) // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
@ -144,18 +147,18 @@ export const metaculus: Platform = {
}, },
]; ];
} }
let id = `metaculus-${result.id}`; let id = `${platformName}-${result.id}`;
let interestingInfo = { let interestingInfo = {
id: id, id: id,
title: result.title, title: result.title,
url: "https://www.metaculus.com" + result.page_url, url: "https://www.metaculus.com" + result.page_url,
platform: "Metaculus", platform: platformName,
description: description, description: description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
numforecasts: Number(result.number_of_predictions), numforecasts: Number(result.number_of_predictions),
stars: calculateStars("Metaculus", { stars: calculateStars(platformName, {
numforecasts: result.number_of_predictions, numforecasts: result.number_of_predictions,
}), }),
}, },

11
src/backend/platforms/polymarket.ts
View File

@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
import { Forecast, Platform } from "./"; import { Forecast, Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "polymarket";
let graphQLendpoint = let graphQLendpoint =
"https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
let units = 10 ** 6; let units = 10 ** 6;
@ -63,7 +64,9 @@ async function fetchIndividualContractData(marketMakerAddress) {
} }
export const polymarket: Platform = { export const polymarket: Platform = {
name: "polymarket", name: platformName,
label: "PolyMarket",
color: "#00314e",
async fetcher() { async fetcher() {
let results: Forecast[] = []; let results: Forecast[] = [];
let webpageEndpointData = await fetchAllContractInfo(); let webpageEndpointData = await fetchAllContractInfo();
@ -79,7 +82,7 @@ export const polymarket: Platform = {
); );
if (moreMarketAnswer.length > 0) { if (moreMarketAnswer.length > 0) {
let moreMarketInfo = moreMarketAnswer[0]; let moreMarketInfo = moreMarketAnswer[0];
let id = `polymarket-${addressLowerCase.slice(0, 10)}`; let id = `${platformName}-${addressLowerCase.slice(0, 10)}`;
// console.log(id); // console.log(id);
let numforecasts = Number(moreMarketInfo.tradesQuantity); let numforecasts = Number(moreMarketInfo.tradesQuantity);
let tradevolume = let tradevolume =
@ -103,7 +106,7 @@ export const polymarket: Platform = {
id: id, id: id,
title: marketInfo.question, title: marketInfo.question,
url: "https://polymarket.com/market/" + marketInfo.slug, url: "https://polymarket.com/market/" + marketInfo.slug,
platform: "PolyMarket", platform: platformName,
description: marketInfo.description, description: marketInfo.description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
@ -111,7 +114,7 @@ export const polymarket: Platform = {
numforecasts: numforecasts.toFixed(0), numforecasts: numforecasts.toFixed(0),
liquidity: liquidity.toFixed(2), liquidity: liquidity.toFixed(2),
tradevolume: tradevolume.toFixed(2), tradevolume: tradevolume.toFixed(2),
stars: calculateStars("Polymarket", { stars: calculateStars(platformName, {
liquidity, liquidity,
option: options[0], option: options[0],
volume: tradevolume, volume: tradevolume,

13
src/backend/platforms/predictit.ts
View File

@ -1,10 +1,11 @@
/* Imports */
import axios from "axios"; import axios from "axios";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown"; import toMarkdown from "../utils/toMarkdown";
import { Platform } from "./"; import { Platform } from "./";
const platformName = "predictit";
/* Support functions */ /* Support functions */
async function fetchmarkets() { async function fetchmarkets() {
let response = await axios({ let response = await axios({
@ -39,7 +40,9 @@ function sleep(ms: number) {
/* Body */ /* Body */
export const predictit: Platform = { export const predictit: Platform = {
name: "predictit", name: platformName,
label: "PredictIt",
color: "#460c00",
async fetcher() { async fetcher() {
let markets = await fetchmarkets(); let markets = await fetchmarkets();
let marketVolumes = await fetchmarketvolumes(); let marketVolumes = await fetchmarketvolumes();
@ -53,7 +56,7 @@ export const predictit: Platform = {
let results = []; let results = [];
for (let market of markets) { for (let market of markets) {
// console.log(market.name) // console.log(market.name)
let id = `predictit-${market.id}`; let id = `${platformName}-${market.id}`;
let isbinary = market.contracts.length == 1; let isbinary = market.contracts.length == 1;
await sleep(3000 * (1 + Math.random())); await sleep(3000 * (1 + Math.random()));
let descriptionraw = await fetchmarketrules(market.id); let descriptionraw = await fetchmarketrules(market.id);
@ -97,12 +100,12 @@ export const predictit: Platform = {
id: id, id: id,
title: market["name"], title: market["name"],
url: market.url, url: market.url,
platform: "PredictIt", platform: platformName,
description: description, description: description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("PredictIt", {}), stars: calculateStars(platformName, {}),
shares_volume: shares_volume, shares_volume: shares_volume,
}, },
}; };

18
src/backend/platforms/rootclaim.ts
View File

@ -1,11 +1,11 @@
/* Imports */
import axios from "axios"; import axios from "axios";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown"; import toMarkdown from "../utils/toMarkdown";
import { Platform } from "./"; import { Forecast, Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "rootclaim";
let jsonEndpoint = let jsonEndpoint =
"https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
@ -24,12 +24,14 @@ async function fetchAllRootclaims() {
} }
export const rootclaim: Platform = { export const rootclaim: Platform = {
name: "rootclaim", name: platformName,
label: "Rootclaim",
color: "#0d1624",
async fetcher() { async fetcher() {
let claims = await fetchAllRootclaims(); let claims = await fetchAllRootclaims();
let results = []; let results: Forecast[] = [];
for (let claim of claims) { for (let claim of claims) {
let id = `rootclaim-${claim.slug.toLowerCase()}`; let id = `${platformName}-${claim.slug.toLowerCase()}`;
let options = []; let options = [];
for (let scenario of claim.scenarios) { for (let scenario of claim.scenarios) {
//console.log(scenario) //console.log(scenario)
@ -42,17 +44,17 @@ export const rootclaim: Platform = {
}); });
} }
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"; let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
let obj = { let obj: Forecast = {
id: id, id: id,
title: toMarkdown(claim.question).replace("\n", ""), title: toMarkdown(claim.question).replace("\n", ""),
url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
platform: "Rootclaim", platform: platformName,
description: toMarkdown(claim.background).replace("&#39;", "'"), description: toMarkdown(claim.background).replace("&#39;", "'"),
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
numforecasts: 1, numforecasts: 1,
stars: calculateStars("Rootclaim", {}), stars: calculateStars(platformName, {}),
}, },
}; };
results.push(obj); results.push(obj);

17
src/backend/platforms/smarkets.ts
View File

@ -1,13 +1,14 @@
/* Imports */
import axios from "axios"; import axios from "axios";
import { calculateStars } from "../utils/stars"; import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Forecast, Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "smarkets";
let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
let VERBOSE = false; let VERBOSE = false;
let empty = () => 0; let empty = () => 0;
/* Support functions */ /* Support functions */
async function fetchEvents(url) { async function fetchEvents(url) {
@ -60,7 +61,9 @@ async function fetchPrices(marketid) {
} }
export const smarkets: Platform = { export const smarkets: Platform = {
name: "smarkets", name: platformName,
label: "Smarkets",
color: "#6f5b41",
async fetcher() { async fetcher() {
let htmlPath = let htmlPath =
"?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
@ -93,7 +96,7 @@ export const smarkets: Platform = {
for (let market of markets) { for (let market of markets) {
VERBOSE ? console.log("================") : empty(); VERBOSE ? console.log("================") : empty();
VERBOSE ? console.log("Market: ", market) : empty(); VERBOSE ? console.log("Market: ", market) : empty();
let id = `smarkets-${market.id}`; let id = `${platformName}-${market.id}`;
let name = market.name; let name = market.name;
let contracts = await fetchContracts(market.id); let contracts = await fetchContracts(market.id);
@ -156,16 +159,16 @@ export const smarkets: Platform = {
name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`)
} }
*/ */
let result = { let result: Forecast = {
id: id, id: id,
title: name, title: name,
url: "https://smarkets.com/event/" + market.event_id + market.slug, url: "https://smarkets.com/event/" + market.event_id + market.slug,
platform: "Smarkets", platform: platformName,
description: market.description, description: market.description,
options: options, options: options,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Smarkets", {}), stars: calculateStars(platformName, {}),
}, },
}; };
VERBOSE ? console.log(result) : empty(); VERBOSE ? console.log(result) : empty();

11
src/backend/platforms/wildeford.ts
View File

@ -7,6 +7,7 @@ import { calculateStars } from "../utils/stars";
import { Platform } from "./"; import { Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "wildeford";
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`; const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`;
// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0 // https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0
@ -73,7 +74,7 @@ async function processPredictions(predictions) {
); );
let results = currentPredictions.map((prediction) => { let results = currentPredictions.map((prediction) => {
let title = prediction["Prediction"].replace(" [update]", ""); let title = prediction["Prediction"].replace(" [update]", "");
let id = `wildeford-${hash(title)}`; let id = `${platformName}-${hash(title)}`;
let probability = Number(prediction["Odds"].replace("%", "")) / 100; let probability = Number(prediction["Odds"].replace("%", "")) / 100;
let options = [ let options = [
{ {
@ -91,14 +92,14 @@ async function processPredictions(predictions) {
id: id, id: id,
title: title, title: title,
url: prediction["url"], url: prediction["url"],
platform: "Peter Wildeford", platform: platformName,
description: prediction["Notes"] || "", description: prediction["Notes"] || "",
options: options, options: options,
timestamp: new Date( timestamp: new Date(
Date.parse(prediction["Prediction Date"] + "Z") Date.parse(prediction["Prediction Date"] + "Z")
).toISOString(), ).toISOString(),
qualityindicators: { qualityindicators: {
stars: calculateStars("Peter Wildeford", null), stars: calculateStars(platformName, null),
}, },
}; };
return result; return result;
@ -120,7 +121,9 @@ export async function wildeford_inner(google_api_key) {
} }
export const wildeford: Platform = { export const wildeford: Platform = {
name: "wildeford", name: platformName,
label: "Peter Wildeford",
color: "#984158",
async fetcher() { async fetcher() {
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);

14
src/backend/platforms/xrisk.ts
View File

@ -1,15 +1,25 @@
import fs from "fs"; import fs from "fs";
import { hash } from "../utils/hash";
import { Platform } from "./"; import { Platform } from "./";
const platformName = "xrisk";
export const xrisk: Platform = { export const xrisk: Platform = {
name: "xrisk", name: "xrisk",
label: "X-risk estimates",
color: "#272600",
async fetcher() { async fetcher() {
return; // not necessary to refill the DB every time // return; // not necessary to refill the DB every time
let fileRaw = fs.readFileSync("./input/xrisk-questions.json", { let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {
encoding: "utf-8", encoding: "utf-8",
}); });
const results = JSON.parse(fileRaw); let results = JSON.parse(fileRaw);
results = results.map((item) => ({
...item,
id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
platform: platformName,
}));
return results; return results;
}, },
}; };

36
src/backend/utils/algolia.ts
View File

@ -1,34 +1,13 @@
import algoliasearch from "algoliasearch"; import algoliasearch from "algoliasearch";
import { databaseReadWithReadCredentials } from "../database/database-wrapper"; import { pgReadWithReadCredentials } from "../database/pg-wrapper";
import { mergeEverythingInner } from "../flow/mergeEverything"; import { platforms } from "../platforms";
let cookie = process.env.ALGOLIA_MASTER_API_KEY; let cookie = process.env.ALGOLIA_MASTER_API_KEY;
const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID; const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
const client = algoliasearch(algoliaAppId, cookie); const client = algoliasearch(algoliaAppId, cookie);
const index = client.initIndex("metaforecast"); const index = client.initIndex("metaforecast");
export async function rebuildAlgoliaDatabaseTheHardWay() {
console.log("Doing this the hard way");
let records = await mergeEverythingInner();
records = records.map((record, index: number) => ({
...record,
has_numforecasts: record.numforecasts ? true : false,
objectID: index,
}));
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
if (index.exists()) {
console.log("Index exists");
index
.replaceAllObjects(records, { safe: true })
.catch((error) => console.log(error));
console.log(
`Pushed ${records.length} records. Algolia will update asynchronously`
);
}
}
let getoptionsstringforsearch = (record: any) => { let getoptionsstringforsearch = (record: any) => {
let result = ""; let result = "";
if (!!record.options && record.options.length > 0) { if (!!record.options && record.options.length > 0) {
@ -41,12 +20,17 @@ let getoptionsstringforsearch = (record: any) => {
}; };
export async function rebuildAlgoliaDatabaseTheEasyWay() { export async function rebuildAlgoliaDatabaseTheEasyWay() {
let records: any[] = await databaseReadWithReadCredentials({ let records: any[] = await pgReadWithReadCredentials({
group: "combined", tableName: "questions",
}); });
const platformNameToLabel = Object.fromEntries(
platforms.map((platform) => [platform.name, platform.label])
);
records = records.map((record, index: number) => ({ records = records.map((record, index: number) => ({
...record, ...record,
platformLabel: platformNameToLabel[record.platform] || record.platform,
has_numforecasts: record.numforecasts ? true : false, has_numforecasts: record.numforecasts ? true : false,
objectID: index, objectID: index,
optionsstringforsearch: getoptionsstringforsearch(record), optionsstringforsearch: getoptionsstringforsearch(record),
@ -62,4 +46,4 @@ export async function rebuildAlgoliaDatabaseTheEasyWay() {
} }
} }
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay;

4
src/backend/utils/evaluations/pullForecastsToCSVForRating.ts
View File

@ -1,7 +1,7 @@
/* Imports */ /* Imports */
import fs from "fs"; import fs from "fs";
import { databaseReadWithReadCredentials } from "../../database/database-wrapper"; import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
/* Definitions */ /* Definitions */
@ -24,7 +24,7 @@ let main = async () => {
"PredictIt", "PredictIt",
"Rootclaim", "Rootclaim",
]; ];
let json = await databaseReadWithReadCredentials({ group: "combined" }); let json = await pgReadWithReadCredentials({ tableName: "questions" });
console.log(json.length); console.log(json.length);
//let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))] //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
//console.log(uniquePlatforms) //console.log(uniquePlatforms)

4
src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts
View File

@ -1,7 +1,7 @@
/* Imports */ /* Imports */
import fs from "fs"; import fs from "fs";
import { databaseReadWithReadCredentials } from "../../database/database-wrapper"; import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
/* Definitions */ /* Definitions */
@ -26,7 +26,7 @@ let shuffleArray = (array) => {
let main = async () => { let main = async () => {
let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim'] let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim']
let json = await databaseReadWithReadCredentials({ group: "combined" }); let json = await pgReadWithReadCredentials({ tableName: "questions" });
console.log(json.length); console.log(json.length);
//let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))] //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
//console.log(uniquePlatforms) //console.log(uniquePlatforms)

6
src/backend/utils/getSecrets.ts
View File

@ -1,10 +1,10 @@
export async function applyIfSecretExists<T>( export async function applyIfSecretExists<T>(
cookie, cookie: string,
fun: (...args: any[]) => T fun: (cookie: string) => T
) { ) {
if (cookie) { if (cookie) {
return await fun(cookie); return await fun(cookie);
} else if (!cookie) { } else {
console.log( console.log(
`Cannot proceed with ${fun.name} because cookie does not exist` `Cannot proceed with ${fun.name} because cookie does not exist`
); );

4
src/backend/utils/misc/process-forecasts-into-elicit.ts
View File

@ -1,7 +1,7 @@
/* Imports */ /* Imports */
import fs from "fs"; import fs from "fs";
import { databaseReadWithReadCredentials } from "../../database/database-wrapper"; import { pgReadWithReadCredentials } from "../../database/pg-wrapper";
/* Definitions */ /* Definitions */
let locationData = "./data/"; let locationData = "./data/";
@ -9,7 +9,7 @@ let locationData = "./data/";
/* Body */ /* Body */
// let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src // let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src
async function main() { async function main() {
let data = await databaseReadWithReadCredentials({ group: "combined" }); //JSON.parse(rawdata) let data = await pgReadWithReadCredentials({ tableName: "questions" }); //JSON.parse(rawdata)
let processDescription = (description) => { let processDescription = (description) => {
if (description == null || description == undefined || description == "") { if (description == null || description == undefined || description == "") {
return ""; return "";

94
src/backend/utils/stars.ts
View File

@ -105,7 +105,7 @@ function calculateStarsGiveWellOpenPhil(data) {
return starsInteger; return starsInteger;
} }
function calculateStarsGoodJudment(data) { function calculateStarsGoodJudgment(data) {
let nuno = (data) => 4; let nuno = (data) => 4;
let eli = (data) => 4; let eli = (data) => 4;
let misha = (data) => 3.5; let misha = (data) => 3.5;
@ -114,7 +114,7 @@ function calculateStarsGoodJudment(data) {
return starsInteger; return starsInteger;
} }
function calculateStarsGoodJudmentOpen(data) { function calculateStarsGoodJudgmentOpen(data) {
let nuno = (data) => (data.numforecasts > 100 ? 3 : 2); let nuno = (data) => (data.numforecasts > 100 ? 3 : 2);
let eli = (data) => 3; let eli = (data) => 3;
let misha = (data) => let misha = (data) =>
@ -173,7 +173,7 @@ function calculateStarsLadbrokes(data) {
return starsInteger; return starsInteger;
} }
function calculateStarsManifoldMarkets(data) { function calculateStarsManifold(data) {
let nuno = (data) => let nuno = (data) =>
data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100) data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100)
? 2 ? 2
@ -268,15 +268,56 @@ function calculateStarsWilliamHill(data) {
return starsInteger; return starsInteger;
} }
export function calculateStars(platform, data) { export function calculateStars(platform: string, data) {
let stars = 2; let stars = 2;
switch (platform) { switch (platform) {
case "betfair":
stars = calculateStarsBetfair(data);
break;
case "infer":
stars = calculateStarsInfer(data);
break;
case "foretold":
stars = calculateStarsForetold(data);
break;
case "givewellopenphil":
stars = calculateStarsGiveWellOpenPhil(data);
break;
case "goodjudgment":
stars = calculateStarsGoodJudgment(data);
break;
case "goodjudgmentopen":
stars = calculateStarsGoodJudgmentOpen(data);
break;
case "kalshi":
stars = calculateStarsKalshi(data);
break;
case "manifold":
stars = calculateStarsManifold(data);
break;
case "metaculus":
stars = calculateStarsMetaculus(data);
break;
case "polymarket":
stars = calculateStarsPolymarket(data);
break;
case "predictit":
stars = calculateStarsPredictIt(data);
break;
case "rootclaim":
stars = calculateStarsRootclaim(data);
break;
case "smarkets":
stars = calculateStarsSmarkets(data);
break;
case "wildeford":
stars = calculateStarsWildeford(data);
break;
// deprecated
case "AstralCodexTen": case "AstralCodexTen":
stars = calculateStarsAstralCodexTen(data); stars = calculateStarsAstralCodexTen(data);
break; break;
case "Betfair":
stars = calculateStarsBetfair(data);
break;
case "CoupCast": case "CoupCast":
stars = calculateStarsCoupCast(data); stars = calculateStarsCoupCast(data);
break; break;
@ -289,54 +330,15 @@ export function calculateStars(platform, data) {
case "Estimize": case "Estimize":
stars = calculateStarsEstimize(data); stars = calculateStarsEstimize(data);
break; break;
case "Foretold":
stars = calculateStarsForetold(data);
break;
case "GiveWell/OpenPhilanthropy":
stars = calculateStarsGiveWellOpenPhil(data);
break;
case "Good Judgment":
stars = calculateStarsGoodJudment(data);
break;
case "Good Judgment Open":
stars = calculateStarsGoodJudmentOpen(data);
break;
case "Hypermind": case "Hypermind":
stars = calculateStarsHypermind(data); stars = calculateStarsHypermind(data);
break; break;
case "Infer":
stars = calculateStarsInfer(data);
break;
case "Kalshi":
stars = calculateStarsKalshi(data);
break;
case "Ladbrokes": case "Ladbrokes":
stars = calculateStarsLadbrokes(data); stars = calculateStarsLadbrokes(data);
break; break;
case "Manifold Markets":
stars = calculateStarsManifoldMarkets(data);
break;
case "Metaculus":
stars = calculateStarsMetaculus(data);
break;
case "Omen": case "Omen":
stars = calculateStarsOmen(data); stars = calculateStarsOmen(data);
break; break;
case "Polymarket":
stars = calculateStarsPolymarket(data);
break;
case "PredictIt":
stars = calculateStarsPredictIt(data);
break;
case "Rootclaim":
stars = calculateStarsRootclaim(data);
break;
case "Smarkets":
stars = calculateStarsSmarkets(data);
break;
case "Peter Wildeford":
stars = calculateStarsWildeford(data);
break;
case "WilliamHill": case "WilliamHill":
stars = calculateStarsWilliamHill(data); stars = calculateStarsWilliamHill(data);
break; break;

4
src/pages/api/all-forecasts.ts
View File

@ -1,12 +1,12 @@
import { NextApiRequest, NextApiResponse } from "next/types"; import { NextApiRequest, NextApiResponse } from "next/types";
import { getFrontpageFullRaw } from "../../backend/frontpage"; import { getFrontpageFull } from "../../backend/frontpage";
export default async function handler( export default async function handler(
req: NextApiRequest, req: NextApiRequest,
res: NextApiResponse res: NextApiResponse
) { ) {
let frontpageFull = await getFrontpageFullRaw(); let frontpageFull = await getFrontpageFull();
console.log(frontpageFull.map((element) => element.title).slice(0, 5)); console.log(frontpageFull.map((element) => element.title).slice(0, 5));
console.log("..."); console.log("...");
res.status(200).json(frontpageFull); res.status(200).json(frontpageFull);

6
src/pages/api/create-dashboard-from-ids.ts
View File

@ -1,7 +1,7 @@
import crypto from "crypto";
import { NextApiRequest, NextApiResponse } from "next/types"; import { NextApiRequest, NextApiResponse } from "next/types";
import { pgInsertIntoDashboard } from "../../backend/database/pg-wrapper"; import { pgInsertIntoDashboard } from "../../backend/database/pg-wrapper";
import { hash } from "../../backend/utils/hash";
export default async function handler( export default async function handler(
req: NextApiRequest, req: NextApiRequest,
@ -14,8 +14,6 @@ export default async function handler(
let body = req.body; let body = req.body;
console.log(body); console.log(body);
const hash = (s: string) =>
crypto.createHash("sha256").update(s).digest("hex").slice(0, 10);
try { try {
let id = hash(JSON.stringify(body.ids)); let id = hash(JSON.stringify(body.ids));
let pgResponse = await pgInsertIntoDashboard({ let pgResponse = await pgInsertIntoDashboard({
@ -27,8 +25,6 @@ export default async function handler(
creator: body.creator || "", creator: body.creator || "",
extra: [], extra: [],
}, },
schema: "latest",
tableName: "dashboards",
}); });
res.status(200).send({ res.status(200).send({
dashboardId: id, dashboardId: id,

4
src/pages/api/dashboard-by-id.ts
View File

@ -16,7 +16,6 @@ export default async function handler(
console.log(id); console.log(id);
let dashboardItemArray = await pgGetByIds({ let dashboardItemArray = await pgGetByIds({
ids: [id], ids: [id],
schema: "latest",
table: "dashboards", table: "dashboards",
}); });
if (!!dashboardItemArray && dashboardItemArray.length > 0) { if (!!dashboardItemArray && dashboardItemArray.length > 0) {
@ -24,8 +23,7 @@ export default async function handler(
console.log(dashboardItem); console.log(dashboardItem);
let dashboardContents = await pgGetByIds({ let dashboardContents = await pgGetByIds({
ids: dashboardItem.contents, ids: dashboardItem.contents,
schema: "latest", table: "questions",
table: "combined",
}); });
res.status(200).send({ res.status(200).send({
dashboardContents, dashboardContents,

4
src/pages/api/frontpage.ts
View File

@ -1,12 +1,12 @@
import { NextApiRequest, NextApiResponse } from "next/types"; import { NextApiRequest, NextApiResponse } from "next/types";
import { getFrontpageRaw } from "../../backend/frontpage"; import { getFrontpage } from "../../backend/frontpage";
export default async function handler( export default async function handler(
req: NextApiRequest, req: NextApiRequest,
res: NextApiResponse res: NextApiResponse
) { ) {
let frontpageElements = await getFrontpageRaw(); let frontpageElements = await getFrontpage();
console.log(frontpageElements.map((element) => element.title).slice(0, 5)); console.log(frontpageElements.map((element) => element.title).slice(0, 5));
console.log("..."); console.log("...");
res.status(200).json(frontpageElements); res.status(200).json(frontpageElements);

2
src/pages/api/questions.ts
View File

@ -6,7 +6,7 @@ export default async function handler(
req: NextApiRequest, req: NextApiRequest,
res: NextApiResponse res: NextApiResponse
) { ) {
let allQuestions = await pgRead({ schema: "latest", tableName: "combined" }); let allQuestions = await pgRead({ tableName: "questions" });
console.log(allQuestions.map((element) => element.title).slice(0, 5)); console.log(allQuestions.map((element) => element.title).slice(0, 5));
console.log("..."); console.log("...");
res.status(200).json(allQuestions); res.status(200).json(allQuestions);

10
src/pages/capture.tsx
View File

@ -8,17 +8,11 @@ import CommonDisplay from "../web/search/CommonDisplay";
export { getServerSideProps } from "../web/search/anySearchPage"; export { getServerSideProps } from "../web/search/anySearchPage";
const CapturePage: NextPage<Props> = ({ const CapturePage: NextPage<Props> = (props) => {
defaultResults,
initialResults,
initialQueryParameters,
}) => {
return ( return (
<Layout page={"capture"}> <Layout page={"capture"}>
<CommonDisplay <CommonDisplay
defaultResults={defaultResults} {...props}
initialResults={initialResults}
initialQueryParameters={initialQueryParameters}
hasSearchbar={true} hasSearchbar={true}
hasCapture={true} hasCapture={true}
hasAdvancedOptions={false} hasAdvancedOptions={false}

2
src/pages/dashboards.tsx
View File

@ -93,7 +93,7 @@ export default function Home({
let isGraubardEasterEgg = (name) => (name == "Clay Graubard" ? true : false); let isGraubardEasterEgg = (name) => (name == "Clay Graubard" ? true : false);
return ( return (
<Layout key="index" page={"dashboard"}> <Layout page="dashboard">
{/* Display forecasts */} {/* Display forecasts */}
<div className="mt-7 mb-7"> <div className="mt-7 mb-7">
<h1 <h1

10
src/pages/index.tsx
View File

@ -8,17 +8,11 @@ import CommonDisplay from "../web/search/CommonDisplay";
export { getServerSideProps } from "../web/search/anySearchPage"; export { getServerSideProps } from "../web/search/anySearchPage";
const IndexPage: NextPage<Props> = ({ const IndexPage: NextPage<Props> = (props) => {
defaultResults,
initialResults,
initialQueryParameters,
}) => {
return ( return (
<Layout page={"search"}> <Layout page={"search"}>
<CommonDisplay <CommonDisplay
defaultResults={defaultResults} {...props}
initialResults={initialResults}
initialQueryParameters={initialQueryParameters}
hasSearchbar={true} hasSearchbar={true}
hasCapture={false} hasCapture={false}
hasAdvancedOptions={true} hasAdvancedOptions={true}

74
src/pages/secretEmbed.tsx
View File

@ -1,32 +1,33 @@
/* Imports */ /* Imports */
import { GetServerSideProps, NextPage } from "next";
import React from "react"; import React from "react";
import { displayForecast } from "../web/display/displayForecasts"; import { platforms } from "../backend/platforms";
import { platformsWithLabels } from "../web/platforms"; import { DisplayForecast } from "../web/display/displayForecasts";
import { FrontendForecast } from "../web/platforms";
import searchAccordingToQueryData from "../web/worker/searchAccordingToQueryData"; import searchAccordingToQueryData from "../web/worker/searchAccordingToQueryData";
/* Helper functions */ interface Props {
results: FrontendForecast[];
}
export async function getServerSideProps(context) { export const getServerSideProps: GetServerSideProps<Props> = async (
context
) => {
let urlQuery = context.query; // this is an object, not a string which I have to parse!! let urlQuery = context.query; // this is an object, not a string which I have to parse!!
let initialQueryParameters = { let initialQueryParameters = {
query: "", query: "",
starsThreshold: 2, starsThreshold: 2,
forecastsThreshold: 0, forecastsThreshold: 0,
forecastingPlatforms: platformsWithLabels, // weird key value format, forecastingPlatforms: platforms.map((platform) => platform.name),
...urlQuery, ...urlQuery,
}; };
let results; let results: FrontendForecast[] = [];
switch (initialQueryParameters.query != "") { if (initialQueryParameters.query != "") {
case true: results = await searchAccordingToQueryData(initialQueryParameters, 1);
results = await searchAccordingToQueryData(initialQueryParameters);
break;
default:
results = [];
break;
} }
return { return {
@ -34,35 +35,32 @@ export async function getServerSideProps(context) {
results: results, results: results,
}, },
}; };
} };
/* Body */ const SecretEmbedPage: NextPage<Props> = ({ results }) => {
export default function Home({ results }) { let result = results.length ? results[0] : null;
/* Final return */
let result = results ? results[0] : null;
return ( return (
<> <div className="mb-4 mt-8 flex flex-row justify-center items-center">
<div className="mb-4 mt-8 flex flex-row justify-center items-center "> <div className="w-6/12 place-self-center">
<div className="w-6/12 place-self-center"> <div>
<div> <div id="secretEmbed">
<div id="secretEmbed"> {result ? (
{result <DisplayForecast
? displayForecast({ forecast={result}
...result.item, showTimeStamp={true}
score: result.score, expandFooterToFullWidth={true}
showTimeStamp: true, />
expandFooterToFullWidth: true, ) : null}
}) </div>
: null} <br></br>
</div> <div id="secretObject">
<br></br> {result ? JSON.stringify(result, null, 4) : null}
<div id="secretObject">
{result ? JSON.stringify(result.item, null, 4) : null}
</div>
</div> </div>
</div> </div>
</div> </div>
</> </div>
); );
} };
export default SecretEmbedPage;

2
src/web/display/dashboardCreator.tsx
View File

@ -3,7 +3,7 @@ import React, { useState } from "react";
let exampleInput = `{ let exampleInput = `{
"title": "Random example", "title": "Random example",
"description": "Just a random description of a random example", "description": "Just a random description of a random example",
"ids": [ "metaculus-372", "goodjudmentopen-2244", "metaculus-7550", "kalshi-09d060ee-b184-4167-b86b-d773e56b4162", "wildeford-5d1a04e1a8", "metaculus-2817" ], "ids": [ "metaculus-372", "goodjudgmentopen-2244", "metaculus-7550", "kalshi-09d060ee-b184-4167-b86b-d773e56b4162", "wildeford-5d1a04e1a8", "metaculus-2817" ],
"creator": "Peter Parker" "creator": "Peter Parker"
}`; }`;

119
src/web/display/displayForecasts.tsx
View File

@ -3,6 +3,8 @@ import React from "react";
import { FaRegClipboard } from "react-icons/fa"; import { FaRegClipboard } from "react-icons/fa";
import ReactMarkdown from "react-markdown"; import ReactMarkdown from "react-markdown";
import { FrontendForecast } from "../platforms";
/* Definitions */ /* Definitions */
/* Support functions */ /* Support functions */
@ -350,9 +352,15 @@ let checkIfDisplayTimeStampAtBottom = (qualityIndicators) => {
} }
}; };
let getCurrencySymbolIfNeeded = ({ indicator, platform }) => { let getCurrencySymbolIfNeeded = ({
indicator,
platform,
}: {
indicator: any;
platform: string;
}) => {
let indicatorsWhichNeedCurrencySymbol = ["Volume", "Interest", "Liquidity"]; let indicatorsWhichNeedCurrencySymbol = ["Volume", "Interest", "Liquidity"];
let dollarPlatforms = ["PredictIt", "Kalshi", "PolyMarket"]; let dollarPlatforms = ["predictit", "kalshi", "polymarket"];
if (indicatorsWhichNeedCurrencySymbol.includes(indicator)) { if (indicatorsWhichNeedCurrencySymbol.includes(indicator)) {
if (dollarPlatforms.includes(platform)) { if (dollarPlatforms.includes(platform)) {
return "$"; return "$";
@ -461,7 +469,13 @@ let showFirstQualityIndicator = ({
} }
}; };
let displayQualityIndicators = ({ const displayQualityIndicators: React.FC<{
numforecasts: number;
timestamp: number;
showTimeStamp: boolean;
qualityindicators: any;
platform: string; // id string - e.g. "goodjudgment", not "Good Judgment"
}> = ({
numforecasts, numforecasts,
timestamp, timestamp,
showTimeStamp, showTimeStamp,
@ -504,6 +518,7 @@ let displayQualityIndicators = ({
let forecastFooter = ({ let forecastFooter = ({
stars, stars,
platform, platform,
platformLabel,
numforecasts, numforecasts,
qualityindicators, qualityindicators,
timestamp, timestamp,
@ -532,7 +547,7 @@ let forecastFooter = ({
expandFooterToFullWidth ? "place-self-center" : "self-center" expandFooterToFullWidth ? "place-self-center" : "self-center"
} col-span-1 font-bold ${debuggingWithBackground ? "bg-red-100" : ""}`} } col-span-1 font-bold ${debuggingWithBackground ? "bg-red-100" : ""}`}
> >
{platform {platformLabel
.replace("Good Judgment Open", "GJOpen") .replace("Good Judgment Open", "GJOpen")
.replace("OpenPhilanthropy", "OpenPhil") .replace("OpenPhilanthropy", "OpenPhil")
.replace("AstralCodexTen", "ACX") .replace("AstralCodexTen", "ACX")
@ -559,22 +574,30 @@ let forecastFooter = ({
/* Body */ /* Body */
export function displayForecast({ interface SingleProps {
id, forecast: FrontendForecast;
title, showTimeStamp: boolean;
url, expandFooterToFullWidth: boolean;
platform, showIdToggle?: boolean;
author, }
description,
options, export const DisplayForecast: React.FC<SingleProps> = ({
qualityindicators, forecast: {
timestamp, id,
visualization, title,
score, url,
platform,
platformLabel,
description,
options,
qualityindicators,
timestamp,
visualization,
},
showTimeStamp, showTimeStamp,
expandFooterToFullWidth, expandFooterToFullWidth,
showIdToggle, showIdToggle,
}) { }) => {
// const [isJustCopiedSignalVisible, setIsJustCopiedSignalVisible] = useState(false) // const [isJustCopiedSignalVisible, setIsJustCopiedSignalVisible] = useState(false)
const isJustCopiedSignalVisible = false; const isJustCopiedSignalVisible = false;
@ -588,7 +611,7 @@ export function displayForecast({
<div className="flex-grow"> <div className="flex-grow">
<div <div
className={`text-gray-800 ${opacityFromScore( className={`text-gray-800 ${opacityFromScore(
score 0
)} text-lg mb-2 font-medium justify-self-start`} )} text-lg mb-2 font-medium justify-self-start`}
> >
<div <div
@ -641,7 +664,7 @@ export function displayForecast({
? "flex" ? "flex"
: "hidden" : "hidden"
} ${opacityFromScore( } ${opacityFromScore(
score 0
)} row-end-2 col-start-2 col-end-2 row-start-1 row-end-1 col-span-1 items-center justify-center text-gray-600 ml-3 mr-2 `} )} row-end-2 col-start-2 col-end-2 row-start-1 row-end-1 col-span-1 items-center justify-center text-gray-600 ml-3 mr-2 `}
> >
<svg className="mt-1" height="10" width="16"> <svg className="mt-1" height="10" width="16">
@ -657,7 +680,7 @@ export function displayForecast({
{(options.length != 2 || {(options.length != 2 ||
(options[0].name != "Yes" && options[0].name != "No")) && ( (options[0].name != "Yes" && options[0].name != "No")) && (
<> <>
<div className={`mb-2 mt-2 ${opacityFromScore(score)}`}> <div className={`mb-2 mt-2 ${opacityFromScore(0)}`}>
{formatForecastOptions(options)} {formatForecastOptions(options)}
</div> </div>
<div <div
@ -667,7 +690,7 @@ export function displayForecast({
? "flex" ? "flex"
: "hidden" : "hidden"
} ${opacityFromScore( } ${opacityFromScore(
score 0
)} col-start-2 col-end-2 row-start-1 row-end-1 text-gray-600 mt-3 mb-3`} )} col-start-2 col-end-2 row-start-1 row-end-1 text-gray-600 mt-3 mb-3`}
> >
<svg className="ml-6 mr-1 mt-2" height="10" width="16"> <svg className="ml-6 mr-1 mt-2" height="10" width="16">
@ -680,13 +703,13 @@ export function displayForecast({
</> </>
)} )}
{platform !== "Guesstimate" && options.length < 3 && ( {platform !== "guesstimate" && options.length < 3 && (
<div className={`text-gray-500 ${opacityFromScore(score)} mt-4`}> <div className={`text-gray-500 ${opacityFromScore(0)} mt-4`}>
{displayMarkdown(description)} {displayMarkdown(description)}
</div> </div>
)} )}
{platform === "Guesstimate" && ( {platform === "guesstimate" && (
<img <img
className="rounded-sm mb-1" className="rounded-sm mb-1"
src={visualization} src={visualization}
@ -705,10 +728,11 @@ export function displayForecast({
</svg> </svg>
{`Last updated: ${timestamp ? timestamp.slice(0, 10) : "unknown"}`} {`Last updated: ${timestamp ? timestamp.slice(0, 10) : "unknown"}`}
</div> </div>
<div className={`${opacityFromScore(score)} w-full`}> <div className={`${opacityFromScore(0)} w-full`}>
{forecastFooter({ {forecastFooter({
stars: qualityindicators.stars, stars: qualityindicators.stars,
platform: author || platform, platform: platform,
platformLabel: platformLabel || platform, // author || platformLabel,
numforecasts: qualityindicators.numforecasts, numforecasts: qualityindicators.numforecasts,
qualityindicators, qualityindicators,
timestamp, timestamp,
@ -718,30 +742,39 @@ export function displayForecast({
</div> </div>
</a> </a>
); );
};
interface Props {
results: FrontendForecast[];
numDisplay: number;
showIdToggle: boolean;
} }
export default function displayForecasts({ const DisplayForecasts: React.FC<Props> = ({
results, results,
numDisplay, numDisplay,
showIdToggle, showIdToggle,
}) { }) => {
return !!results && !!results.slice ? ( if (!results) {
results.slice(0, numDisplay).map((fuseSearchResult) => { return <></>;
/*let displayWithMetaculusCapture = }
return (
<>
{results.slice(0, numDisplay).map((result) => (
/*let displayWithMetaculusCapture =
fuseSearchResult.item.platform == "Metaculus" fuseSearchResult.item.platform == "Metaculus"
? metaculusEmbed(fuseSearchResult.item) ? metaculusEmbed(fuseSearchResult.item)
: displayForecast({ ...fuseSearchResult.item }); : displayForecast({ ...fuseSearchResult.item });
*/ */
let display = displayForecast({ <DisplayForecast
...fuseSearchResult.item, forecast={result}
score: fuseSearchResult.score, showTimeStamp={false}
showTimeStamp: false, expandFooterToFullWidth={false}
expandFooterToFullWidth: false, showIdToggle={showIdToggle}
showIdToggle, />
}); ))}
return display; </>
})
) : (
<></>
); );
} };
export default DisplayForecasts;

22
src/web/display/displayOneForecastForCapture.tsx
View File

@ -2,20 +2,20 @@ import domtoimage from "dom-to-image"; // https://github.com/tsayen/dom-to-image
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import { CopyToClipboard } from "react-copy-to-clipboard"; import { CopyToClipboard } from "react-copy-to-clipboard";
import { FrontendForecast } from "../platforms";
import { uploadToImgur } from "../worker/uploadToImgur"; import { uploadToImgur } from "../worker/uploadToImgur";
import { displayForecast } from "./displayForecasts"; import { DisplayForecast } from "./displayForecasts";
function displayOneForecastInner(result, containerRef) { function displayOneForecastInner(result: FrontendForecast, containerRef) {
return ( return (
<div ref={containerRef}> <div ref={containerRef}>
{result {result ? (
? displayForecast({ <DisplayForecast
...result.item, forecast={result}
score: result.score, showTimeStamp={true}
showTimeStamp: true, expandFooterToFullWidth={true}
expandFooterToFullWidth: true, />
}) ) : null}
: null}
</div> </div>
); );
} }
@ -170,7 +170,7 @@ let generateMetaculusSource = (result, hasDisplayBeenCaptured) => {
}; };
interface Props { interface Props {
result: any; result: FrontendForecast;
} }
const DisplayOneForecast: React.FC<Props> = ({ result }) => { const DisplayOneForecast: React.FC<Props> = ({ result }) => {

45
src/web/display/multiSelectPlatforms.tsx
View File

@ -2,7 +2,7 @@ import chroma from "chroma-js";
import React from "react"; import React from "react";
import Select from "react-select"; import Select from "react-select";
import { platformsWithLabels } from "../platforms"; import { PlatformConfig } from "../platforms";
const colourStyles = { const colourStyles = {
control: (styles) => ({ ...styles, backgroundColor: "white" }), control: (styles) => ({ ...styles, backgroundColor: "white" }),
@ -59,17 +59,48 @@ const colourStyles = {
}), }),
}; };
export default function MultiSelectPlatform({ onChange, value }) { interface Props {
onChange: (platforms: string[]) => void;
value: string[];
platformsConfig: PlatformConfig[];
}
export const MultiSelectPlatform: React.FC<Props> = ({
onChange,
value,
platformsConfig,
}) => {
type Option = {
value: string;
label: string;
color: string;
};
const options: Option[] = platformsConfig.map((platform) => ({
value: platform.name,
label: platform.label,
color: platform.color,
}));
const id2option: { [k: string]: Option } = {};
for (const option of options) id2option[option.value] = option;
const selectValue = value.map((v) => id2option[v]).filter((v) => v);
const onSelectChange = (newValue: Option[]) => {
onChange(newValue.map((o) => o.value));
};
return ( return (
<Select <Select
defaultValue={platformsWithLabels} defaultValue={options}
isMulti isMulti
className="basic-multi-select w-full text-gray-700" className="basic-multi-select w-full text-gray-700"
onChange={onChange} onChange={onSelectChange}
closeMenuOnSelect={false} closeMenuOnSelect={false}
options={platformsWithLabels} options={options}
value={value} value={selectValue}
styles={colourStyles} styles={colourStyles}
/> />
); );
} };

63
src/web/platforms.ts
View File

@ -1,61 +1,12 @@
export const distinctColors = [ import { Forecast } from "../backend/platforms";
"#3d674a",
"#231149",
"#62520b",
"#32407e",
"#7d4f1b",
"#002455",
"#223900",
"#615691",
"#003419",
"#793466",
"#006669",
"#984158",
"#00314e",
"#460c00",
"#0d1624",
"#6f5b41",
"#240d23",
"#272600",
"#755469",
"#2a323d",
];
// https://medialab.github.io/iwanthue/ fancy light background export interface PlatformConfig {
export const platformNames = [ name: string;
"Betfair",
"FantasySCOTUS",
"Foretold",
"GiveWell/OpenPhilanthropy",
"Good Judgment",
"Good Judgment Open",
"Guesstimate",
"Infer",
"Kalshi",
"Manifold Markets",
"Metaculus",
"Peter Wildeford",
"PolyMarket",
"PredictIt",
"Rootclaim",
"Smarkets",
"X-risk estimates",
];
export interface PlatformWithLabel {
value: string;
label: string; label: string;
color: string; color: string;
} }
export const platformsWithLabels: PlatformWithLabel[] = platformNames.map( export type FrontendForecast = Forecast & {
(name, i) => ({ platformLabel: string;
value: name, visualization?: any;
label: name, };
color: distinctColors[i],
})
);
export const platforms = platformsWithLabels;
// Deprecated: AstralCodexTen, CoupCast, CSET-foretell, Estimize, Elicit, Hypermind, Omen, WilliamHill

109
src/web/search/CommonDisplay.tsx
View File

@ -1,54 +1,13 @@
import { useRouter } from "next/router"; import { useRouter } from "next/router";
import React, { import React, { DependencyList, EffectCallback, Fragment, useEffect, useState } from "react";
DependencyList,
EffectCallback,
Fragment,
useEffect,
useState,
} from "react";
import ButtonsForStars from "../display/buttonsForStars"; import ButtonsForStars from "../display/buttonsForStars";
import Form from "../display/form"; import Form from "../display/form";
import MultiSelectPlatform from "../display/multiSelectPlatforms"; import { MultiSelectPlatform } from "../display/multiSelectPlatforms";
import { SliderElement } from "../display/slider"; import { SliderElement } from "../display/slider";
import { platformsWithLabels, PlatformWithLabel } from "../platforms"; import { FrontendForecast } from "../platforms";
import searchAccordingToQueryData from "../worker/searchAccordingToQueryData"; import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
import { Props as AnySearchPageProps, QueryParameters } from "./anySearchPage";
interface QueryParametersWithoutNum {
query: string;
starsThreshold: number;
forecastsThreshold: number;
forecastingPlatforms: PlatformWithLabel[];
}
export interface QueryParameters extends QueryParametersWithoutNum {
numDisplay: number;
}
interface Props {
defaultResults: any;
initialResults: any;
initialQueryParameters: QueryParameters;
hasSearchbar: boolean;
hasCapture: boolean;
hasAdvancedOptions: boolean;
placeholder: string;
displaySeeMoreHint: boolean;
displayForecastsWrapper: (opts: {
results: any;
numDisplay: number;
whichResultToDisplayAndCapture: number;
showIdToggle: boolean;
}) => React.ReactNode;
}
export const defaultQueryParameters: QueryParametersWithoutNum = {
query: "",
starsThreshold: 2,
forecastsThreshold: 0,
forecastingPlatforms: platformsWithLabels, // weird key value format,
};
export const defaultNumDisplay = 21;
const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => { const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => {
const initial = React.useRef(true); const initial = React.useRef(true);
@ -61,11 +20,29 @@ const useNoInitialEffect = (effect: EffectCallback, deps: DependencyList) => {
}, deps); }, deps);
}; };
interface Props extends AnySearchPageProps {
hasSearchbar: boolean;
hasCapture: boolean;
hasAdvancedOptions: boolean;
placeholder: string;
displaySeeMoreHint: boolean;
displayForecastsWrapper: (opts: {
results: FrontendForecast[];
numDisplay: number;
whichResultToDisplayAndCapture: number;
showIdToggle: boolean;
}) => React.ReactNode;
}
/* Body */ /* Body */
const CommonDisplay: React.FC<Props> = ({ const CommonDisplay: React.FC<Props> = ({
defaultResults, defaultResults,
initialResults, initialResults,
initialQueryParameters, initialQueryParameters,
defaultQueryParameters,
initialNumDisplay,
defaultNumDisplay,
platformsConfig,
hasSearchbar, hasSearchbar,
hasCapture, hasCapture,
hasAdvancedOptions, hasAdvancedOptions,
@ -76,13 +53,12 @@ const CommonDisplay: React.FC<Props> = ({
const router = useRouter(); const router = useRouter();
/* States */ /* States */
const [queryParameters, setQueryParameters] = const [queryParameters, setQueryParameters] = useState<QueryParameters>(
useState<QueryParametersWithoutNum>(initialQueryParameters); initialQueryParameters
const [numDisplay, setNumDisplay] = useState(
initialQueryParameters.numDisplay ?? defaultNumDisplay
); );
const [numDisplay, setNumDisplay] = useState(initialNumDisplay);
// used to distinguish numDisplay updates which force search and don't force search, see effects below // used to distinguish numDisplay updates which force search and don't force search, see effects below
const [forceSearch, setForceSearch] = useState(0); const [forceSearch, setForceSearch] = useState(0);
@ -100,25 +76,25 @@ const CommonDisplay: React.FC<Props> = ({
numDisplay, numDisplay,
}; };
let filterManually = (queryData: QueryParameters, results) => { let filterManually = (
queryData: QueryParameters,
results: FrontendForecast[]
) => {
if ( if (
queryData.forecastingPlatforms && queryData.forecastingPlatforms &&
queryData.forecastingPlatforms.length > 0 queryData.forecastingPlatforms.length > 0
) { ) {
let forecastingPlatforms = queryData.forecastingPlatforms.map(
(platformObj) => platformObj.value
);
results = results.filter((result) => results = results.filter((result) =>
forecastingPlatforms.includes(result.item.platform) queryData.forecastingPlatforms.includes(result.platform)
); );
} }
if (queryData.starsThreshold === 4) { if (queryData.starsThreshold === 4) {
results = results.filter( results = results.filter(
(result) => result.item.qualityindicators.stars >= 4 (result) => result.qualityindicators.stars >= 4
); );
} }
if (queryData.forecastsThreshold) { if (queryData.forecastsThreshold) {
// results = results.filter(result => (result.qualityindicators && result.item.qualityindicators.numforecasts > forecastsThreshold)) // results = results.filter(result => (result.qualityindicators && result.qualityindicators.numforecasts > forecastsThreshold))
} }
return results; return results;
}; };
@ -128,17 +104,13 @@ const CommonDisplay: React.FC<Props> = ({
let results = queryIsEmpty let results = queryIsEmpty
? filterManually(queryData, defaultResults) ? filterManually(queryData, defaultResults)
: await searchAccordingToQueryData(queryData); : await searchAccordingToQueryData(queryData, numDisplay);
setResults(results); setResults(results);
} }
// I don't want the function which display forecasts (displayForecasts) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home. // I don't want the function which display forecasts (displayForecasts) to change with a change in queryParameters. But I want it to have access to the queryParameters, and in particular access to queryParameters.numDisplay. Hence why this function lives inside Home.
let getInfoToDisplayForecastsFunction = ({ let getInfoToDisplayForecastsFunction = () => {
results,
whichResultToDisplayAndCapture,
showIdToggle,
}) => {
let numDisplayRounded = let numDisplayRounded =
numDisplay % 3 != 0 numDisplay % 3 != 0
? numDisplay + (3 - (Math.round(numDisplay) % 3)) ? numDisplay + (3 - (Math.round(numDisplay) % 3))
@ -154,7 +126,7 @@ const CommonDisplay: React.FC<Props> = ({
const updateRoute = () => { const updateRoute = () => {
const stringify = (key: string, value: any) => { const stringify = (key: string, value: any) => {
if (key === "forecastingPlatforms") { if (key === "forecastingPlatforms") {
return value.map((x) => x.value).join("|"); return value.join("|");
} else { } else {
return String(value); return String(value);
} }
@ -330,6 +302,7 @@ const CommonDisplay: React.FC<Props> = ({
</div> </div>
<div className="flex col-span-3 items-center justify-center"> <div className="flex col-span-3 items-center justify-center">
<MultiSelectPlatform <MultiSelectPlatform
platformsConfig={platformsConfig}
value={queryParameters.forecastingPlatforms} value={queryParameters.forecastingPlatforms}
onChange={onChangeSelectedPlatforms} onChange={onChangeSelectedPlatforms}
/> />
@ -344,13 +317,7 @@ const CommonDisplay: React.FC<Props> = ({
</div> </div>
) : null} ) : null}
<div> <div>{getInfoToDisplayForecastsFunction()}</div>
{getInfoToDisplayForecastsFunction({
results,
whichResultToDisplayAndCapture,
showIdToggle,
})}
</div>
{displaySeeMoreHint && {displaySeeMoreHint &&
(!results || (results.length != 0 && numDisplay < results.length)) ? ( (!results || (results.length != 0 && numDisplay < results.length)) ? (

85
src/web/search/anySearchPage.tsx
View File

@ -1,46 +1,103 @@
import { GetServerSideProps } from "next"; import { GetServerSideProps } from "next";
import { getFrontpage } from "../../backend/frontpage"; import { getFrontpage } from "../../backend/frontpage";
import { platforms } from "../../backend/platforms";
import { FrontendForecast, PlatformConfig } from "../platforms";
import searchAccordingToQueryData from "../worker/searchAccordingToQueryData"; import searchAccordingToQueryData from "../worker/searchAccordingToQueryData";
import {
defaultNumDisplay,
defaultQueryParameters,
QueryParameters,
} from "./CommonDisplay";
/* Common code for / and /capture */ /* Common code for / and /capture */
export interface QueryParameters {
query: string;
starsThreshold: number;
forecastsThreshold: number;
forecastingPlatforms: string[]; // platform names
}
export interface Props { export interface Props {
defaultResults: any; defaultResults: FrontendForecast[];
initialResults: any; initialResults: FrontendForecast[];
initialQueryParameters: QueryParameters; initialQueryParameters: QueryParameters;
defaultQueryParameters: QueryParameters;
initialNumDisplay: number;
defaultNumDisplay: number;
platformsConfig: PlatformConfig[];
} }
export const getServerSideProps: GetServerSideProps<Props> = async ( export const getServerSideProps: GetServerSideProps<Props> = async (
context context
) => { ) => {
let urlQuery = context.query; const urlQuery = context.query;
let initialQueryParameters: QueryParameters = { const platformsConfig = platforms.map((platform) => ({
...defaultQueryParameters, name: platform.name,
numDisplay: defaultNumDisplay, label: platform.label,
...urlQuery, // FIXME - parse numerical fields color: platform.color,
}));
platformsConfig.push({
name: "guesstimate",
label: "Guesstimate",
color: "223900",
});
const defaultQueryParameters: QueryParameters = {
query: "",
starsThreshold: 2,
forecastsThreshold: 0,
forecastingPlatforms: platforms.map((platform) => platform.name),
}; };
let defaultResults = await getFrontpage(); const initialQueryParameters: QueryParameters = {
...defaultQueryParameters,
};
if (urlQuery.query) {
initialQueryParameters.query = String(urlQuery.query);
}
if (urlQuery.starsThreshold) {
initialQueryParameters.starsThreshold = Number(urlQuery.starsThreshold);
}
if (urlQuery.forecastsThreshold !== undefined) {
initialQueryParameters.forecastsThreshold = Number(
urlQuery.forecastsThreshold
);
}
if (urlQuery.forecastingPlatforms !== undefined) {
initialQueryParameters.forecastingPlatforms = String(
urlQuery.forecastingPlatforms
).split("|");
}
const platformNameToLabel = Object.fromEntries(
platforms.map((platform) => [platform.name, platform.label])
);
const defaultNumDisplay = 21;
const initialNumDisplay = Number(urlQuery.numDisplay) || defaultNumDisplay;
const defaultResults = (await getFrontpage()).map((result) => ({
...result,
platformLabel: platformNameToLabel[result.platform] || result.platform,
}));
const initialResults = const initialResults =
!!initialQueryParameters && !!initialQueryParameters &&
initialQueryParameters.query != "" && initialQueryParameters.query != "" &&
initialQueryParameters.query != undefined initialQueryParameters.query != undefined
? await searchAccordingToQueryData(initialQueryParameters) ? await searchAccordingToQueryData(
initialQueryParameters,
initialNumDisplay
)
: defaultResults; : defaultResults;
return { return {
props: { props: {
initialQueryParameters, initialQueryParameters,
defaultQueryParameters,
initialNumDisplay,
defaultNumDisplay,
initialResults, initialResults,
defaultResults, defaultResults,
platformsConfig,
}, },
}; };
}; };

15
src/web/worker/getDashboardForecasts.ts
View File

@ -2,7 +2,7 @@ import axios from "axios";
export async function getDashboardForecastsByDashboardId({ dashboardId }) { export async function getDashboardForecastsByDashboardId({ dashboardId }) {
console.log("getDashboardForecastsByDashboardId: "); console.log("getDashboardForecastsByDashboardId: ");
let dashboardForecastCompatibleWithFuse = []; let dashboardContents = [];
let dashboardItem = null; let dashboardItem = null;
try { try {
let { data } = await axios({ let { data } = await axios({
@ -13,22 +13,13 @@ export async function getDashboardForecastsByDashboardId({ dashboardId }) {
}, },
}); });
console.log(data); console.log(data);
let dashboardContents = data.dashboardContents; dashboardContents = data.dashboardContents;
dashboardItem = data.dashboardItem; dashboardItem = data.dashboardItem;
// let { dashboardContents, dashboardItem } = data
if (!!dashboardContents && !!dashboardContents.map) {
dashboardForecastCompatibleWithFuse = dashboardContents.map((result) => ({
item: result,
score: 0,
}));
} else {
console.log("Error in getDashboardForecastsByDashboardId");
}
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} finally { } finally {
return { return {
dashboardForecasts: dashboardForecastCompatibleWithFuse, dashboardForecasts: dashboardContents,
dashboardItem, dashboardItem,
}; };
} }

56
src/web/worker/searchAccordingToQueryData.ts
View File

@ -1,56 +1,48 @@
import { FrontendForecast } from "../platforms";
import { QueryParameters } from "../search/anySearchPage";
import searchGuesstimate from "./searchGuesstimate"; import searchGuesstimate from "./searchGuesstimate";
import searchWithAlgolia from "./searchWithAlgolia"; import searchWithAlgolia from "./searchWithAlgolia";
export default async function searchAccordingToQueryData(queryData) { export default async function searchAccordingToQueryData(
let results = []; queryData: QueryParameters,
limit: number
): Promise<FrontendForecast[]> {
let results: FrontendForecast[] = [];
try { try {
// defs // defs
let query = queryData.query == undefined ? "" : queryData.query; let query = queryData.query == undefined ? "" : queryData.query;
if (query == "") return -1; if (query == "") return [];
let forecastsThreshold = queryData.forecastsThreshold; let forecastsThreshold = queryData.forecastsThreshold;
let starsThreshold = queryData.starsThreshold; let starsThreshold = queryData.starsThreshold;
let forecastingPlatforms = queryData.forecastingPlatforms.map(
(x) => x.value
);
let platformsIncludeGuesstimate = let platformsIncludeGuesstimate =
forecastingPlatforms.includes("Guesstimate") && starsThreshold <= 1; queryData.forecastingPlatforms.includes("guesstimate") &&
starsThreshold <= 1;
// preparation // preparation
let unawaitedAlgoliaResponse = searchWithAlgolia({ let unawaitedAlgoliaResponse = searchWithAlgolia({
queryString: query, queryString: query,
hitsPerPage: queryData.numDisplay + 50, hitsPerPage: limit + 50,
starsThreshold, starsThreshold,
filterByPlatforms: forecastingPlatforms, filterByPlatforms: queryData.forecastingPlatforms,
forecastsThreshold, forecastsThreshold,
}); });
// consider the guesstimate and the non-guesstimate cases separately. // consider the guesstimate and the non-guesstimate cases separately.
switch (platformsIncludeGuesstimate) { if (platformsIncludeGuesstimate) {
case false: let responses = await Promise.all([
results = await unawaitedAlgoliaResponse; unawaitedAlgoliaResponse,
break; searchGuesstimate(query),
case true: ]); // faster than two separate requests
let responses = await Promise.all([ let responsesNotGuesstimate = responses[0];
unawaitedAlgoliaResponse, let responsesGuesstimate = responses[1];
searchGuesstimate(query), results = [...responsesNotGuesstimate, ...responsesGuesstimate];
]); // faster than two separate requests //results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
let responsesNotGuesstimate = responses[0]; } else {
let responsesGuesstimate = responses[1]; results = await unawaitedAlgoliaResponse;
results = [...responsesNotGuesstimate, ...responsesGuesstimate];
//results.sort((x,y)=> x.ranking < y.ranking ? -1: 1)
break;
default:
return -1;
} }
// Maintain compatibility with fuse
let makeCompatibleWithFuse = (results) =>
results.map((result, index) => ({
item: result,
score: 0, // 0.4 - 0.4 / (index + 1),
}));
results = makeCompatibleWithFuse(results); return results;
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} finally { } finally {

87
src/web/worker/searchGuesstimate.ts
View File

@ -1,14 +1,18 @@
/* Imports */ /* Imports */
import axios from "axios"; import axios from "axios";
import { FrontendForecast } from "../platforms";
/* Definitions */ /* Definitions */
let urlEndPoint = let urlEndPoint =
"https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c"; "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c";
/* Body */ /* Body */
export default function searchGuesstimate(query) { export default async function searchGuesstimate(
let response = axios({ query
): Promise<FrontendForecast[]> {
let response = await axios({
url: urlEndPoint, url: urlEndPoint,
// credentials: "omit", // credentials: "omit",
headers: { headers: {
@ -24,48 +28,45 @@ export default function searchGuesstimate(query) {
"%20" "%20"
)}&hitsPerPage=20&page=0&getRankingInfo=true\"}`, )}&hitsPerPage=20&page=0&getRankingInfo=true\"}`,
method: "POST", method: "POST",
}) });
.then((res) => res.data.hits)
.then((models) =>
models.map((model, index) => {
let description = model.description
? model.description.replace(/\n/g, " ").replace(/ /g, " ")
: "";
let stars = description.length > 250 ? 2 : 1;
return {
title: model.name,
url: `https://www.getguesstimate.com/models/${model.id}`,
platform: "Guesstimate",
description: description,
options: [],
qualityindicators: {
stars: stars,
numforecasts: 1,
numforecasters: 1,
},
visualization: model.big_screenshot,
ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
};
})
)
.then((models) => {
// filter for duplicates. Surprisingly common.
let uniqueTitles = [];
let uniqueModels = [];
for (let model of models) {
if (
!uniqueTitles.includes(model.title) &&
!model.title.includes("copy")
) {
uniqueModels.push(model);
uniqueTitles.push(model.title);
}
}
return uniqueModels;
});
console.log(response); const models: any[] = response.data.hits;
return response; // This is a promise. Usable with either async/await (a mess in React) or with .then(guesstimateModels => doSomething(guesstimateModels)) const mappedModels: FrontendForecast[] = models.map((model, index) => {
let description = model.description
? model.description.replace(/\n/g, " ").replace(/ /g, " ")
: "";
let stars = description.length > 250 ? 2 : 1;
return {
id: `guesstimate-${model.id}`,
title: model.name,
url: `https://www.getguesstimate.com/models/${model.id}`,
timestamp: model.created_at, // TODO - check that format matches
platform: "guesstimate",
platformLabel: "Guesstimate",
description: description,
options: [],
qualityindicators: {
stars: stars,
numforecasts: 1,
numforecasters: 1,
},
visualization: model.big_screenshot,
ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack
};
});
// filter for duplicates. Surprisingly common.
let uniqueTitles = [];
let uniqueModels: FrontendForecast[] = [];
for (let model of mappedModels) {
if (!uniqueTitles.includes(model.title) && !model.title.includes("copy")) {
uniqueModels.push(model);
uniqueTitles.push(model.title);
}
}
console.log(uniqueModels);
return uniqueModels;
} }
// searchGuesstimate("COVID-19").then(guesstimateModels => console.log(guesstimateModels)) // searchGuesstimate("COVID-19").then(guesstimateModels => console.log(guesstimateModels))

48
src/web/worker/searchWithAlgolia.ts
View File

@ -1,5 +1,7 @@
import algoliasearch from "algoliasearch"; import algoliasearch from "algoliasearch";
import { FrontendForecast } from "../platforms";
const client = algoliasearch( const client = algoliasearch(
process.env.NEXT_PUBLIC_ALGOLIA_APP_ID, process.env.NEXT_PUBLIC_ALGOLIA_APP_ID,
process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_KEY process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_KEY
@ -76,13 +78,12 @@ let noExactMatch = (queryString, result) => {
// only query string // only query string
export default async function searchWithAlgolia({ export default async function searchWithAlgolia({
queryString, queryString,
hitsPerPage, hitsPerPage = 5,
starsThreshold, starsThreshold,
filterByPlatforms, filterByPlatforms,
forecastsThreshold, forecastsThreshold,
}) { }): Promise<FrontendForecast[]> {
hitsPerPage = hitsPerPage || 5; let response = await index.search<FrontendForecast>(queryString, {
let response = await index.search(queryString, {
hitsPerPage, hitsPerPage,
filters: buildFilter({ filters: buildFilter({
starsThreshold, starsThreshold,
@ -92,12 +93,7 @@ export default async function searchWithAlgolia({
//facetFilters: buildFacetFilter({filterByPlatforms}), //facetFilters: buildFacetFilter({filterByPlatforms}),
getRankingInfo: true, getRankingInfo: true,
}); });
let results: any[] = response.hits; let results: FrontendForecast[] = response.hits;
console.log(
"searchWithAlgolia.js/searchWithAlgolia/queryString",
queryString
);
console.log("searchWithAlgolia.js/searchWithAlgolia/results", results);
let recursionError = ["metaforecast", "metaforecasts", "metaforecasting"]; let recursionError = ["metaforecast", "metaforecasts", "metaforecasting"];
if ( if (
@ -106,9 +102,11 @@ export default async function searchWithAlgolia({
) { ) {
results = [ results = [
{ {
id: "not-found",
title: "No search results match your query", title: "No search results match your query",
url: "https://metaforecast.org", url: "https://metaforecast.org",
platform: "Metaforecast", platform: "metaforecast",
platformLabel: "Metaforecast",
description: "Maybe try a broader query?", description: "Maybe try a broader query?",
options: [ options: [
{ {
@ -128,17 +126,19 @@ export default async function searchWithAlgolia({
numforecasters: 1, numforecasters: 1,
stars: 5, stars: 5,
}, },
noExactSearchResults: true, // noExactSearchResults: true,
optionsstringforsearch: "Yes, No", // optionsstringforsearch: "Yes, No",
has_numforecasts: true, // has_numforecasts: true,
}, },
]; ];
} else if (recursionError.includes(queryString.toLowerCase())) { } else if (recursionError.includes(queryString.toLowerCase())) {
results = [ results = [
{ {
id: "recursion-error",
title: `Did you mean: ${queryString}?`, title: `Did you mean: ${queryString}?`,
url: "https://metaforecast.org/recursion?bypassEasterEgg=true", url: "https://metaforecast.org/recursion?bypassEasterEgg=true",
platform: "Metaforecast", platform: "metaforecast",
platformLabel: "Metaforecast",
description: description:
"Fatal error: Too much recursion. Click to proceed anyways", "Fatal error: Too much recursion. Click to proceed anyways",
options: [ options: [
@ -159,9 +159,9 @@ export default async function searchWithAlgolia({
numforecasters: 1, numforecasters: 1,
stars: 5, stars: 5,
}, },
noExactSearchResults: true, // noExactSearchResults: true,
optionsstringforsearch: "Yes, No", // optionsstringforsearch: "Yes, No",
has_numforecasts: true, // has_numforecasts: true,
}, },
...results, ...results,
]; ];
@ -171,9 +171,11 @@ export default async function searchWithAlgolia({
noExactMatch(queryString, results[0]) noExactMatch(queryString, results[0])
) { ) {
results.unshift({ results.unshift({
id: "not-found-2",
title: "No search results appear to match your query", title: "No search results appear to match your query",
url: "https://metaforecast.org", url: "https://metaforecast.org",
platform: "Metaforecast", platform: "metaforecast",
platformLabel: "Metaforecast",
description: "Maybe try a broader query? That said, we could be wrong.", description: "Maybe try a broader query? That said, we could be wrong.",
options: [ options: [
{ {
@ -193,12 +195,12 @@ export default async function searchWithAlgolia({
numforecasters: 1, numforecasters: 1,
stars: 1, stars: 1,
}, },
noExactSearchResults: true, // noExactSearchResults: true,
optionsstringforsearch: "Yes, No", // optionsstringforsearch: "Yes, No",
has_numforecasts: true, // has_numforecasts: true,
}); });
} else { } else {
results[0].noExactSearchResults = false; // results[0].noExactSearchResults = false;
} }
return results; return results;

3
tsconfig.json
View File

@ -17,7 +17,8 @@
"incremental": true, "incremental": true,
"moduleResolution": "node", "moduleResolution": "node",
"resolveJsonModule": true, "resolveJsonModule": true,
"isolatedModules": true "isolatedModules": true,
"allowJs": true
}, },
"include": [ "include": [
"next-env.d.ts", "next-env.d.ts",