feat: new db structure; platform labels
This commit is contained in:
parent
a1ba23e340
commit
72db637972
9
.gitignore
vendored
9
.gitignore
vendored
|
@ -26,12 +26,6 @@ npm-debug.log*
|
|||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# local env files
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
|
@ -41,5 +35,4 @@ package-lock.json ## use yarn.lock instead
|
|||
# Local Netlify folder
|
||||
.netlify
|
||||
|
||||
/.env
|
||||
/.env.production
|
||||
/.env*
|
||||
|
|
|
@ -1,13 +1,11 @@
|
|||
import { Pool, PoolClient } from "pg";
|
||||
|
||||
import { Forecast, platforms } from "../platforms";
|
||||
import { Forecast } from "../platforms";
|
||||
import { hash } from "../utils/hash";
|
||||
import { measureTime } from "../utils/measureTime";
|
||||
import { roughSizeOfObject } from "../utils/roughSize";
|
||||
|
||||
const platformTableNames = platforms.map((platform) => platform.name);
|
||||
|
||||
const forecastTableNames = [...platformTableNames, "combined", "history"];
|
||||
const forecastTableNames = ["questions", "history"];
|
||||
|
||||
const allTableNames = [...forecastTableNames, "dashboards", "frontpage"];
|
||||
|
||||
|
@ -111,28 +109,27 @@ let buildMetaforecastTable = (table: string) => `CREATE TABLE ${table} (
|
|||
extra json
|
||||
);`;
|
||||
|
||||
async function pgInitializeLatest() {
|
||||
async function pgInitializeQuestions() {
|
||||
let YOLO = false;
|
||||
if (YOLO) {
|
||||
console.log("Create tables & their indexes");
|
||||
for (const table of platformTableNames) {
|
||||
await runPgCommand({
|
||||
command: dropTable(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: buildMetaforecastTable(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: createUniqueIndex(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
}
|
||||
const table = "questions";
|
||||
await runPgCommand({
|
||||
command: dropTable(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: buildMetaforecastTable(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
await runPgCommand({
|
||||
command: createUniqueIndex(table),
|
||||
pool: readWritePool,
|
||||
});
|
||||
console.log("");
|
||||
} else {
|
||||
console.log(
|
||||
"pgInitializeLatest: This command is dangerous, set YOLO to true in the code to invoke it"
|
||||
"pgInitializeQuestions: This command is dangerous, set YOLO to true in the code to invoke it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
@ -234,7 +231,7 @@ async function pgInitializeFrontpage() {
|
|||
|
||||
export async function pgInitialize() {
|
||||
await pgInitializeScaffolding();
|
||||
await pgInitializeLatest();
|
||||
await pgInitializeQuestions();
|
||||
await pgInitializeHistories();
|
||||
await pgInitializeDashboards();
|
||||
await pgInitializeFrontpage();
|
||||
|
@ -416,11 +413,11 @@ pgInsertIntoDashboard({
|
|||
export async function pgUpsert({
|
||||
contents,
|
||||
tableName,
|
||||
replace,
|
||||
replacePlatform,
|
||||
}: {
|
||||
contents: Forecast[];
|
||||
tableName: string;
|
||||
replace: boolean;
|
||||
replacePlatform?: string;
|
||||
}) {
|
||||
if (!forecastTableNames.includes(tableName)) {
|
||||
throw Error(
|
||||
|
@ -432,8 +429,10 @@ export async function pgUpsert({
|
|||
const client = await readWritePool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
if (replace) {
|
||||
client.query(`DELETE FROM ${tableName}`);
|
||||
if (replacePlatform) {
|
||||
await client.query(`DELETE FROM ${tableName} WHERE platform = $1`, [
|
||||
replacePlatform,
|
||||
]);
|
||||
}
|
||||
console.log(
|
||||
`Upserting ${contents.length} rows into postgres table ${tableName}.`
|
||||
|
|
|
@ -1,10 +1,9 @@
|
|||
import { pgReadWithReadCredentials, pgUpsert } from "../../database/pg-wrapper";
|
||||
|
||||
export async function updateHistory() {
|
||||
let latest = await pgReadWithReadCredentials({ tableName: "combined" });
|
||||
let latest = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
await pgUpsert({
|
||||
contents: latest,
|
||||
tableName: "history",
|
||||
replace: false,
|
||||
});
|
||||
}
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
import { pgInitialize } from "../database/pg-wrapper";
|
||||
import { doEverything } from "../flow/doEverything";
|
||||
import { updateHistory } from "../flow/history/updateHistory";
|
||||
import { mergeEverything } from "../flow/mergeEverything";
|
||||
import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData";
|
||||
import { rebuildFrontpage } from "../frontpage";
|
||||
import { platforms, processPlatform } from "../platforms";
|
||||
|
@ -20,13 +19,6 @@ export const jobs: Job[] = [
|
|||
message: `Download predictions from ${platform.name}`,
|
||||
run: () => processPlatform(platform),
|
||||
})),
|
||||
{
|
||||
name: "merge",
|
||||
message:
|
||||
"Merge tables into one big table (and push the result to a pg database)",
|
||||
run: mergeEverything,
|
||||
separate: true,
|
||||
},
|
||||
{
|
||||
name: "algolia",
|
||||
message: 'Rebuild algolia database ("index")',
|
||||
|
|
|
@ -1,28 +0,0 @@
|
|||
import { pgRead, pgUpsert } from "../database/pg-wrapper";
|
||||
import { platforms } from "../platforms";
|
||||
|
||||
/* Merge everything */
|
||||
|
||||
export async function mergeEverythingInner() {
|
||||
let merged = [];
|
||||
for (let platform of platforms) {
|
||||
const platformName = platform.name;
|
||||
let json = await pgRead({ tableName: platformName });
|
||||
console.log(`${platformName} has ${json.length} questions\n`);
|
||||
merged = merged.concat(json);
|
||||
}
|
||||
let mergedprocessed = merged.map((element) => ({
|
||||
...element,
|
||||
optionsstringforsearch: element.options
|
||||
.map((option) => option.name)
|
||||
.join(", "),
|
||||
}));
|
||||
console.log(`In total, there are ${mergedprocessed.length} questions`);
|
||||
return mergedprocessed;
|
||||
}
|
||||
|
||||
export async function mergeEverything() {
|
||||
let merged = await mergeEverythingInner();
|
||||
await pgUpsert({ contents: merged, tableName: "combined", replace: true });
|
||||
console.log("Done");
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
import { pgRead, readWritePool } from "./database/pg-wrapper";
|
||||
import { Forecast } from "./platforms";
|
||||
|
||||
export async function getFrontpageRaw() {
|
||||
export async function getFrontpage(): Promise<Forecast[]> {
|
||||
const client = await readWritePool.connect();
|
||||
const res = await client.query(
|
||||
"SELECT frontpage_sliced FROM frontpage ORDER BY id DESC LIMIT 1"
|
||||
|
@ -10,7 +11,7 @@ export async function getFrontpageRaw() {
|
|||
return res.rows[0].frontpage_sliced;
|
||||
}
|
||||
|
||||
export async function getFrontpageFullRaw() {
|
||||
export async function getFrontpageFull(): Promise<Forecast[]> {
|
||||
const client = await readWritePool.connect();
|
||||
const res = await client.query(
|
||||
"SELECT frontpage_full FROM frontpage ORDER BY id DESC LIMIT 1"
|
||||
|
@ -20,31 +21,15 @@ export async function getFrontpageFullRaw() {
|
|||
return res.rows[0].frontpage_full;
|
||||
}
|
||||
|
||||
export async function getFrontpage() {
|
||||
let frontPageForecastsCompatibleWithFuse = [];
|
||||
try {
|
||||
let data = await getFrontpageRaw();
|
||||
frontPageForecastsCompatibleWithFuse = data.map((result) => ({
|
||||
item: result,
|
||||
score: 0,
|
||||
}));
|
||||
return frontPageForecastsCompatibleWithFuse;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
} finally {
|
||||
return frontPageForecastsCompatibleWithFuse;
|
||||
}
|
||||
}
|
||||
|
||||
export async function rebuildFrontpage() {
|
||||
const frontpageFull = await pgRead({
|
||||
tableName: "combined",
|
||||
tableName: "questions",
|
||||
});
|
||||
|
||||
const client = await readWritePool.connect();
|
||||
const frontpageSliced = (
|
||||
await client.query(`
|
||||
SELECT * FROM combined
|
||||
SELECT * FROM questions
|
||||
WHERE
|
||||
(qualityindicators->>'stars')::int >= 3
|
||||
AND description != ''
|
||||
|
|
|
@ -5,7 +5,7 @@ import fs from "fs";
|
|||
import { pgReadWithReadCredentials } from "../database/pg-wrapper";
|
||||
|
||||
let main = async () => {
|
||||
let json = await pgReadWithReadCredentials({ tableName: "combined" });
|
||||
let json = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
let string = JSON.stringify(json, null, 2);
|
||||
let filename = "metaforecasts.json";
|
||||
fs.writeFileSync(filename, string);
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
import "dotenv/config";
|
||||
|
||||
import { readWritePool } from "../database/pg-wrapper";
|
||||
import { platforms } from "../platforms";
|
||||
|
||||
const migrate = async () => {
|
||||
const client = await readWritePool.connect();
|
||||
|
@ -11,6 +10,25 @@ const migrate = async () => {
|
|||
await client.query(q);
|
||||
};
|
||||
|
||||
const platformTitleToName = {
|
||||
Betfair: "betfair",
|
||||
FantasySCOTUS: "fantasyscotus",
|
||||
Foretold: "foretold",
|
||||
"GiveWell/OpenPhilanthropy": "givewellopenphil",
|
||||
"Good Judgment": "goodjudgement",
|
||||
"Good Judgment Open": "goodjudgmentopen",
|
||||
Infer: "infer",
|
||||
Kalshi: "kalshi",
|
||||
"Manifold Markets": "manifold",
|
||||
Metaculus: "metaculus",
|
||||
"Peter Wildeford": "wildeford",
|
||||
PolyMarket: "polymarket",
|
||||
PredictIt: "predictit",
|
||||
Rootclaim: "rootclaim",
|
||||
Smarkets: "smarkets",
|
||||
"X-risk estimates": "xrisk",
|
||||
};
|
||||
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
const copyTable = async (from: string, to: string) => {
|
||||
|
@ -19,13 +37,49 @@ const migrate = async () => {
|
|||
await execQuery(`INSERT INTO ${to} SELECT * FROM ${from}`);
|
||||
};
|
||||
|
||||
for (const platform of platforms) {
|
||||
await copyTable(`latest.${platform.name}`, platform.name);
|
||||
}
|
||||
await copyTable("latest.dashboards", "dashboards");
|
||||
await copyTable("latest.combined", "combined");
|
||||
await copyTable("latest.combined", "questions");
|
||||
await copyTable("latest.frontpage", "frontpage");
|
||||
await copyTable("history.h2022", "history");
|
||||
|
||||
for (const [title, name] of Object.entries(platformTitleToName)) {
|
||||
console.log(`Updating ${title} -> ${name}`);
|
||||
for (const table of ["questions", "history"]) {
|
||||
await client.query(
|
||||
`UPDATE ${table} SET platform=$1 WHERE platform=$2`,
|
||||
[name, title]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Fixing GJOpen ids in questions and history");
|
||||
for (const table of ["questions", "history"]) {
|
||||
await client.query(
|
||||
`UPDATE ${table} SET id=REPLACE(id, 'goodjudmentopen-', 'goodjudgmentopen-') WHERE id LIKE 'goodjudmentopen-%'`
|
||||
);
|
||||
}
|
||||
|
||||
const fixId = (id: string) =>
|
||||
id.replace("goodjudmentopen-", "goodjudgmentopen-");
|
||||
|
||||
console.log(
|
||||
"Please rebuild frontpage manually - current version includes invalid GJOpen and xrisk ids"
|
||||
);
|
||||
|
||||
const updateDashboards = async () => {
|
||||
const res = await client.query("SELECT id, contents FROM dashboards");
|
||||
for (const row of res.rows) {
|
||||
let { id, contents } = row;
|
||||
contents = contents.map(fixId);
|
||||
await client.query(
|
||||
"UPDATE dashboards SET contents = $1 WHERE id = $2",
|
||||
[JSON.stringify(contents), id]
|
||||
);
|
||||
}
|
||||
};
|
||||
console.log("Updating dashboards");
|
||||
await updateDashboards();
|
||||
|
||||
await client.query("COMMIT");
|
||||
} catch (e) {
|
||||
await client.query("ROLLBACK");
|
||||
|
|
|
@ -5,6 +5,8 @@ import https from "https";
|
|||
import { calculateStars } from "../utils/stars";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
const platformName = "betfair";
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
|
||||
|
||||
|
@ -121,12 +123,14 @@ async function processPredictions(data) {
|
|||
id: id,
|
||||
title: title,
|
||||
url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`,
|
||||
platform: "Betfair",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Betfair", { volume: prediction.totalMatched }),
|
||||
stars: calculateStars(platformName, {
|
||||
volume: prediction.totalMatched,
|
||||
}),
|
||||
volume: prediction.totalMatched,
|
||||
},
|
||||
};
|
||||
|
@ -136,7 +140,9 @@ async function processPredictions(data) {
|
|||
}
|
||||
|
||||
export const betfair: Platform = {
|
||||
name: "betfair",
|
||||
name: platformName,
|
||||
label: "Betfair",
|
||||
color: "#3d674a",
|
||||
async fetcher() {
|
||||
const data = await fetchPredictions();
|
||||
const results = await processPredictions(data); // somehow needed
|
||||
|
|
|
@ -23,7 +23,7 @@ async function fetchData() {
|
|||
|
||||
async function processPredictions(predictions) {
|
||||
let results = await predictions.map((prediction) => {
|
||||
let id = `platform-${prediction.id}`;
|
||||
let id = `example-${prediction.id}`;
|
||||
let probability = prediction.probability;
|
||||
let options = [
|
||||
{
|
||||
|
@ -61,6 +61,8 @@ async function processPredictions(predictions) {
|
|||
|
||||
export const example: Platform = {
|
||||
name: "example",
|
||||
label: "Example platform",
|
||||
color: "#ff0000",
|
||||
async fetcher() {
|
||||
let data = await fetchData();
|
||||
let results = await processPredictions(data); // somehow needed
|
||||
|
|
|
@ -2,7 +2,9 @@
|
|||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
const platformName = "fantasyscotus";
|
||||
|
||||
/* Definitions */
|
||||
let unixtime = new Date().getTime();
|
||||
|
@ -65,19 +67,19 @@ async function processData(data) {
|
|||
let historicalPercentageCorrect = data.stats.pcnt_correct;
|
||||
let historicalProbabilityCorrect =
|
||||
Number(historicalPercentageCorrect.replace("%", "")) / 100;
|
||||
let results = [];
|
||||
let results: Forecast[] = [];
|
||||
for (let event of events) {
|
||||
if (event.accuracy == "") {
|
||||
let id = `fantasyscotus-${event.id}`;
|
||||
let id = `${platformName}-${event.id}`;
|
||||
// if the thing hasn't already resolved
|
||||
let predictionData = await getPredictionsData(event.docket_url);
|
||||
let pAffirm = predictionData.proportionAffirm;
|
||||
//let trackRecord = event.prediction.includes("Affirm") ? historicalProbabilityCorrect : 1-historicalProbabilityCorrect
|
||||
let eventObject = {
|
||||
let eventObject: Forecast = {
|
||||
id: id,
|
||||
title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`,
|
||||
url: `https://fantasyscotus.net/user-predictions${event.docket_url}`,
|
||||
platform: "FantasySCOTUS",
|
||||
platform: platformName,
|
||||
description: `${(pAffirm * 100).toFixed(2)}% (${
|
||||
predictionData.numAffirm
|
||||
} out of ${
|
||||
|
@ -100,7 +102,7 @@ async function processData(data) {
|
|||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number(predictionData.numForecasts),
|
||||
stars: calculateStars("FantasySCOTUS", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
};
|
||||
results.push(eventObject);
|
||||
|
@ -112,7 +114,9 @@ async function processData(data) {
|
|||
|
||||
/* Body */
|
||||
export const fantasyscotus: Platform = {
|
||||
name: "fantasyscotus",
|
||||
name: platformName,
|
||||
label: "FantasySCOTUS",
|
||||
color: "#231149",
|
||||
async fetcher() {
|
||||
let rawData = await fetchData();
|
||||
let results = await processData(rawData);
|
||||
|
|
|
@ -5,6 +5,9 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
|
||||
const platformName = "foretold";
|
||||
|
||||
let graphQLendpoint = "https://api.foretold.io/graphql";
|
||||
let highQualityCommunities = [
|
||||
"0104d8e8-07e4-464b-8b32-74ef22b49f21",
|
||||
|
@ -54,7 +57,9 @@ async function fetchAllCommunityQuestions(communityId) {
|
|||
}
|
||||
|
||||
export const foretold: Platform = {
|
||||
name: "foretold",
|
||||
name: platformName,
|
||||
label: "Foretold",
|
||||
color: "#62520b",
|
||||
async fetcher() {
|
||||
let results = [];
|
||||
for (let community of highQualityCommunities) {
|
||||
|
@ -62,7 +67,7 @@ export const foretold: Platform = {
|
|||
questions = questions.map((question) => question.node);
|
||||
questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
|
||||
questions.forEach((question) => {
|
||||
let id = `foretold-${question.id}`;
|
||||
let id = `${platformName}-${question.id}`;
|
||||
let options = [];
|
||||
if (question.valueType == "PERCENTAGE") {
|
||||
let probability = question.previousAggregate.value.percentage;
|
||||
|
@ -83,13 +88,13 @@ export const foretold: Platform = {
|
|||
id: id,
|
||||
title: question.name,
|
||||
url: `https://www.foretold.io/c/${community}/m/${question.id}`,
|
||||
platform: "Foretold",
|
||||
platform: platformName,
|
||||
description: "",
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Math.floor(Number(question.measurementCount) / 2),
|
||||
stars: calculateStars("Foretold", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
/*liquidity: liquidity.toFixed(2),
|
||||
tradevolume: tradevolume.toFixed(2),
|
||||
|
|
|
@ -5,6 +5,8 @@ import fs from "fs";
|
|||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
|
||||
const platformName = "givewellopenphil";
|
||||
|
||||
/* Support functions */
|
||||
async function fetchPage(url: string) {
|
||||
let response = await axios({
|
||||
|
@ -48,11 +50,11 @@ async function main1() {
|
|||
let result = {
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "GiveWell",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("GiveWell/OpenPhilanthropy", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
}; // Note: This requires some processing afterwards
|
||||
// console.log(result)
|
||||
|
@ -65,7 +67,9 @@ async function main1() {
|
|||
}
|
||||
|
||||
export const givewellopenphil: Platform = {
|
||||
name: "givewellopenphil",
|
||||
name: platformName,
|
||||
label: "GiveWell/OpenPhilanthropy",
|
||||
color: "#32407e",
|
||||
async fetcher() {
|
||||
// main1()
|
||||
return; // not necessary to refill the DB every time
|
||||
|
|
|
@ -8,6 +8,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "goodjudgment";
|
||||
let endpoint = "https://goodjudgment.io/superforecasts/";
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
|
@ -15,7 +16,9 @@ String.prototype.replaceAll = function replaceAll(search, replace) {
|
|||
|
||||
/* Body */
|
||||
export const goodjudgment: Platform = {
|
||||
name: "goodjudgment",
|
||||
name: platformName,
|
||||
label: "Good Judgment",
|
||||
color: "#7d4f1b",
|
||||
async fetcher() {
|
||||
// Proxy fuckery
|
||||
let proxy;
|
||||
|
@ -64,7 +67,7 @@ export const goodjudgment: Platform = {
|
|||
let title = table[0]["0"].split("\t\t\t").splice(3)[0];
|
||||
if (title != undefined) {
|
||||
title = title.replaceAll("</a>", "");
|
||||
let id = `goodjudgment-${hash(title)}`;
|
||||
let id = `${platformName}-${hash(title)}`;
|
||||
let description = table
|
||||
.filter((row) => row["0"].includes("BACKGROUND:"))
|
||||
.map((row) => row["0"])
|
||||
|
@ -101,12 +104,12 @@ export const goodjudgment: Platform = {
|
|||
id: id,
|
||||
title: title,
|
||||
url: endpoint,
|
||||
platform: "Good Judgment",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Good Judgment", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
extra: {
|
||||
superforecastercommentary: analysis || "",
|
||||
|
|
|
@ -8,6 +8,8 @@ import toMarkdown from "../utils/toMarkdown";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "goodjudgmentopen";
|
||||
|
||||
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
|
||||
let annoyingPromptUrls = [
|
||||
"https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
|
||||
|
@ -185,12 +187,12 @@ async function goodjudgmentopen_inner(cookie) {
|
|||
}
|
||||
let questionNumRegex = new RegExp("questions/([0-9]+)");
|
||||
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
|
||||
let id = `goodjudmentopen-${questionNum}`;
|
||||
let id = `${platformName}-${questionNum}`;
|
||||
let question = {
|
||||
id: id,
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "Good Judgment Open",
|
||||
platform: platformName,
|
||||
...moreinfo,
|
||||
};
|
||||
if (j % 30 == 0 || DEBUG_MODE == "on") {
|
||||
|
@ -236,8 +238,10 @@ async function goodjudgmentopen_inner(cookie) {
|
|||
return results;
|
||||
}
|
||||
|
||||
export const goodjudmentopen: Platform = {
|
||||
name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen`
|
||||
export const goodjudgmentopen: Platform = {
|
||||
name: platformName,
|
||||
label: "Good Judgment Open",
|
||||
color: "#002455",
|
||||
async fetcher() {
|
||||
let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
|
||||
return await applyIfSecretExists(cookie, goodjudgmentopen_inner);
|
|
@ -4,10 +4,10 @@ import { fantasyscotus } from "./fantasyscotus";
|
|||
import { foretold } from "./foretold";
|
||||
import { givewellopenphil } from "./givewellopenphil";
|
||||
import { goodjudgment } from "./goodjudgment";
|
||||
import { goodjudmentopen } from "./goodjudmentopen";
|
||||
import { goodjudgmentopen } from "./goodjudgmentopen";
|
||||
import { infer } from "./infer";
|
||||
import { kalshi } from "./kalshi";
|
||||
import { manifoldmarkets } from "./manifoldmarkets";
|
||||
import { manifold } from "./manifold";
|
||||
import { metaculus } from "./metaculus";
|
||||
import { polymarket } from "./polymarket";
|
||||
import { predictit } from "./predictit";
|
||||
|
@ -67,7 +67,9 @@ export interface Forecast {
|
|||
export type PlatformFetcher = () => Promise<Forecast[] | null>;
|
||||
|
||||
export interface Platform {
|
||||
name: string;
|
||||
name: string; // short name for ids and `platform` db column, e.g. "xrisk"
|
||||
label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates"
|
||||
color: string; // used on frontend
|
||||
fetcher?: PlatformFetcher;
|
||||
}
|
||||
|
||||
|
@ -92,10 +94,10 @@ export const platforms: Platform[] = [
|
|||
foretold,
|
||||
givewellopenphil,
|
||||
goodjudgment,
|
||||
goodjudmentopen,
|
||||
goodjudgmentopen,
|
||||
infer,
|
||||
kalshi,
|
||||
manifoldmarkets,
|
||||
manifold,
|
||||
metaculus,
|
||||
polymarket,
|
||||
predictit,
|
||||
|
@ -114,8 +116,8 @@ export const processPlatform = async (platform: Platform) => {
|
|||
if (results && results.length) {
|
||||
await pgUpsert({
|
||||
contents: results,
|
||||
tableName: platform.name,
|
||||
replace: true,
|
||||
tableName: "questions",
|
||||
replacePlatform: platform.name,
|
||||
});
|
||||
console.log("Done");
|
||||
} else {
|
||||
|
|
|
@ -8,6 +8,7 @@ import toMarkdown from "../utils/toMarkdown";
|
|||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "infer";
|
||||
let htmlEndPoint = "https://www.infer-pub.com/questions";
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
|
@ -145,7 +146,7 @@ async function fetchStats(questionUrl, cookie) {
|
|||
qualityindicators: {
|
||||
numforecasts: Number(numforecasts),
|
||||
numforecasters: Number(numforecasters),
|
||||
stars: calculateStars("Infer", { numforecasts }),
|
||||
stars: calculateStars(platformName, { numforecasts }),
|
||||
},
|
||||
};
|
||||
|
||||
|
@ -218,7 +219,7 @@ async function infer_inner(cookie) {
|
|||
let moreinfo = await fetchStats(url, cookie);
|
||||
let questionNumRegex = new RegExp("questions/([0-9]+)");
|
||||
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
|
||||
let id = `infer-${questionNum}`;
|
||||
let id = `${platformName}-${questionNum}`;
|
||||
let question = {
|
||||
id: id,
|
||||
title: title,
|
||||
|
@ -278,7 +279,9 @@ async function infer_inner(cookie) {
|
|||
}
|
||||
|
||||
export const infer: Platform = {
|
||||
name: "infer",
|
||||
name: platformName,
|
||||
label: "Infer",
|
||||
color: "#223900",
|
||||
async fetcher() {
|
||||
let cookie = process.env.INFER_COOKIE;
|
||||
return await applyIfSecretExists(cookie, infer_inner);
|
||||
|
|
|
@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "kalshi";
|
||||
let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
|
||||
|
||||
async function fetchAllMarkets() {
|
||||
|
@ -34,17 +35,17 @@ async function processMarkets(markets) {
|
|||
type: "PROBABILITY",
|
||||
},
|
||||
];
|
||||
let id = `kalshi-${market.id}`;
|
||||
let id = `${platformName}-${market.id}`;
|
||||
let result = {
|
||||
id: id,
|
||||
title: market.title.replaceAll("*", ""),
|
||||
url: `https://kalshi.com/markets/${market.ticker_name}`,
|
||||
platform: "Kalshi",
|
||||
platform: platformName,
|
||||
description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Kalshi", {
|
||||
stars: calculateStars(platformName, {
|
||||
shares_volume: market.volume,
|
||||
interest: market.open_interest,
|
||||
}),
|
||||
|
@ -70,7 +71,9 @@ async function processMarkets(markets) {
|
|||
}
|
||||
|
||||
export const kalshi: Platform = {
|
||||
name: "kalshi",
|
||||
name: platformName,
|
||||
label: "Kalshi",
|
||||
color: "#615691",
|
||||
fetcher: async function () {
|
||||
let markets = await fetchAllMarkets();
|
||||
return await processMarkets(markets);
|
||||
|
|
|
@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "manifold";
|
||||
let endpoint = "https://manifold.markets/api/v0/markets";
|
||||
// See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5
|
||||
|
||||
|
@ -44,7 +45,7 @@ function showStatistics(results) {
|
|||
|
||||
async function processPredictions(predictions) {
|
||||
let results = await predictions.map((prediction) => {
|
||||
let id = `manifold-${prediction.id}`;
|
||||
let id = `manifold-${prediction.id}`; // oops, doesn't match platform name
|
||||
let probability = prediction.probability;
|
||||
let options = [
|
||||
{
|
||||
|
@ -67,7 +68,7 @@ async function processPredictions(predictions) {
|
|||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Manifold Markets", {
|
||||
stars: calculateStars(platformName, {
|
||||
volume7Days: prediction.volume7Days,
|
||||
volume24Hours: prediction.volume24Hours,
|
||||
pool: prediction.pool,
|
||||
|
@ -88,8 +89,10 @@ async function processPredictions(predictions) {
|
|||
return unresolvedResults; //resultsProcessed
|
||||
}
|
||||
|
||||
export const manifoldmarkets: Platform = {
|
||||
name: "manifoldmarkets",
|
||||
export const manifold: Platform = {
|
||||
name: platformName,
|
||||
label: "Manifold Markets",
|
||||
color: "#793466",
|
||||
async fetcher() {
|
||||
let data = await fetchData();
|
||||
let results = await processPredictions(data); // somehow needed
|
|
@ -6,6 +6,7 @@ import toMarkdown from "../utils/toMarkdown";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "metaculus";
|
||||
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
|
||||
let now = new Date().toISOString();
|
||||
let DEBUG_MODE = "off";
|
||||
|
@ -94,7 +95,9 @@ async function fetchMetaculusQuestionDescription(slug) {
|
|||
}
|
||||
|
||||
export const metaculus: Platform = {
|
||||
name: "metaculus",
|
||||
name: platformName,
|
||||
label: "Metaculus",
|
||||
color: "#006669",
|
||||
async fetcher() {
|
||||
// let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
|
||||
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
|
||||
|
@ -144,18 +147,18 @@ export const metaculus: Platform = {
|
|||
},
|
||||
];
|
||||
}
|
||||
let id = `metaculus-${result.id}`;
|
||||
let id = `${platformName}-${result.id}`;
|
||||
let interestingInfo = {
|
||||
id: id,
|
||||
title: result.title,
|
||||
url: "https://www.metaculus.com" + result.page_url,
|
||||
platform: "Metaculus",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number(result.number_of_predictions),
|
||||
stars: calculateStars("Metaculus", {
|
||||
stars: calculateStars(platformName, {
|
||||
numforecasts: result.number_of_predictions,
|
||||
}),
|
||||
},
|
||||
|
|
|
@ -5,6 +5,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "polymarket";
|
||||
let graphQLendpoint =
|
||||
"https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
|
||||
let units = 10 ** 6;
|
||||
|
@ -63,7 +64,9 @@ async function fetchIndividualContractData(marketMakerAddress) {
|
|||
}
|
||||
|
||||
export const polymarket: Platform = {
|
||||
name: "polymarket",
|
||||
name: platformName,
|
||||
label: "PolyMarket",
|
||||
color: "#00314e",
|
||||
async fetcher() {
|
||||
let results: Forecast[] = [];
|
||||
let webpageEndpointData = await fetchAllContractInfo();
|
||||
|
@ -79,7 +82,7 @@ export const polymarket: Platform = {
|
|||
);
|
||||
if (moreMarketAnswer.length > 0) {
|
||||
let moreMarketInfo = moreMarketAnswer[0];
|
||||
let id = `polymarket-${addressLowerCase.slice(0, 10)}`;
|
||||
let id = `${platformName}-${addressLowerCase.slice(0, 10)}`;
|
||||
// console.log(id);
|
||||
let numforecasts = Number(moreMarketInfo.tradesQuantity);
|
||||
let tradevolume =
|
||||
|
@ -103,7 +106,7 @@ export const polymarket: Platform = {
|
|||
id: id,
|
||||
title: marketInfo.question,
|
||||
url: "https://polymarket.com/market/" + marketInfo.slug,
|
||||
platform: "PolyMarket",
|
||||
platform: platformName,
|
||||
description: marketInfo.description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
|
@ -111,7 +114,7 @@ export const polymarket: Platform = {
|
|||
numforecasts: numforecasts.toFixed(0),
|
||||
liquidity: liquidity.toFixed(2),
|
||||
tradevolume: tradevolume.toFixed(2),
|
||||
stars: calculateStars("Polymarket", {
|
||||
stars: calculateStars(platformName, {
|
||||
liquidity,
|
||||
option: options[0],
|
||||
volume: tradevolume,
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import { Platform } from "./";
|
||||
|
||||
const platformName = "predictit";
|
||||
|
||||
/* Support functions */
|
||||
async function fetchmarkets() {
|
||||
let response = await axios({
|
||||
|
@ -39,7 +40,9 @@ function sleep(ms: number) {
|
|||
|
||||
/* Body */
|
||||
export const predictit: Platform = {
|
||||
name: "predictit",
|
||||
name: platformName,
|
||||
label: "PredictIt",
|
||||
color: "#460c00",
|
||||
async fetcher() {
|
||||
let markets = await fetchmarkets();
|
||||
let marketVolumes = await fetchmarketvolumes();
|
||||
|
@ -53,7 +56,7 @@ export const predictit: Platform = {
|
|||
let results = [];
|
||||
for (let market of markets) {
|
||||
// console.log(market.name)
|
||||
let id = `predictit-${market.id}`;
|
||||
let id = `${platformName}-${market.id}`;
|
||||
let isbinary = market.contracts.length == 1;
|
||||
await sleep(3000 * (1 + Math.random()));
|
||||
let descriptionraw = await fetchmarketrules(market.id);
|
||||
|
@ -97,12 +100,12 @@ export const predictit: Platform = {
|
|||
id: id,
|
||||
title: market["name"],
|
||||
url: market.url,
|
||||
platform: "PredictIt",
|
||||
platform: platformName,
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("PredictIt", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
shares_volume: shares_volume,
|
||||
},
|
||||
};
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import { Platform } from "./";
|
||||
import { Forecast, Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "rootclaim";
|
||||
let jsonEndpoint =
|
||||
"https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
|
||||
|
||||
|
@ -24,12 +24,14 @@ async function fetchAllRootclaims() {
|
|||
}
|
||||
|
||||
export const rootclaim: Platform = {
|
||||
name: "rootclaim",
|
||||
name: platformName,
|
||||
label: "Rootclaim",
|
||||
color: "#0d1624",
|
||||
async fetcher() {
|
||||
let claims = await fetchAllRootclaims();
|
||||
let results = [];
|
||||
let results: Forecast[] = [];
|
||||
for (let claim of claims) {
|
||||
let id = `rootclaim-${claim.slug.toLowerCase()}`;
|
||||
let id = `${platformName}-${claim.slug.toLowerCase()}`;
|
||||
let options = [];
|
||||
for (let scenario of claim.scenarios) {
|
||||
//console.log(scenario)
|
||||
|
@ -42,17 +44,17 @@ export const rootclaim: Platform = {
|
|||
});
|
||||
}
|
||||
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
|
||||
let obj = {
|
||||
let obj: Forecast = {
|
||||
id: id,
|
||||
title: toMarkdown(claim.question).replace("\n", ""),
|
||||
url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
|
||||
platform: "Rootclaim",
|
||||
platform: platformName,
|
||||
description: toMarkdown(claim.background).replace("'", "'"),
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: 1,
|
||||
stars: calculateStars("Rootclaim", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
};
|
||||
results.push(obj);
|
||||
|
|
|
@ -1,13 +1,14 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "smarkets";
|
||||
let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
|
||||
let VERBOSE = false;
|
||||
let empty = () => 0;
|
||||
|
||||
/* Support functions */
|
||||
|
||||
async function fetchEvents(url) {
|
||||
|
@ -60,7 +61,9 @@ async function fetchPrices(marketid) {
|
|||
}
|
||||
|
||||
export const smarkets: Platform = {
|
||||
name: "smarkets",
|
||||
name: platformName,
|
||||
label: "Smarkets",
|
||||
color: "#6f5b41",
|
||||
async fetcher() {
|
||||
let htmlPath =
|
||||
"?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
|
||||
|
@ -93,7 +96,7 @@ export const smarkets: Platform = {
|
|||
for (let market of markets) {
|
||||
VERBOSE ? console.log("================") : empty();
|
||||
VERBOSE ? console.log("Market: ", market) : empty();
|
||||
let id = `smarkets-${market.id}`;
|
||||
let id = `${platformName}-${market.id}`;
|
||||
let name = market.name;
|
||||
|
||||
let contracts = await fetchContracts(market.id);
|
||||
|
@ -160,12 +163,12 @@ export const smarkets: Platform = {
|
|||
id: id,
|
||||
title: name,
|
||||
url: "https://smarkets.com/event/" + market.event_id + market.slug,
|
||||
platform: "Smarkets",
|
||||
platform: platformName,
|
||||
description: market.description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Smarkets", {}),
|
||||
stars: calculateStars(platformName, {}),
|
||||
},
|
||||
};
|
||||
VERBOSE ? console.log(result) : empty();
|
||||
|
|
|
@ -7,6 +7,7 @@ import { calculateStars } from "../utils/stars";
|
|||
import { Platform } from "./";
|
||||
|
||||
/* Definitions */
|
||||
const platformName = "wildeford";
|
||||
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
|
||||
const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0`;
|
||||
// https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0
|
||||
|
@ -73,7 +74,7 @@ async function processPredictions(predictions) {
|
|||
);
|
||||
let results = currentPredictions.map((prediction) => {
|
||||
let title = prediction["Prediction"].replace(" [update]", "");
|
||||
let id = `wildeford-${hash(title)}`;
|
||||
let id = `${platformName}-${hash(title)}`;
|
||||
let probability = Number(prediction["Odds"].replace("%", "")) / 100;
|
||||
let options = [
|
||||
{
|
||||
|
@ -91,14 +92,14 @@ async function processPredictions(predictions) {
|
|||
id: id,
|
||||
title: title,
|
||||
url: prediction["url"],
|
||||
platform: "Peter Wildeford",
|
||||
platform: platformName,
|
||||
description: prediction["Notes"] || "",
|
||||
options: options,
|
||||
timestamp: new Date(
|
||||
Date.parse(prediction["Prediction Date"] + "Z")
|
||||
).toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Peter Wildeford", null),
|
||||
stars: calculateStars(platformName, null),
|
||||
},
|
||||
};
|
||||
return result;
|
||||
|
@ -120,7 +121,9 @@ export async function wildeford_inner(google_api_key) {
|
|||
}
|
||||
|
||||
export const wildeford: Platform = {
|
||||
name: "wildeford",
|
||||
name: platformName,
|
||||
label: "Peter Wildeford",
|
||||
color: "#984158",
|
||||
async fetcher() {
|
||||
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
|
||||
return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);
|
||||
|
|
|
@ -1,15 +1,25 @@
|
|||
import fs from "fs";
|
||||
|
||||
import { hash } from "../utils/hash";
|
||||
import { Platform } from "./";
|
||||
|
||||
const platformName = "xrisk";
|
||||
|
||||
export const xrisk: Platform = {
|
||||
name: "xrisk",
|
||||
label: "X-risk estimates",
|
||||
color: "#272600",
|
||||
async fetcher() {
|
||||
return; // not necessary to refill the DB every time
|
||||
// return; // not necessary to refill the DB every time
|
||||
let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {
|
||||
encoding: "utf-8",
|
||||
});
|
||||
const results = JSON.parse(fileRaw);
|
||||
let results = JSON.parse(fileRaw);
|
||||
results = results.map((item) => ({
|
||||
...item,
|
||||
id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
|
||||
platform: platformName,
|
||||
}));
|
||||
return results;
|
||||
},
|
||||
};
|
||||
|
|
|
@ -1,34 +1,13 @@
|
|||
import algoliasearch from "algoliasearch";
|
||||
|
||||
import { pgReadWithReadCredentials } from "../database/pg-wrapper";
|
||||
import { mergeEverythingInner } from "../flow/mergeEverything";
|
||||
import { platforms } from "../platforms";
|
||||
|
||||
let cookie = process.env.ALGOLIA_MASTER_API_KEY;
|
||||
const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
|
||||
const client = algoliasearch(algoliaAppId, cookie);
|
||||
const index = client.initIndex("metaforecast");
|
||||
|
||||
export async function rebuildAlgoliaDatabaseTheHardWay() {
|
||||
console.log("Doing this the hard way");
|
||||
let records = await mergeEverythingInner();
|
||||
records = records.map((record, index: number) => ({
|
||||
...record,
|
||||
has_numforecasts: record.numforecasts ? true : false,
|
||||
objectID: index,
|
||||
}));
|
||||
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
|
||||
|
||||
if (index.exists()) {
|
||||
console.log("Index exists");
|
||||
index
|
||||
.replaceAllObjects(records, { safe: true })
|
||||
.catch((error) => console.log(error));
|
||||
console.log(
|
||||
`Pushed ${records.length} records. Algolia will update asynchronously`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let getoptionsstringforsearch = (record: any) => {
|
||||
let result = "";
|
||||
if (!!record.options && record.options.length > 0) {
|
||||
|
@ -42,11 +21,16 @@ let getoptionsstringforsearch = (record: any) => {
|
|||
|
||||
export async function rebuildAlgoliaDatabaseTheEasyWay() {
|
||||
let records: any[] = await pgReadWithReadCredentials({
|
||||
tableName: "combined",
|
||||
tableName: "questions",
|
||||
});
|
||||
|
||||
const platformNameToLabel = Object.fromEntries(
|
||||
platforms.map((platform) => [platform.name, platform.label])
|
||||
);
|
||||
|
||||
records = records.map((record, index: number) => ({
|
||||
...record,
|
||||
platformLabel: platformNameToLabel[record.platform] || record.platform,
|
||||
has_numforecasts: record.numforecasts ? true : false,
|
||||
objectID: index,
|
||||
optionsstringforsearch: getoptionsstringforsearch(record),
|
||||
|
@ -62,4 +46,4 @@ export async function rebuildAlgoliaDatabaseTheEasyWay() {
|
|||
}
|
||||
}
|
||||
|
||||
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay; //rebuildAlgoliaDatabaseTheHardWay
|
||||
export const rebuildAlgoliaDatabase = rebuildAlgoliaDatabaseTheEasyWay;
|
||||
|
|
|
@ -24,7 +24,7 @@ let main = async () => {
|
|||
"PredictIt",
|
||||
"Rootclaim",
|
||||
];
|
||||
let json = await pgReadWithReadCredentials({ tableName: "combined" });
|
||||
let json = await pgReadWithReadCredentials({ tableName: "questions" });
|
||||
console.log(json.length);
|
||||
//let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
|
||||
//console.log(uniquePlatforms)
|
||||
|
|
|
@ -26,7 +26,7 @@ let shuffleArray = (array) => {
|
|||