feat: metaculus validates api, supports --id cli arg

This commit is contained in:
Vyacheslav Matyukhin 2022-05-19 13:39:53 +04:00
parent 159f9c2b45
commit 4d736f711d
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
3 changed files with 306 additions and 152 deletions

76
package-lock.json generated
View File

@ -28,6 +28,7 @@
"@types/textversionjs": "^1.1.1", "@types/textversionjs": "^1.1.1",
"@types/tunnel": "^0.0.3", "@types/tunnel": "^0.0.3",
"airtable": "^0.11.1", "airtable": "^0.11.1",
"ajv": "^8.11.0",
"algoliasearch": "^4.10.3", "algoliasearch": "^4.10.3",
"autoprefixer": "^10.1.0", "autoprefixer": "^10.1.0",
"axios": "^0.25.0", "axios": "^0.25.0",
@ -3498,6 +3499,21 @@
"node": ">=8.0.0" "node": ">=8.0.0"
} }
}, },
"node_modules/ajv": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
"integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==",
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/algoliasearch": { "node_modules/algoliasearch": {
"version": "4.10.3", "version": "4.10.3",
"resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz", "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
@ -5771,6 +5787,11 @@
"url": "https://github.com/sponsors/jaydenseric" "url": "https://github.com/sponsors/jaydenseric"
} }
}, },
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
},
"node_modules/fast-equals": { "node_modules/fast-equals": {
"version": "2.0.4", "version": "2.0.4",
"resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-2.0.4.tgz", "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-2.0.4.tgz",
@ -7454,6 +7475,11 @@
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/json-schema-traverse": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
},
"node_modules/json-stable-stringify": { "node_modules/json-stable-stringify": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
@ -37587,6 +37613,14 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/require-from-string": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/require-main-filename": { "node_modules/require-main-filename": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
@ -39027,6 +39061,14 @@
"tslib": "^2.0.3" "tslib": "^2.0.3"
} }
}, },
"node_modules/uri-js": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
"dependencies": {
"punycode": "^2.1.0"
}
},
"node_modules/url-parse-lax": { "node_modules/url-parse-lax": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",
@ -42453,6 +42495,17 @@
"node-fetch": "^2.6.7" "node-fetch": "^2.6.7"
} }
}, },
"ajv": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
"integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==",
"requires": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
}
},
"algoliasearch": { "algoliasearch": {
"version": "4.10.3", "version": "4.10.3",
"resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz", "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
@ -44096,6 +44149,11 @@
"resolved": "https://registry.npmjs.org/extract-files/-/extract-files-9.0.0.tgz", "resolved": "https://registry.npmjs.org/extract-files/-/extract-files-9.0.0.tgz",
"integrity": "sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==" "integrity": "sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ=="
}, },
"fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
},
"fast-equals": { "fast-equals": {
"version": "2.0.4", "version": "2.0.4",
"resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-2.0.4.tgz", "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-2.0.4.tgz",
@ -45304,6 +45362,11 @@
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="
}, },
"json-schema-traverse": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
},
"json-stable-stringify": { "json-stable-stringify": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
@ -67926,6 +67989,11 @@
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"dev": true "dev": true
}, },
"require-from-string": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="
},
"require-main-filename": { "require-main-filename": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
@ -68920,6 +68988,14 @@
"tslib": "^2.0.3" "tslib": "^2.0.3"
} }
}, },
"uri-js": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
"requires": {
"punycode": "^2.1.0"
}
},
"url-parse-lax": { "url-parse-lax": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",

View File

@ -46,6 +46,7 @@
"@types/textversionjs": "^1.1.1", "@types/textversionjs": "^1.1.1",
"@types/tunnel": "^0.0.3", "@types/tunnel": "^0.0.3",
"airtable": "^0.11.1", "airtable": "^0.11.1",
"ajv": "^8.11.0",
"algoliasearch": "^4.10.3", "algoliasearch": "^4.10.3",
"autoprefixer": "^10.1.0", "autoprefixer": "^10.1.0",
"axios": "^0.25.0", "axios": "^0.25.0",

View File

@ -1,4 +1,5 @@
/* Imports */ /* Imports */
import Ajv, { JTDDataType } from "ajv/dist/jtd";
import axios from "axios"; import axios from "axios";
import { average } from "../../utils"; import { average } from "../../utils";
@ -8,24 +9,87 @@ import { FetchedQuestion, Platform } from "./";
/* Definitions */ /* Definitions */
const platformName = "metaculus"; const platformName = "metaculus";
let now = new Date().toISOString(); const now = new Date().toISOString();
let DEBUG_MODE = "off"; const SLEEP_TIME = 5000;
let SLEEP_TIME = 5000;
/* Support functions */ const apiQuestionSchema = {
async function fetchMetaculusQuestions(next: string) { properties: {
// Numbers about a given address: how many, how much, at what price, etc. page_url: {
let response; type: "string",
let data; },
title: {
type: "string",
},
publish_time: {
type: "string",
},
close_time: {
type: "string",
},
resolve_time: {
type: "string",
},
number_of_predictions: {
type: "uint32",
},
possibilities: {
properties: {
type: {
type: "string", // TODO - enum?
},
},
additionalProperties: true,
},
community_prediction: {
properties: {
full: {
properties: {
q1: {
type: "float64",
},
q2: {
type: "float64",
},
q3: {
type: "float64",
},
},
additionalProperties: true,
},
},
additionalProperties: true,
},
},
additionalProperties: true,
} as const;
const apiMultipleQuestionsSchema = {
properties: {
results: {
elements: apiQuestionSchema,
},
next: {
type: "string",
nullable: true,
},
},
additionalProperties: true,
} as const;
type ApiQuestion = JTDDataType<typeof apiQuestionSchema>;
type ApiMultipleQuestions = JTDDataType<typeof apiMultipleQuestionsSchema>;
const validateApiQuestion = new Ajv().compile<ApiQuestion>(apiQuestionSchema);
const validateApiMultipleQuestions = new Ajv().compile<
JTDDataType<typeof apiMultipleQuestionsSchema>
>(apiMultipleQuestionsSchema);
async function fetchWithRetries<T = unknown>(url: string): Promise<T> {
try { try {
response = await axios({ const response = await axios.get<T>(url);
url: next, return response.data;
method: "GET",
headers: { "Content-Type": "application/json" },
});
data = response.data;
} catch (error) { } catch (error) {
console.log(`Error in async function fetchMetaculusQuestions(next)`); console.log(`Error while fetching ${url}`);
console.log(error); console.log(error);
if (axios.isAxiosError(error)) { if (axios.isAxiosError(error)) {
if (error.response?.headers["retry-after"]) { if (error.response?.headers["retry-after"]) {
@ -36,105 +100,72 @@ async function fetchMetaculusQuestions(next: string) {
await sleep(SLEEP_TIME); await sleep(SLEEP_TIME);
} }
} }
} finally {
try {
response = await axios({
url: next,
method: "GET",
headers: { "Content-Type": "application/json" },
});
data = response.data;
} catch (error) {
console.log(error);
return { results: [] };
} }
} const response = await axios.get<T>(url);
// console.log(response) return response.data;
}
/* Support functions */
async function fetchApiQuestions(next: string): Promise<ApiMultipleQuestions> {
const data = await fetchWithRetries<object>(next);
if (validateApiMultipleQuestions(data)) {
return data; return data;
}
throw new Error("Response validation failed");
} }
async function fetchMetaculusQuestionDescription(slug: string) { async function fetchSingleApiQuestion(url: string): Promise<ApiQuestion> {
try { const data = await fetchWithRetries<object>(url);
let response = await axios({ if (validateApiQuestion(data)) {
method: "get", return data;
url: "https://www.metaculus.com" + slug,
}).then((response) => response.data);
return response;
} catch (error) {
console.log(`Error in: fetchMetaculusQuestionDescription`);
console.log(
`We encountered some error when attempting to fetch a metaculus page. Trying again`
);
if (
axios.isAxiosError(error) &&
typeof error.response != "undefined" &&
typeof error.response.headers != "undefined" &&
typeof error.response.headers["retry-after"] != "undefined"
) {
const timeout = error.response.headers["retry-after"];
console.log(`Timeout: ${timeout}`);
await sleep(Number(timeout) * 1000 + SLEEP_TIME);
} else {
await sleep(SLEEP_TIME);
}
try {
let response = await axios({
method: "get",
url: "https://www.metaculus.com" + slug,
}).then((response) => response.data);
// console.log(response)
return response;
} catch (error) {
console.log(
`We encountered some error when attempting to fetch a metaculus page.`
);
console.log("Error", error);
throw "Giving up";
}
} }
throw new Error("Response validation failed");
} }
export const metaculus: Platform = { async function fetchQuestionHtml(slug: string) {
name: platformName, return await fetchWithRetries<string>("https://www.metaculus.com" + slug);
label: "Metaculus", }
color: "#006669",
version: "v1", async function fetchQuestionPage(slug: string) {
async fetcher() { const questionPage = await fetchQuestionHtml(slug);
// let metaculusQuestionsInit = await fetchMetaculusQuestions(1) const isPublicFigurePrediction = questionPage.includes(
// let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) "A public prediction by"
// console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
// for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test }
let all_questions = [];
let next = "https://www.metaculus.com/api2/questions/";
let i = 1;
while (next) {
if (i % 20 == 0) {
console.log("Sleeping for 500ms");
await sleep(SLEEP_TIME);
}
console.log(`\nQuery #${i}`);
let metaculusQuestions = await fetchMetaculusQuestions(next);
let results = metaculusQuestions.results;
let j = false;
for (let result of results) {
if (result.publish_time < now && now < result.resolve_time) {
await sleep(SLEEP_TIME / 2);
let questionPage = await fetchMetaculusQuestionDescription(
result.page_url
); );
if (!questionPage.includes("A public prediction by")) {
// console.log(questionPage) let description: string = "";
let descriptionraw = questionPage.split( if (!isPublicFigurePrediction) {
const descriptionraw = questionPage.split(
`<div class="content" ng-bind-html-compile="qctrl.question.description_html">` `<div class="content" ng-bind-html-compile="qctrl.question.description_html">`
)[1]; //.split(`<div class="question__content">`)[1] )[1];
let descriptionprocessed1 = descriptionraw.split("</div>")[0]; const descriptionprocessed1 = descriptionraw.split("</div>")[0];
let descriptionprocessed2 = toMarkdown(descriptionprocessed1); description = toMarkdown(descriptionprocessed1);
let description = descriptionprocessed2; }
let isbinary = result.possibilities.type == "binary"; return {
isPublicFigurePrediction,
description,
};
}
async function apiQuestionToFetchedQuestion(
apiQuestion: ApiQuestion
): Promise<FetchedQuestion | null> {
if (apiQuestion.publish_time > now || now > apiQuestion.resolve_time) {
return null;
}
await sleep(SLEEP_TIME / 2);
const questionPage = await fetchQuestionPage(apiQuestion.page_url);
if (questionPage.isPublicFigurePrediction) {
console.log("- [Skipping public prediction]");
return null;
}
const isBinary = apiQuestion.possibilities.type === "binary";
let options: FetchedQuestion["options"] = []; let options: FetchedQuestion["options"] = [];
if (isbinary) { if (isBinary) {
let probability = Number(result.community_prediction.full.q2); const probability = Number(apiQuestion.community_prediction.full.q2);
options = [ options = [
{ {
name: "Yes", name: "Yes",
@ -148,22 +179,21 @@ export const metaculus: Platform = {
}, },
]; ];
} }
let id = `${platformName}-${result.id}`; const question: FetchedQuestion = {
let interestingInfo: FetchedQuestion = { id: `${platformName}-${apiQuestion.id}`,
id, title: apiQuestion.title,
title: result.title, url: "https://www.metaculus.com" + apiQuestion.page_url,
url: "https://www.metaculus.com" + result.page_url, description: questionPage.description,
description,
options, options,
qualityindicators: { qualityindicators: {
numforecasts: Number(result.number_of_predictions), numforecasts: apiQuestion.number_of_predictions,
}, },
extra: { extra: {
resolution_data: { resolution_data: {
publish_time: result.publish_time, publish_time: apiQuestion.publish_time,
resolution: result.resolution, resolution: apiQuestion.resolution,
close_time: result.close_time, close_time: apiQuestion.close_time,
resolve_time: result.resolve_time, resolve_time: apiQuestion.resolve_time,
}, },
}, },
//"status": result.status, //"status": result.status,
@ -172,34 +202,81 @@ export const metaculus: Platform = {
//"type": result.possibilities.type, // We want binary ones here. //"type": result.possibilities.type, // We want binary ones here.
//"last_activity_time": result.last_activity_time, //"last_activity_time": result.last_activity_time,
}; };
if (Number(result.number_of_predictions) >= 10) { if (apiQuestion.number_of_predictions < 10) {
console.log(`- ${interestingInfo.title}`); return null;
all_questions.push(interestingInfo); }
if ((!j && i % 20 == 0) || DEBUG_MODE == "on") {
console.log(interestingInfo); return question;
}
export const metaculus: Platform<"id" | "debug"> = {
name: platformName,
label: "Metaculus",
color: "#006669",
version: "v2",
fetcherArgs: ["id", "debug"],
async fetcher(opts) {
let allQuestions: FetchedQuestion[] = [];
if (opts.args?.id) {
const apiQuestion = await fetchSingleApiQuestion(
`https://www.metaculus.com/api2/questions/${opts.args?.id}`
);
const question = await apiQuestionToFetchedQuestion(apiQuestion);
console.log(question);
return {
questions: question ? [question] : [],
partial: true,
};
}
let next: string | null = "https://www.metaculus.com/api2/questions/";
let i = 1;
while (next) {
if (i % 20 === 0) {
console.log("Sleeping for 500ms");
await sleep(SLEEP_TIME);
}
console.log(`\nQuery #${i}`);
const metaculusQuestions: ApiMultipleQuestions = await fetchApiQuestions(
next
);
const results = metaculusQuestions.results;
let j = false;
for (const result of results) {
const question = await apiQuestionToFetchedQuestion(result);
if (!question) {
continue;
}
console.log(`- ${question.title}`);
if ((!j && i % 20 === 0) || opts.args?.debug) {
console.log(question);
j = true; j = true;
} }
allQuestions.push(question);
} }
} else {
console.log("- [Skipping public prediction]");
}
}
}
next = metaculusQuestions.next; next = metaculusQuestions.next;
i = i + 1; i = i + 1;
} }
return all_questions; return {
questions: allQuestions,
partial: false,
};
}, },
calculateStars(data) { calculateStars(data) {
const { numforecasts } = data.qualityindicators; const { numforecasts } = data.qualityindicators;
let nuno = () => const nuno = () =>
(numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2; (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
let eli = () => 3; const eli = () => 3;
let misha = () => 3; const misha = () => 3;
let starsDecimal = average([nuno(), eli(), misha()]); const starsDecimal = average([nuno(), eli(), misha()]);
let starsInteger = Math.round(starsDecimal); const starsInteger = Math.round(starsDecimal);
return starsInteger; return starsInteger;
}, },
}; };