quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: catch metaculus errors

Browse Source 
Current code isn't particularly resilient
to API changes.
This commit is contained in:
NunoSempere 2022-10-26 13:44:14 +01:00
parent 83a01e6156
commit fc9c222a44
2 changed files with 113 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/backend/platforms/metaculus/api.ts
View File

@ -211,15 +211,17 @@ const fetchAndValidate = async <T = unknown>(
url: string, url: string,
validator: ValidateFunction<T> validator: ValidateFunction<T>
): Promise<T> => { ): Promise<T> => {
console.log(url); // console.log(url);
const data = await fetchWithRetries<object>(url); const data = await fetchWithRetries<object>(url);
if (validator(data)) { if (validator(data)) {
return data; return data;
}else{
console.log(data)
throw new Error(
`Response validation for url ${url} failed: ` +
JSON.stringify(validator.errors, null, 4)
);
} }
throw new Error(
`Response validation for url ${url} failed: ` +
JSON.stringify(validator.errors)
);
}; };
export async function fetchApiQuestions( export async function fetchApiQuestions(

194
src/backend/platforms/metaculus/index.ts
View File

@ -1,28 +1,27 @@
import { FetchedQuestion, Platform } from ".."; import Error from "next/error";
import { average } from "../../../utils"; import {FetchedQuestion, Platform} from "..";
import { sleep } from "../../utils/sleep"; import {average} from "../../../utils";
import {sleep} from "../../utils/sleep";
import { import {
ApiCommon, ApiCommon,
ApiMultipleQuestions, ApiMultipleQuestions,
ApiPredictable, ApiPredictable,
ApiQuestion, ApiQuestion,
fetchApiQuestions, fetchApiQuestions,
fetchSingleApiQuestion, fetchSingleApiQuestion
} from "./api"; } from "./api";
const platformName = "metaculus"; const platformName = "metaculus";
const now = new Date().toISOString(); const now = new Date().toISOString();
const SLEEP_TIME = 1000; const SLEEP_TIME = 1000;
async function apiQuestionToFetchedQuestions( async function apiQuestionToFetchedQuestions(apiQuestion: ApiQuestion): Promise<FetchedQuestion[]> {
apiQuestion: ApiQuestion
): Promise<FetchedQuestion[]> {
// one item can expand: // one item can expand:
// - to 0 questions if we don't want it; // - to 0 questions if we don't want it;
// - to 1 question if it's a simple forecast // - to 1 question if it's a simple forecast
// - to multiple questions if it's a group (see https://github.com/quantified-uncertainty/metaforecast/pull/84 for details) // - to multiple questions if it's a group (see https://github.com/quantified-uncertainty/metaforecast/pull/84 for details)
const skip = (q: ApiPredictable): boolean => { const skip = (q : ApiPredictable) : boolean => {
if (q.publish_time > now || now > q.resolve_time) { if (q.publish_time > now || now > q.resolve_time) {
return true; return true;
} }
@ -32,44 +31,44 @@ async function apiQuestionToFetchedQuestions(
return false; return false;
}; };
const buildFetchedQuestion = ( const buildFetchedQuestion = (q : ApiPredictable & ApiCommon) : Omit < FetchedQuestion,
q: ApiPredictable & ApiCommon "url" | "description" | "title" > => {
): Omit<FetchedQuestion, "url" | "description" | "title"> => { const isBinary = q.possibilities.type === "binary";
const isBinary = q.possibilities.type === "binary"; let options: FetchedQuestion["options"] = [];
let options: FetchedQuestion["options"] = []; if (isBinary) {
if (isBinary) { const probability = q.community_prediction.full.q2;
const probability = q.community_prediction.full.q2; if (probability !== undefined) {
if (probability !== undefined) { options = [
options = [ {
{ name: "Yes",
name: "Yes", probability: probability,
probability: probability, type: "PROBABILITY"
type: "PROBABILITY", }, {
}, name: "No",
{ probability: 1 - probability,
name: "No", type: "PROBABILITY"
probability: 1 - probability, },
type: "PROBABILITY", ];
}, }
];
} }
} return {
return { id: `${platformName}-${
id: `${platformName}-${q.id}`, q.id
options, }`,
qualityindicators: { options,
numforecasts: q.number_of_predictions, qualityindicators: {
}, numforecasts: q.number_of_predictions
extra: {
resolution_data: {
publish_time: apiQuestion.publish_time,
resolution: apiQuestion.resolution,
close_time: apiQuestion.close_time,
resolve_time: apiQuestion.resolve_time,
}, },
}, extra: {
resolution_data: {
publish_time: apiQuestion.publish_time,
resolution: apiQuestion.resolution,
close_time: apiQuestion.close_time,
resolve_time: apiQuestion.resolve_time
}
}
};
}; };
};
if (apiQuestion.type === "group") { if (apiQuestion.type === "group") {
await sleep(SLEEP_TIME); await sleep(SLEEP_TIME);
@ -77,44 +76,64 @@ async function apiQuestionToFetchedQuestions(
if (apiQuestionDetails.type !== "group") { if (apiQuestionDetails.type !== "group") {
throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript
} }
return (apiQuestionDetails.sub_questions || []) try{
.filter((q) => !skip(q)) let result = (apiQuestionDetails.sub_questions || []).filter((q) => ! skip(q)).map((sq) => {
.map((sq) => { const tmp = buildFetchedQuestion(sq);
const tmp = buildFetchedQuestion(sq); return {
return { ... tmp,
...tmp, title: `${
title: `${apiQuestion.title} (${sq.title})`, apiQuestion.title
description: apiQuestionDetails.description || "", } (${
url: `https://www.metaculus.com${apiQuestion.page_url}?sub-question=${sq.id}`, sq.title
}; })`,
}); description: apiQuestionDetails.description || "",
url: `https://www.metaculus.com${
apiQuestion.page_url
}?sub-question=${
sq.id
}`
};
});
return result
}catch(error){
console.log(error)
return []
}
} else if (apiQuestion.type === "forecast") { } else if (apiQuestion.type === "forecast") {
if (apiQuestion.group) { if (apiQuestion.group) {
return []; // sub-question, should be handled on the group level return []; // sub-question, should be handled on the group level
} }
if (skip(apiQuestion)) { if (skip(apiQuestion)) {
console.log(`- [Skipping]: ${
apiQuestion.title
}`)
/*console.log(`Close time: ${
apiQuestion.close_time
}, resolve time: ${
apiQuestion.resolve_time
}`)*/
return []; return [];
} }
await sleep(SLEEP_TIME); await sleep(SLEEP_TIME);
const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id); const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
const tmp = buildFetchedQuestion(apiQuestion); try{
return [ const tmp = buildFetchedQuestion(apiQuestion);
{ return [{
...tmp, ... tmp,
title: apiQuestion.title, title: apiQuestion.title,
description: apiQuestionDetails.description || "", description: apiQuestionDetails.description || "",
url: "https://www.metaculus.com" + apiQuestion.page_url, url: "https://www.metaculus.com" + apiQuestion.page_url
}, },];
]; }catch(error){
console.log(error)
return []
}
} else { } else {
if (apiQuestion.type !== "claim") { if (apiQuestion.type !== "claim") { // should never happen, since `discriminator` in JTD schema causes a strict runtime check
// should never happen, since `discriminator` in JTD schema causes a strict runtime check console.log(`Unknown metaculus question type: ${
console.log( (apiQuestion as any).type
`Unknown metaculus question type: ${ }, skipping`);
(apiQuestion as any).type
}, skipping`
);
} }
return []; return [];
} }
@ -125,22 +144,22 @@ export const metaculus: Platform<"id" | "debug"> = {
label: "Metaculus", label: "Metaculus",
color: "#006669", color: "#006669",
version: "v2", version: "v2",
fetcherArgs: ["id", "debug"], fetcherArgs: [
"id", "debug"
],
async fetcher(opts) { async fetcher(opts) {
let allQuestions: FetchedQuestion[] = []; let allQuestions: FetchedQuestion[] = [];
if (opts.args?.id) { if (opts.args ?. id) {
console.log("Using optional id arg.")
const id = Number(opts.args.id); const id = Number(opts.args.id);
const apiQuestion = await fetchSingleApiQuestion(id); const apiQuestion = await fetchSingleApiQuestion(id);
const questions = await apiQuestionToFetchedQuestions(apiQuestion); const questions = await apiQuestionToFetchedQuestions(apiQuestion);
console.log(questions); console.log(questions);
return { return {questions, partial: true};
questions,
partial: true,
};
} }
let next: string | null = "https://www.metaculus.com/api2/questions/"; let next: string |null = "https://www.metaculus.com/api2/questions/";
let i = 1; let i = 1;
while (next) { while (next) {
console.log(`\nQuery #${i} - ${next}`); console.log(`\nQuery #${i} - ${next}`);
@ -148,14 +167,17 @@ export const metaculus: Platform<"id" | "debug"> = {
await sleep(SLEEP_TIME); await sleep(SLEEP_TIME);
const apiQuestions: ApiMultipleQuestions = await fetchApiQuestions(next); const apiQuestions: ApiMultipleQuestions = await fetchApiQuestions(next);
const results = apiQuestions.results; const results = apiQuestions.results;
// console.log(results)
let j = false; let j = false;
for (const result of results) { for (const result of results) {
const questions = await apiQuestionToFetchedQuestions(result); const questions = await apiQuestionToFetchedQuestions(result);
// console.log(questions)
for (const question of questions) { for (const question of questions) {
console.log(`- ${question.title}`); console.log(`- ${
if ((!j && i % 20 === 0) || opts.args?.debug) { question.title
}`);
if ((! j && i % 20 === 0) || opts.args ?. debug) {
console.log(question); console.log(question);
j = true; j = true;
} }
@ -167,20 +189,16 @@ export const metaculus: Platform<"id" | "debug"> = {
i += 1; i += 1;
} }
return { return {questions: allQuestions, partial: false};
questions: allQuestions,
partial: false,
};
}, },
calculateStars(data) { calculateStars(data) {
const { numforecasts } = data.qualityindicators; const {numforecasts} = data.qualityindicators;
const nuno = () => const nuno = () => (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
(numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
const eli = () => 3; const eli = () => 3;
const misha = () => 3; const misha = () => 3;
const starsDecimal = average([nuno(), eli(), misha()]); const starsDecimal = average([nuno(), eli(), misha()]);
const starsInteger = Math.round(starsDecimal); const starsInteger = Math.round(starsDecimal);
return starsInteger; return starsInteger;
}, }
}; };