From fc9c222a444b25eb446bf731f5707958fadb5d0d Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Wed, 26 Oct 2022 13:44:14 +0100 Subject: [PATCH] fix: catch metaculus errors Current code isn't particularly resilient to API changes. --- src/backend/platforms/metaculus/api.ts | 12 +- src/backend/platforms/metaculus/index.ts | 194 +++++++++++++---------- 2 files changed, 113 insertions(+), 93 deletions(-) diff --git a/src/backend/platforms/metaculus/api.ts b/src/backend/platforms/metaculus/api.ts index d8e2ad8..9abbf0b 100644 --- a/src/backend/platforms/metaculus/api.ts +++ b/src/backend/platforms/metaculus/api.ts @@ -211,15 +211,17 @@ const fetchAndValidate = async ( url: string, validator: ValidateFunction ): Promise => { - console.log(url); + // console.log(url); const data = await fetchWithRetries(url); if (validator(data)) { return data; + }else{ + console.log(data) + throw new Error( + `Response validation for url ${url} failed: ` + + JSON.stringify(validator.errors, null, 4) + ); } - throw new Error( - `Response validation for url ${url} failed: ` + - JSON.stringify(validator.errors) - ); }; export async function fetchApiQuestions( diff --git a/src/backend/platforms/metaculus/index.ts b/src/backend/platforms/metaculus/index.ts index 8dbfd63..c383643 100644 --- a/src/backend/platforms/metaculus/index.ts +++ b/src/backend/platforms/metaculus/index.ts @@ -1,28 +1,27 @@ -import { FetchedQuestion, Platform } from ".."; -import { average } from "../../../utils"; -import { sleep } from "../../utils/sleep"; +import Error from "next/error"; +import {FetchedQuestion, Platform} from ".."; +import {average} from "../../../utils"; +import {sleep} from "../../utils/sleep"; import { ApiCommon, ApiMultipleQuestions, ApiPredictable, ApiQuestion, fetchApiQuestions, - fetchSingleApiQuestion, + fetchSingleApiQuestion } from "./api"; const platformName = "metaculus"; const now = new Date().toISOString(); const SLEEP_TIME = 1000; -async function apiQuestionToFetchedQuestions( - apiQuestion: ApiQuestion -): Promise { +async function apiQuestionToFetchedQuestions(apiQuestion: ApiQuestion): Promise { // one item can expand: // - to 0 questions if we don't want it; // - to 1 question if it's a simple forecast // - to multiple questions if it's a group (see https://github.com/quantified-uncertainty/metaforecast/pull/84 for details) - const skip = (q: ApiPredictable): boolean => { + const skip = (q : ApiPredictable) : boolean => { if (q.publish_time > now || now > q.resolve_time) { return true; } @@ -32,44 +31,44 @@ async function apiQuestionToFetchedQuestions( return false; }; - const buildFetchedQuestion = ( - q: ApiPredictable & ApiCommon - ): Omit => { - const isBinary = q.possibilities.type === "binary"; - let options: FetchedQuestion["options"] = []; - if (isBinary) { - const probability = q.community_prediction.full.q2; - if (probability !== undefined) { - options = [ - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability, - type: "PROBABILITY", - }, - ]; + const buildFetchedQuestion = (q : ApiPredictable & ApiCommon) : Omit < FetchedQuestion, + "url" | "description" | "title" > => { + const isBinary = q.possibilities.type === "binary"; + let options: FetchedQuestion["options"] = []; + if (isBinary) { + const probability = q.community_prediction.full.q2; + if (probability !== undefined) { + options = [ + { + name: "Yes", + probability: probability, + type: "PROBABILITY" + }, { + name: "No", + probability: 1 - probability, + type: "PROBABILITY" + }, + ]; + } } - } - return { - id: `${platformName}-${q.id}`, - options, - qualityindicators: { - numforecasts: q.number_of_predictions, - }, - extra: { - resolution_data: { - publish_time: apiQuestion.publish_time, - resolution: apiQuestion.resolution, - close_time: apiQuestion.close_time, - resolve_time: apiQuestion.resolve_time, + return { + id: `${platformName}-${ + q.id + }`, + options, + qualityindicators: { + numforecasts: q.number_of_predictions }, - }, + extra: { + resolution_data: { + publish_time: apiQuestion.publish_time, + resolution: apiQuestion.resolution, + close_time: apiQuestion.close_time, + resolve_time: apiQuestion.resolve_time + } + } + }; }; - }; if (apiQuestion.type === "group") { await sleep(SLEEP_TIME); @@ -77,44 +76,64 @@ async function apiQuestionToFetchedQuestions( if (apiQuestionDetails.type !== "group") { throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript } - return (apiQuestionDetails.sub_questions || []) - .filter((q) => !skip(q)) - .map((sq) => { - const tmp = buildFetchedQuestion(sq); - return { - ...tmp, - title: `${apiQuestion.title} (${sq.title})`, - description: apiQuestionDetails.description || "", - url: `https://www.metaculus.com${apiQuestion.page_url}?sub-question=${sq.id}`, - }; - }); + try{ + let result = (apiQuestionDetails.sub_questions || []).filter((q) => ! skip(q)).map((sq) => { + const tmp = buildFetchedQuestion(sq); + return { + ... tmp, + title: `${ + apiQuestion.title + } (${ + sq.title + })`, + description: apiQuestionDetails.description || "", + url: `https://www.metaculus.com${ + apiQuestion.page_url + }?sub-question=${ + sq.id + }` + }; + }); + return result + }catch(error){ + console.log(error) + return [] + } } else if (apiQuestion.type === "forecast") { if (apiQuestion.group) { return []; // sub-question, should be handled on the group level } if (skip(apiQuestion)) { + console.log(`- [Skipping]: ${ + apiQuestion.title + }`) + /*console.log(`Close time: ${ + apiQuestion.close_time + }, resolve time: ${ + apiQuestion.resolve_time + }`)*/ return []; } await sleep(SLEEP_TIME); const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id); - const tmp = buildFetchedQuestion(apiQuestion); - return [ - { - ...tmp, - title: apiQuestion.title, - description: apiQuestionDetails.description || "", - url: "https://www.metaculus.com" + apiQuestion.page_url, - }, - ]; + try{ + const tmp = buildFetchedQuestion(apiQuestion); + return [{ + ... tmp, + title: apiQuestion.title, + description: apiQuestionDetails.description || "", + url: "https://www.metaculus.com" + apiQuestion.page_url + },]; + }catch(error){ + console.log(error) + return [] + } } else { - if (apiQuestion.type !== "claim") { - // should never happen, since `discriminator` in JTD schema causes a strict runtime check - console.log( - `Unknown metaculus question type: ${ - (apiQuestion as any).type - }, skipping` - ); + if (apiQuestion.type !== "claim") { // should never happen, since `discriminator` in JTD schema causes a strict runtime check + console.log(`Unknown metaculus question type: ${ + (apiQuestion as any).type + }, skipping`); } return []; } @@ -125,22 +144,22 @@ export const metaculus: Platform<"id" | "debug"> = { label: "Metaculus", color: "#006669", version: "v2", - fetcherArgs: ["id", "debug"], + fetcherArgs: [ + "id", "debug" + ], async fetcher(opts) { let allQuestions: FetchedQuestion[] = []; - if (opts.args?.id) { + if (opts.args ?. id) { + console.log("Using optional id arg.") const id = Number(opts.args.id); const apiQuestion = await fetchSingleApiQuestion(id); const questions = await apiQuestionToFetchedQuestions(apiQuestion); console.log(questions); - return { - questions, - partial: true, - }; + return {questions, partial: true}; } - let next: string | null = "https://www.metaculus.com/api2/questions/"; + let next: string |null = "https://www.metaculus.com/api2/questions/"; let i = 1; while (next) { console.log(`\nQuery #${i} - ${next}`); @@ -148,14 +167,17 @@ export const metaculus: Platform<"id" | "debug"> = { await sleep(SLEEP_TIME); const apiQuestions: ApiMultipleQuestions = await fetchApiQuestions(next); const results = apiQuestions.results; - + // console.log(results) let j = false; for (const result of results) { const questions = await apiQuestionToFetchedQuestions(result); + // console.log(questions) for (const question of questions) { - console.log(`- ${question.title}`); - if ((!j && i % 20 === 0) || opts.args?.debug) { + console.log(`- ${ + question.title + }`); + if ((! j && i % 20 === 0) || opts.args ?. debug) { console.log(question); j = true; } @@ -167,20 +189,16 @@ export const metaculus: Platform<"id" | "debug"> = { i += 1; } - return { - questions: allQuestions, - partial: false, - }; + return {questions: allQuestions, partial: false}; }, calculateStars(data) { - const { numforecasts } = data.qualityindicators; - const nuno = () => - (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2; + const {numforecasts} = data.qualityindicators; + const nuno = () => (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2; const eli = () => 3; const misha = () => 3; const starsDecimal = average([nuno(), eli(), misha()]); const starsInteger = Math.round(starsDecimal); return starsInteger; - }, + } };