fix: catch metaculus errors
Current code isn't particularly resilient to API changes.
This commit is contained in:
parent
83a01e6156
commit
fc9c222a44
|
|
@ -211,15 +211,17 @@ const fetchAndValidate = async <T = unknown>(
|
|||
url: string,
|
||||
validator: ValidateFunction<T>
|
||||
): Promise<T> => {
|
||||
console.log(url);
|
||||
// console.log(url);
|
||||
const data = await fetchWithRetries<object>(url);
|
||||
if (validator(data)) {
|
||||
return data;
|
||||
}else{
|
||||
console.log(data)
|
||||
throw new Error(
|
||||
`Response validation for url ${url} failed: ` +
|
||||
JSON.stringify(validator.errors, null, 4)
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
`Response validation for url ${url} failed: ` +
|
||||
JSON.stringify(validator.errors)
|
||||
);
|
||||
};
|
||||
|
||||
export async function fetchApiQuestions(
|
||||
|
|
|
|||
|
|
@ -1,28 +1,27 @@
|
|||
import { FetchedQuestion, Platform } from "..";
|
||||
import { average } from "../../../utils";
|
||||
import { sleep } from "../../utils/sleep";
|
||||
import Error from "next/error";
|
||||
import {FetchedQuestion, Platform} from "..";
|
||||
import {average} from "../../../utils";
|
||||
import {sleep} from "../../utils/sleep";
|
||||
import {
|
||||
ApiCommon,
|
||||
ApiMultipleQuestions,
|
||||
ApiPredictable,
|
||||
ApiQuestion,
|
||||
fetchApiQuestions,
|
||||
fetchSingleApiQuestion,
|
||||
fetchSingleApiQuestion
|
||||
} from "./api";
|
||||
|
||||
const platformName = "metaculus";
|
||||
const now = new Date().toISOString();
|
||||
const SLEEP_TIME = 1000;
|
||||
|
||||
async function apiQuestionToFetchedQuestions(
|
||||
apiQuestion: ApiQuestion
|
||||
): Promise<FetchedQuestion[]> {
|
||||
async function apiQuestionToFetchedQuestions(apiQuestion: ApiQuestion): Promise<FetchedQuestion[]> {
|
||||
// one item can expand:
|
||||
// - to 0 questions if we don't want it;
|
||||
// - to 1 question if it's a simple forecast
|
||||
// - to multiple questions if it's a group (see https://github.com/quantified-uncertainty/metaforecast/pull/84 for details)
|
||||
|
||||
const skip = (q: ApiPredictable): boolean => {
|
||||
const skip = (q : ApiPredictable) : boolean => {
|
||||
if (q.publish_time > now || now > q.resolve_time) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -32,44 +31,44 @@ async function apiQuestionToFetchedQuestions(
|
|||
return false;
|
||||
};
|
||||
|
||||
const buildFetchedQuestion = (
|
||||
q: ApiPredictable & ApiCommon
|
||||
): Omit<FetchedQuestion, "url" | "description" | "title"> => {
|
||||
const isBinary = q.possibilities.type === "binary";
|
||||
let options: FetchedQuestion["options"] = [];
|
||||
if (isBinary) {
|
||||
const probability = q.community_prediction.full.q2;
|
||||
if (probability !== undefined) {
|
||||
options = [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
];
|
||||
const buildFetchedQuestion = (q : ApiPredictable & ApiCommon) : Omit < FetchedQuestion,
|
||||
"url" | "description" | "title" > => {
|
||||
const isBinary = q.possibilities.type === "binary";
|
||||
let options: FetchedQuestion["options"] = [];
|
||||
if (isBinary) {
|
||||
const probability = q.community_prediction.full.q2;
|
||||
if (probability !== undefined) {
|
||||
options = [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY"
|
||||
}, {
|
||||
name: "No",
|
||||
probability: 1 - probability,
|
||||
type: "PROBABILITY"
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
id: `${platformName}-${q.id}`,
|
||||
options,
|
||||
qualityindicators: {
|
||||
numforecasts: q.number_of_predictions,
|
||||
},
|
||||
extra: {
|
||||
resolution_data: {
|
||||
publish_time: apiQuestion.publish_time,
|
||||
resolution: apiQuestion.resolution,
|
||||
close_time: apiQuestion.close_time,
|
||||
resolve_time: apiQuestion.resolve_time,
|
||||
return {
|
||||
id: `${platformName}-${
|
||||
q.id
|
||||
}`,
|
||||
options,
|
||||
qualityindicators: {
|
||||
numforecasts: q.number_of_predictions
|
||||
},
|
||||
},
|
||||
extra: {
|
||||
resolution_data: {
|
||||
publish_time: apiQuestion.publish_time,
|
||||
resolution: apiQuestion.resolution,
|
||||
close_time: apiQuestion.close_time,
|
||||
resolve_time: apiQuestion.resolve_time
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
if (apiQuestion.type === "group") {
|
||||
await sleep(SLEEP_TIME);
|
||||
|
|
@ -77,44 +76,64 @@ async function apiQuestionToFetchedQuestions(
|
|||
if (apiQuestionDetails.type !== "group") {
|
||||
throw new Error("Expected `group` type"); // shouldn't happen, this is mostly for typescript
|
||||
}
|
||||
return (apiQuestionDetails.sub_questions || [])
|
||||
.filter((q) => !skip(q))
|
||||
.map((sq) => {
|
||||
const tmp = buildFetchedQuestion(sq);
|
||||
return {
|
||||
...tmp,
|
||||
title: `${apiQuestion.title} (${sq.title})`,
|
||||
description: apiQuestionDetails.description || "",
|
||||
url: `https://www.metaculus.com${apiQuestion.page_url}?sub-question=${sq.id}`,
|
||||
};
|
||||
});
|
||||
try{
|
||||
let result = (apiQuestionDetails.sub_questions || []).filter((q) => ! skip(q)).map((sq) => {
|
||||
const tmp = buildFetchedQuestion(sq);
|
||||
return {
|
||||
... tmp,
|
||||
title: `${
|
||||
apiQuestion.title
|
||||
} (${
|
||||
sq.title
|
||||
})`,
|
||||
description: apiQuestionDetails.description || "",
|
||||
url: `https://www.metaculus.com${
|
||||
apiQuestion.page_url
|
||||
}?sub-question=${
|
||||
sq.id
|
||||
}`
|
||||
};
|
||||
});
|
||||
return result
|
||||
}catch(error){
|
||||
console.log(error)
|
||||
return []
|
||||
}
|
||||
} else if (apiQuestion.type === "forecast") {
|
||||
if (apiQuestion.group) {
|
||||
return []; // sub-question, should be handled on the group level
|
||||
}
|
||||
if (skip(apiQuestion)) {
|
||||
console.log(`- [Skipping]: ${
|
||||
apiQuestion.title
|
||||
}`)
|
||||
/*console.log(`Close time: ${
|
||||
apiQuestion.close_time
|
||||
}, resolve time: ${
|
||||
apiQuestion.resolve_time
|
||||
}`)*/
|
||||
return [];
|
||||
}
|
||||
|
||||
await sleep(SLEEP_TIME);
|
||||
const apiQuestionDetails = await fetchSingleApiQuestion(apiQuestion.id);
|
||||
const tmp = buildFetchedQuestion(apiQuestion);
|
||||
return [
|
||||
{
|
||||
...tmp,
|
||||
title: apiQuestion.title,
|
||||
description: apiQuestionDetails.description || "",
|
||||
url: "https://www.metaculus.com" + apiQuestion.page_url,
|
||||
},
|
||||
];
|
||||
try{
|
||||
const tmp = buildFetchedQuestion(apiQuestion);
|
||||
return [{
|
||||
... tmp,
|
||||
title: apiQuestion.title,
|
||||
description: apiQuestionDetails.description || "",
|
||||
url: "https://www.metaculus.com" + apiQuestion.page_url
|
||||
},];
|
||||
}catch(error){
|
||||
console.log(error)
|
||||
return []
|
||||
}
|
||||
} else {
|
||||
if (apiQuestion.type !== "claim") {
|
||||
// should never happen, since `discriminator` in JTD schema causes a strict runtime check
|
||||
console.log(
|
||||
`Unknown metaculus question type: ${
|
||||
(apiQuestion as any).type
|
||||
}, skipping`
|
||||
);
|
||||
if (apiQuestion.type !== "claim") { // should never happen, since `discriminator` in JTD schema causes a strict runtime check
|
||||
console.log(`Unknown metaculus question type: ${
|
||||
(apiQuestion as any).type
|
||||
}, skipping`);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
|
@ -125,22 +144,22 @@ export const metaculus: Platform<"id" | "debug"> = {
|
|||
label: "Metaculus",
|
||||
color: "#006669",
|
||||
version: "v2",
|
||||
fetcherArgs: ["id", "debug"],
|
||||
fetcherArgs: [
|
||||
"id", "debug"
|
||||
],
|
||||
async fetcher(opts) {
|
||||
let allQuestions: FetchedQuestion[] = [];
|
||||
|
||||
if (opts.args?.id) {
|
||||
if (opts.args ?. id) {
|
||||
console.log("Using optional id arg.")
|
||||
const id = Number(opts.args.id);
|
||||
const apiQuestion = await fetchSingleApiQuestion(id);
|
||||
const questions = await apiQuestionToFetchedQuestions(apiQuestion);
|
||||
console.log(questions);
|
||||
return {
|
||||
questions,
|
||||
partial: true,
|
||||
};
|
||||
return {questions, partial: true};
|
||||
}
|
||||
|
||||
let next: string | null = "https://www.metaculus.com/api2/questions/";
|
||||
let next: string |null = "https://www.metaculus.com/api2/questions/";
|
||||
let i = 1;
|
||||
while (next) {
|
||||
console.log(`\nQuery #${i} - ${next}`);
|
||||
|
|
@ -148,14 +167,17 @@ export const metaculus: Platform<"id" | "debug"> = {
|
|||
await sleep(SLEEP_TIME);
|
||||
const apiQuestions: ApiMultipleQuestions = await fetchApiQuestions(next);
|
||||
const results = apiQuestions.results;
|
||||
|
||||
// console.log(results)
|
||||
let j = false;
|
||||
|
||||
for (const result of results) {
|
||||
const questions = await apiQuestionToFetchedQuestions(result);
|
||||
// console.log(questions)
|
||||
for (const question of questions) {
|
||||
console.log(`- ${question.title}`);
|
||||
if ((!j && i % 20 === 0) || opts.args?.debug) {
|
||||
console.log(`- ${
|
||||
question.title
|
||||
}`);
|
||||
if ((! j && i % 20 === 0) || opts.args ?. debug) {
|
||||
console.log(question);
|
||||
j = true;
|
||||
}
|
||||
|
|
@ -167,20 +189,16 @@ export const metaculus: Platform<"id" | "debug"> = {
|
|||
i += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
questions: allQuestions,
|
||||
partial: false,
|
||||
};
|
||||
return {questions: allQuestions, partial: false};
|
||||
},
|
||||
|
||||
calculateStars(data) {
|
||||
const { numforecasts } = data.qualityindicators;
|
||||
const nuno = () =>
|
||||
(numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
|
||||
const {numforecasts} = data.qualityindicators;
|
||||
const nuno = () => (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2;
|
||||
const eli = () => 3;
|
||||
const misha = () => 3;
|
||||
const starsDecimal = average([nuno(), eli(), misha()]);
|
||||
const starsInteger = Math.round(starsDecimal);
|
||||
return starsInteger;
|
||||
},
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user