feat: metaculus fetcher skips unknown types instead of failing

This commit is contained in:
Vyacheslav Matyukhin 2022-06-03 13:00:08 +03:00
parent 646397d8d4
commit 5bf24a58ef
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C

View File

@ -88,7 +88,7 @@ const optionalPageProps = {
},
} as const;
const apiQuestionSchema = {
const questionSchema = {
discriminator: "type",
mapping: {
forecast: {
@ -141,10 +141,19 @@ const apiQuestionSchema = {
},
} as const;
const apiMultipleQuestionsSchema = {
const knownQuestionTypes = Object.keys(questionSchema.mapping);
const shallowMultipleQuestionsSchema = {
properties: {
results: {
elements: apiQuestionSchema,
elements: {
properties: {
type: {
type: "string",
},
},
additionalProperties: true,
},
},
next: {
type: "string",
@ -160,14 +169,21 @@ export type ApiCommon = JTDDataType<{
export type ApiPredictable = JTDDataType<{
properties: typeof predictableProps;
}>;
export type ApiQuestion = JTDDataType<typeof apiQuestionSchema>;
export type ApiMultipleQuestions = JTDDataType<
typeof apiMultipleQuestionsSchema
export type ApiQuestion = JTDDataType<typeof questionSchema>;
type ApiShallowMultipleQuestions = JTDDataType<
typeof shallowMultipleQuestionsSchema
>;
const validateApiQuestion = new Ajv().compile<ApiQuestion>(apiQuestionSchema);
const validateApiMultipleQuestions = new Ajv().compile<ApiMultipleQuestions>(
apiMultipleQuestionsSchema
export type ApiMultipleQuestions = {
results: ApiQuestion[];
next: ApiShallowMultipleQuestions["next"]; // Omit<ApiShallowMultipleQuestions, "results"> doesn't work correctly here
};
const validateQuestion = new Ajv().compile<ApiQuestion>(questionSchema);
const validateShallowMultipleQuestions =
new Ajv().compile<ApiShallowMultipleQuestions>(
shallowMultipleQuestionsSchema
);
async function fetchWithRetries<T = unknown>(url: string): Promise<T> {
@ -209,12 +225,35 @@ const fetchAndValidate = async <T = unknown>(
export async function fetchApiQuestions(
next: string
): Promise<ApiMultipleQuestions> {
return await fetchAndValidate(next, validateApiMultipleQuestions);
const data = await fetchAndValidate(next, validateShallowMultipleQuestions);
const isDefined = <T>(argument: T | undefined): argument is T => {
return argument !== undefined;
};
return {
...data,
results: data.results
.map((result) => {
if (!knownQuestionTypes.includes(result.type)) {
console.warn(`Unknown result type ${result.type}, skipping`);
return undefined;
}
if (!validateQuestion(result)) {
throw new Error(
`Response validation failed: ` +
JSON.stringify(validateQuestion.errors)
);
}
return result;
})
.filter(isDefined),
};
}
export async function fetchSingleApiQuestion(id: number): Promise<ApiQuestion> {
return await fetchAndValidate(
`https://www.metaculus.com/api2/questions/${id}/`,
validateApiQuestion
validateQuestion
);
}