quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Deal with Metaculus throttling

Browse Source 
Parse the error messages and wait for the indicated amount.
This commit is contained in:
NunoSempere 2022-03-20 10:46:58 -04:00
parent b8e6435fee
commit 2df73b4933
1 changed files with 47 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/platforms/metaculus-fetch.js
View File

@ -9,17 +9,44 @@ import { databaseUpsert } from "../database/database-wrapper.js";
let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page=";
let now = new Date().toISOString(); let now = new Date().toISOString();
let DEBUG_MODE = "off"; let DEBUG_MODE = "off";
let SLEEP_TIME = 500; let SLEEP_TIME = 5000;
/* Support functions */ /* Support functions */
async function fetchMetaculusQuestions(next) { async function fetchMetaculusQuestions(next) {
// Numbers about a given address: how many, how much, at what price, etc. // Numbers about a given address: how many, how much, at what price, etc.
let response = await axios({ let response;
let data;
try {
response = await axios({
url: next, url: next,
method: "GET", method: "GET",
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
}).then((res) => res.data); });
data = response.data;
} catch (error) {
console.log(`Error in async function fetchMetaculusQuestions(next)`);
if (!!error.response.headers["retry-after"]) {
let timeout = error.response.headers["retry-after"];
console.log(`Timeout: ${timeout}`);
await sleep(Number(timeout) * 1000 + SLEEP_TIME);
} else {
await sleep(SLEEP_TIME);
}
console.log(error);
} finally {
try {
response = await axios({
url: next,
method: "GET",
headers: { "Content-Type": "application/json" },
});
data = response.data;
} catch (error) {
console.log(error);
return { results: [] };
}
}
// console.log(response) // console.log(response)
return response; return data;
} }
function sleep(ms) { function sleep(ms) {
@ -34,11 +61,21 @@ async function fetchMetaculusQuestionDescription(slug) {
}).then((response) => response.data); }).then((response) => response.data);
return response; return response;
} catch (error) { } catch (error) {
console.log(error); console.log(`Error in: fetchMetaculusQuestionDescription`);
console.log( console.log(
`We encountered some error when attempting to fetch a metaculus page. Trying again` `We encountered some error when attempting to fetch a metaculus page. Trying again`
); );
if (
typeof error.response != "undefined" &&
typeof error.response.headers != "undefined" &&
typeof error.response.headers["retry-after"] != "undefined"
) {
let timeout = error.response.headers["retry-after"];
console.log(`Timeout: ${timeout}`);
await sleep(Number(timeout) * 1000 + SLEEP_TIME);
} else {
await sleep(SLEEP_TIME); await sleep(SLEEP_TIME);
}
try { try {
let response = await axios({ let response = await axios({
method: "get", method: "get",
@ -77,7 +114,7 @@ export async function metaculus() {
let j = false; let j = false;
for (let result of results) { for (let result of results) {
if (result.publish_time < now && now < result.resolve_time) { if (result.publish_time < now && now < result.resolve_time) {
await sleep(SLEEP_TIME); await sleep(SLEEP_TIME / 2);
let questionPage = await fetchMetaculusQuestionDescription( let questionPage = await fetchMetaculusQuestionDescription(
result.page_url result.page_url
); );