From ffbd09416da8f31ab2aa8c4901ee03cbc9ff1d90 Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Tue, 8 Jun 2021 15:00:11 +0200 Subject: [PATCH] Added more throtlling when fetching from metaculus --- src/platforms/metaculus-fetch.js | 2 +- src/utils/mergeEverything.js | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/platforms/metaculus-fetch.js b/src/platforms/metaculus-fetch.js index b90f9f7..28004cd 100644 --- a/src/platforms/metaculus-fetch.js +++ b/src/platforms/metaculus-fetch.js @@ -79,7 +79,7 @@ export async function metaculus() { (now < result.resolve_time) ) { // console.log(result) - await sleep(1000) + await sleep(5000) let questionPage = await fetchMetaculusQuestionDescription(result.page_url) let descriptionraw = questionPage.split(`
`)[1] let descriptionprocessed1 = descriptionraw.split("
")[0] diff --git a/src/utils/mergeEverything.js b/src/utils/mergeEverything.js index 38ef7dd..496c261 100644 --- a/src/utils/mergeEverything.js +++ b/src/utils/mergeEverything.js @@ -5,14 +5,12 @@ let sets = ["astralcodexten","coupcast", "csetforetell", "elicit", "estimize", " let suffix = "-questions" export async function mergeEverything(){ - let merged = [] - for(let set of sets){ - let json = await mongoRead(set+suffix) - merged = merged.concat(json) - } - let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")})) - await upsert( mergedprocessed,"metaforecasts") - console.log("Done") + let merged = [] + for(let set of sets){ + let json = await mongoRead(set+suffix) + merged = merged.concat(json) } - - + let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")})) + await upsert( mergedprocessed,"metaforecasts") + console.log("Done") +}