quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added more throtlling when fetching from metaculus

Browse Source
This commit is contained in:
NunoSempere 2021-06-08 15:00:11 +02:00
parent 2ca3815de1
commit ffbd09416d
2 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/platforms/metaculus-fetch.js
View File

@ -79,7 +79,7 @@ export async function metaculus() {
(now < result.resolve_time) (now < result.resolve_time)
) { ) {
// console.log(result) // console.log(result)
await sleep(1000) await sleep(5000)
let questionPage = await fetchMetaculusQuestionDescription(result.page_url) let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
let descriptionraw = questionPage.split(`<div class="question__content">`)[1] let descriptionraw = questionPage.split(`<div class="question__content">`)[1]
let descriptionprocessed1 = descriptionraw.split("</div>")[0] let descriptionprocessed1 = descriptionraw.split("</div>")[0]

18
src/utils/mergeEverything.js
View File

@ -5,14 +5,12 @@ let sets = ["astralcodexten","coupcast", "csetforetell", "elicit", "estimize", "
let suffix = "-questions" let suffix = "-questions"
export async function mergeEverything(){ export async function mergeEverything(){
let merged = [] let merged = []
for(let set of sets){ for(let set of sets){
let json = await mongoRead(set+suffix) let json = await mongoRead(set+suffix)
merged = merged.concat(json) merged = merged.concat(json)
}
let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")}))
await upsert( mergedprocessed,"metaforecasts")
console.log("Done")
} }
let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")}))
await upsert( mergedprocessed,"metaforecasts")
console.log("Done")
}