quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: regiggled metaculus predictions to deal with new interface

Browse Source
This commit is contained in:
NunoSempere 2021-12-02 13:30:16 +01:00
parent 36e3d4af46
commit ee0ee3ca16
5 changed files with 51747 additions and 49379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101038
data/frontpage.json
View File

File diff suppressed because one or more lines are too long

4
package.json
View File

@ -5,7 +5,9 @@
"main": "src/index.js", "main": "src/index.js",
"scripts": { "scripts": {
"start": "node src/index.js", "start": "node src/index.js",
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1",
"reload": "heroku run:detached node src/utils/doEverythingForScheduler.js"
}, },
"repository": { "repository": {
"type": "git", "type": "git",

1
src/platforms/astralcodexten-fetch.js
View File

@ -93,6 +93,7 @@ export async function astralcodexten(){
} }
// let string = JSON.stringify(results, null, 2) // let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/foretold-questions.json', string); // fs.writeFileSync('./data/foretold-questions.json', string);
// console.log(JSON.stringify(results, null, 2))
await upsert(results, "astralcodexten-questions") await upsert(results, "astralcodexten-questions")
// console.log(results) // console.log(results)
console.log("Done") console.log("Done")

7
src/platforms/metaculus-fetch.js
View File

@ -81,6 +81,7 @@ export async function metaculus() {
) { ) {
await sleep(5000) await sleep(5000)
let questionPage = await fetchMetaculusQuestionDescription(result.page_url) let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
if(!questionPage.includes("A public prediction by")){
let descriptionraw = questionPage.split(`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1] let descriptionraw = questionPage.split(`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
let descriptionprocessed1 = descriptionraw.split("</div>")[0] let descriptionprocessed1 = descriptionraw.split("</div>")[0]
let descriptionprocessed2 = toMarkdown(descriptionprocessed1) let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
@ -134,6 +135,10 @@ export async function metaculus() {
j = true j = true
} }
} }
}else{
console.log("- [Skipping public prediction]")
}
} }
} }
@ -142,7 +147,7 @@ export async function metaculus() {
} }
// let string = JSON.stringify(all_questions, null, 2) // let string = JSON.stringify(all_questions, null, 2)
// fs.writeFileSync('./data/metaculus-questions.json', string); fs.writeFileSync('./metaculus-questions.json', string);
await upsert(all_questions, "metaculus-questions") await upsert(all_questions, "metaculus-questions")
console.log("Done") console.log("Done")

4
src/utils/downloadFrontpage.js
View File

@ -2,7 +2,7 @@ import fs from "fs"
import { mongoReadWithReadCredentials } from "./mongo-wrapper.js" import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
let filename = 'frontpage.json' let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json'
let shuffle = (array) => { let shuffle = (array) => {
// https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array // https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
let currentIndex = array.length, randomIndex; let currentIndex = array.length, randomIndex;
@ -32,7 +32,7 @@ let main = async () => {
let string = JSON.stringify(json, null, 2) let string = JSON.stringify(json, null, 2)
fs.writeFileSync(filename, string); fs.writeFileSync(filename, string);
console.log(`File downloaded to ./${filename}`) console.log(`File downloaded to ${filename}`)
let end = Date.now() let end = Date.now()
let difference = end - init let difference = end - init