quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: regiggled metaculus predictions to deal with new interface

Browse Source
This commit is contained in:
NunoSempere 2021-12-02 13:30:16 +01:00
parent 36e3d4af46
commit ee0ee3ca16
5 changed files with 51747 additions and 49379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101006
data/frontpage.json
View File

File diff suppressed because one or more lines are too long

4
package.json
View File

@ -5,7 +5,9 @@
"main": "src/index.js",
"scripts": {
"start": "node src/index.js",
"test": "echo \"Error: no test specified\" && exit 1"
"test": "echo \"Error: no test specified\" && exit 1",
"reload": "heroku run:detached node src/utils/doEverythingForScheduler.js"
},
"repository": {
"type": "git",

1
src/platforms/astralcodexten-fetch.js
View File

@ -93,6 +93,7 @@ export async function astralcodexten(){
}
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('./data/foretold-questions.json', string);
// console.log(JSON.stringify(results, null, 2))
await upsert(results, "astralcodexten-questions")
// console.log(results)
console.log("Done")

109
src/platforms/metaculus-fetch.js
View File

@ -79,61 +79,66 @@ export async function metaculus() {
(result.publish_time < now) &&
(now < result.resolve_time)
) {
await sleep(5000)
await sleep(5000)
let questionPage = await fetchMetaculusQuestionDescription(result.page_url)
let descriptionraw = questionPage.split(`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
let descriptionprocessed1 = descriptionraw.split("</div>")[0]
let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
let description = descriptionprocessed2
let isbinary = result.possibilities.type == "binary"
let options = []
if (isbinary) {
let probability = Number(result.community_prediction.full.q2)
options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
if(!questionPage.includes("A public prediction by")){
let descriptionraw = questionPage.split(`<div class="content" ng-bind-html-compile="qctrl.question.description_html">`)[1] //.split(`<div class="question__content">`)[1]
let descriptionprocessed1 = descriptionraw.split("</div>")[0]
let descriptionprocessed2 = toMarkdown(descriptionprocessed1)
let description = descriptionprocessed2
let isbinary = result.possibilities.type == "binary"
let options = []
if (isbinary) {
let probability = Number(result.community_prediction.full.q2)
options = [
{
"name": "Yes",
"probability": probability,
"type": "PROBABILITY"
},
{
"name": "No",
"probability": 1 - probability,
"type": "PROBABILITY"
}
]
}
let interestingInfo = ({
"title": result.title,
"url": "https://www.metaculus.com" + result.page_url,
"platform": "Metaculus",
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(result.number_of_predictions),
"resolution_data": {
"publish_time": result.publish_time,
"resolution": result.resolution,
"close_time": result.close_time,
"resolve_time": result.resolve_time
},
"stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions }))
}
//"status": result.status,
//"publish_time": result.publish_time,
//"close_time": result.close_time,
//"type": result.possibilities.type, // We want binary ones here.
//"last_activity_time": result.last_activity_time,
})
if (Number(result.number_of_predictions) >= 10) {
console.log(`- ${interestingInfo.title}`)
all_questions.push(interestingInfo)
if(!j && (i % 20 == 0)){
console.log(interestingInfo)
j = true
}
]
}
let interestingInfo = ({
"title": result.title,
"url": "https://www.metaculus.com" + result.page_url,
"platform": "Metaculus",
"description": description,
"options": options,
"timestamp": new Date().toISOString(),
"qualityindicators": {
"numforecasts": Number(result.number_of_predictions),
"resolution_data": {
"publish_time": result.publish_time,
"resolution": result.resolution,
"close_time": result.close_time,
"resolve_time": result.resolve_time
},
"stars": calculateStars("Metaculus", ({ numforecasts: result.number_of_predictions }))
}
//"status": result.status,
//"publish_time": result.publish_time,
//"close_time": result.close_time,
//"type": result.possibilities.type, // We want binary ones here.
//"last_activity_time": result.last_activity_time,
})
if (Number(result.number_of_predictions) >= 10) {
console.log(`- ${interestingInfo.title}`)
all_questions.push(interestingInfo)
if(!j && (i % 20 == 0)){
console.log(interestingInfo)
j = true
}
}else{
console.log("- [Skipping public prediction]")
}
}
}
@ -142,7 +147,7 @@ export async function metaculus() {
}
// let string = JSON.stringify(all_questions, null, 2)
// fs.writeFileSync('./data/metaculus-questions.json', string);
fs.writeFileSync('./metaculus-questions.json', string);
await upsert(all_questions, "metaculus-questions")
console.log("Done")

6
src/utils/downloadFrontpage.js
View File

@ -2,9 +2,9 @@ import fs from "fs"
import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
let filename = 'frontpage.json'
let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json'
let shuffle = (array) => {
// https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array
// https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
let currentIndex = array.length, randomIndex;
// While there remain elements to shuffle...
@ -32,7 +32,7 @@ let main = async () => {
let string = JSON.stringify(json, null, 2)
fs.writeFileSync(filename, string);
console.log(`File downloaded to ./${filename}`)
console.log(`File downloaded to ${filename}`)
let end = Date.now()
let difference = end - init