127 lines
3.7 KiB
JavaScript
127 lines
3.7 KiB
JavaScript
/* Imports */
|
|
import fs from "fs";
|
|
import axios from "axios";
|
|
import Papa from "papaparse";
|
|
import open from "open";
|
|
import readline from "readline";
|
|
import { calculateStars } from "../utils/stars.js";
|
|
import { databaseUpsert } from "../utils/database-wrapper.js";
|
|
|
|
/* Definitions */
|
|
let elicitEndpoint =
|
|
"https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";
|
|
|
|
/* Support functions */
|
|
let avg = (array) =>
|
|
array.reduce((a, b) => Number(a) + Number(b)) / array.length;
|
|
let unique = (arr) => [...new Set(arr)];
|
|
let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
async function processArray(arrayQuestions) {
|
|
let questions = arrayQuestions.map((question) => question.questionTitle);
|
|
let uniqueQuestions = unique(questions);
|
|
let questionsObj = {};
|
|
uniqueQuestions.forEach((questionTitle) => {
|
|
questionsObj[questionTitle] = {
|
|
title: questionTitle,
|
|
forecasters: [],
|
|
forecasts: [],
|
|
};
|
|
});
|
|
arrayQuestions.forEach((question) => {
|
|
// console.log(question.questionTitle)
|
|
let questionTitle = question.questionTitle;
|
|
let correspondingQuestion = questionsObj[questionTitle];
|
|
let forecasters = correspondingQuestion.forecasters.concat(
|
|
question.predictionCreator
|
|
);
|
|
let forecasts = correspondingQuestion.forecasts.concat(question.prediction);
|
|
questionsObj[questionTitle] = {
|
|
forecasters,
|
|
forecasts,
|
|
};
|
|
});
|
|
let onlyQuestionsWithMoreThan;
|
|
|
|
let results = [];
|
|
for (let question in questionsObj) {
|
|
let title = question;
|
|
|
|
let forecasters = questionsObj[question].forecasters;
|
|
|
|
let numforecasters = unique(forecasters).length;
|
|
if (numforecasters >= 10) {
|
|
let url = `https://elicit.org/binary?binaryQuestions.search=${title.replace(
|
|
/ /g,
|
|
"%20"
|
|
)}&binaryQuestions.sortBy=popularity&limit=20&offset=0`;
|
|
let forecasts = questionsObj[question].forecasts;
|
|
|
|
//console.log(forecasts)
|
|
//console.log(avg(forecasts))
|
|
let probability = avg(forecasts) / 100;
|
|
let numforecasts = forecasts.length;
|
|
let standardObj = {
|
|
title: title,
|
|
url: url,
|
|
platform: "Elicit",
|
|
options: [
|
|
{
|
|
name: "Yes",
|
|
probability: probability,
|
|
type: "PROBABILITY",
|
|
},
|
|
{
|
|
name: "No",
|
|
probability: 1 - probability,
|
|
type: "PROBABILITY",
|
|
},
|
|
],
|
|
timestamp: new Date().toISOString(),
|
|
qualityindicators: {
|
|
numforecasts: Number(numforecasts),
|
|
numforecasters: Number(numforecasters),
|
|
stars: calculateStars("Elicit", {}),
|
|
},
|
|
};
|
|
results.push(standardObj);
|
|
}
|
|
}
|
|
// let string = JSON.stringify(results, null, 2)
|
|
// fs.writeFileSync('./data/elicit-questions.json', string);
|
|
await databaseUpsert(results, "elicit-questions");
|
|
|
|
console.log("Done");
|
|
}
|
|
|
|
async function awaitdownloadconfirmation(message, callback) {
|
|
const rl = readline.createInterface({
|
|
input: process.stdin,
|
|
output: process.stdout,
|
|
});
|
|
rl.question(message, (answer) => {
|
|
//console.log("Received");
|
|
rl.close();
|
|
|
|
callback();
|
|
});
|
|
}
|
|
|
|
/* Body */
|
|
let filePath = "./data/elicit-binary_export.csv";
|
|
|
|
export async function elicit() {
|
|
let csvContent = await axios.get(elicitEndpoint).then((query) => query.data);
|
|
await Papa.parse(csvContent, {
|
|
header: true,
|
|
complete: async (results) => {
|
|
console.log("Downloaded", results.data.length, "records.");
|
|
//resolve(results.data);
|
|
//console.log(results.data)
|
|
await processArray(results.data);
|
|
},
|
|
});
|
|
await sleep(5000); // needed to wait for Papaparse's callback to be executed.
|
|
}
|
|
//elicit()
|