/* Imports */
import fs from 'fs'
import axios from "axios"
import {Tabletojson} from "tabletojson"
import toMarkdown from "./toMarkdown.js"
import {calculateStars} from "./stars.js"
/* Definitions */
let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
/* Support functions */
function getcookie(){
try {
let rawcookie = fs.readFileSync("./src/privatekeys.json")
let cookie = JSON.parse(rawcookie).csetforetellcookie
if(cookie == undefined){
throw new Error('No cookie for CSET-foretell!');
}
return cookie
} catch(error) {
console.log("Error: No cookies for CSET-foretell on src/privatekeys.json! See the README.md")
process.exit()
}
}
async function fetchPage(page, cookie){
let response = await axios({
url: htmlEndPoint+page,
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie
}),
})
.then(res => res.data)
//console.log(response)
return response
}
async function fetchStats(questionUrl, cookie){
let response = await axios({
url: questionUrl+"/stats",
method: 'GET',
headers: ({
'Content-Type': 'text/html',
'Cookie': cookie,
'Referer': questionUrl,
}),
})
.then(res => res.data)
//console.log(response)
// Is binary?
let isbinary = response.includes("binary?":true")
let options = []
if(isbinary){
// Crowd percentage
let htmlElements = response.split("\n")
let h3Element = htmlElements.filter(str => str.includes("
"))[0]
let crowdpercentage = h3Element.split(">")[1].split("<")[0]
let probability = Number(crowdpercentage.replace("%", ""))/100
options.push(({
name: "Yes",
probability: probability,
type: "PROBABILITY"
}), ({
name: "No",
probability: +(1-probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY"
}))
}else{
let optionsHtmlElement = "
"
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
let firstTable = tablesAsJson[0]
options = firstTable.map(element => ({
name: element['0'],
probability: Number(element['1'].replace("%",""))/100,
type: "PROBABILITY"
}))
}
// Description
let descriptionraw = response.split(` `)[0]
let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
let descriptionprocessed4=descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
let descriptionprocessed5=descriptionprocessed4.replaceAll("\n\n", "\n")
let descriptionprocessed6=descriptionprocessed5.replaceAll(""", `"`)
let descriptionprocessed7=descriptionprocessed6.replaceAll("'", "'")
let descriptionprocessed8=toMarkdown(descriptionprocessed7)
let description = descriptionprocessed8
// Number of forecasts
let numforecasts = response.split("prediction_sets_count":")[1].split(",")[0]
//console.log(numforecasts)
// Number of predictors
let numforecasters = response.split("predictors_count":")[1].split(",")[0]
//console.log(numpredictors)
let result = {
"description": description,
"options": options,
"numforecasts": numforecasts,
"numforecasters": numforecasters,
"stars": calculateStars("CSET-foretell", {numforecasts})
}
return result
}
function isEnd(html){
return html.includes("No questions match your filter")
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/* Body */
export async function csetforetell(){
let cookie = getcookie()
let i=1
let response = await fetchPage(i, cookie)
let questions = []
let init = Date.now()
console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while(!isEnd(response)){
console.log(`Page #${i}`)
let htmlLines = response.split("\n")
let h4elements = htmlLines.filter(str => str.includes("