quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: add proxy

Browse Source
This commit is contained in:
NunoSempere 2022-02-04 16:25:31 -05:00
parent e0bc00d6e9
commit 9bda1c2558
1 changed files with 81 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
src/platforms/goodjudgment-fetch.js
View File

@ -1,5 +1,6 @@
/* Imports */ /* Imports */
import axios from "axios"; import axios from "axios";
import tunnel from "tunnel";
import fs from "fs"; import fs from "fs";
import { Tabletojson } from "tabletojson"; import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js"; import toMarkdown from "../utils/toMarkdown.js";
@ -7,92 +8,97 @@ import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js"; import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let endpoints = [ let endpoint = "https://goodjudgment.io/superforecasts/";
"https://goodjudgment.io/superforecasts/",
// "https://goodjudgment.io/economist/",
];
String.prototype.replaceAll = function replaceAll(search, replace) { String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace); return this.split(search).join(replace);
}; };
// Tunelling
/* Support functions */ /* Support functions */
/* Body */ /* Body */
export async function goodjudgment() { export async function goodjudgment() {
let results = []; // Proxy fuckery
let proxy = await axios let proxy = await axios
.get("http://pubproxy.com/api/proxy?cookies=true") .get("http://pubproxy.com/api/proxy")
.then((query) => query.data); .then((query) => query.data);
console.log(proxy); console.log(proxy);
for (let endpoint of endpoints) {
let content = await axios let agent = tunnel.httpsOverHttp({
.get(endpoint, { proxy: {
headers: { host: proxy.ip,
"User-Agent": "Axios", port: proxy.port,
}, },
proxy: { });
host: proxy.ip,
port: proxy.port, let content = await axios
}, .request({
}) url: "https://goodjudgment.io/superforecasts/",
.then((query) => query.data); method: "get",
let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); headers: {
jsonTable.shift(); // deletes first element "User-Agent": "Chrome",
jsonTable.pop(); // deletes last element },
if (endpoint == endpoints[1]) jsonTable.pop(); // pop again\ agent,
// console.log(jsonTable) port: 80,
for (let table of jsonTable) { })
// console.log(table) .then((query) => query.data);
let title = table[0]["0"].split("\t\t\t").splice(3)[0];
let description = table // Processing
.filter((row) => row["0"].includes("BACKGROUND:")) let results = [];
.map((row) => row["0"]) let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false });
.map((text) => jsonTable.shift(); // deletes first element
text jsonTable.pop(); // deletes last element
.split("BACKGROUND:")[1] // console.log(jsonTable)
.split("Examples of Superforecaster")[0] for (let table of jsonTable) {
.split("AT A GLANCE")[0] // console.log(table)
.replaceAll("\n\n", "\n") let title = table[0]["0"].split("\t\t\t").splice(3)[0];
.split("\n") let description = table
.slice(3) .filter((row) => row["0"].includes("BACKGROUND:"))
.join(" ") .map((row) => row["0"])
.replaceAll(" ", "") .map((text) =>
.replaceAll("<br> ", "") text
)[0]; .split("BACKGROUND:")[1]
let options = table .split("Examples of Superforecaster")[0]
.filter((row) => "4" in row) .split("AT A GLANCE")[0]
.map((row) => ({ .replaceAll("\n\n", "\n")
name: row["2"] .split("\n")
.split('<span class="qTitle">')[1] .slice(3)
.replace("</span>", ""), .join(" ")
probability: Number(row["3"].split("%")[0]) / 100, .replaceAll(" ", "")
type: "PROBABILITY", .replaceAll("<br> ", "")
})); )[0];
let analysis = table.filter((row) => let options = table
row[0] ? row[0].toLowerCase().includes("commentary") : false .filter((row) => "4" in row)
); .map((row) => ({
// "Examples of Superforecaster Commentary" / Analysis name: row["2"].split('<span class="qTitle">')[1].replace("</span>", ""),
// The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list. probability: Number(row["3"].split("%")[0]) / 100,
analysis = analysis ? analysis[0] : ""; type: "PROBABILITY",
analysis = analysis ? analysis[0] : ""; }));
// console.log(analysis) let analysis = table.filter((row) =>
let standardObj = { row[0] ? row[0].toLowerCase().includes("commentary") : false
title: title, );
url: endpoint, // "Examples of Superforecaster Commentary" / Analysis
platform: "Good Judgment", // The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list.
description: description, analysis = analysis ? analysis[0] : "";
options: options, analysis = analysis ? analysis[0] : "";
timestamp: new Date().toISOString(), // console.log(analysis)
qualityindicators: { let standardObj = {
stars: calculateStars("Good Judgment", {}), title: title,
}, url: endpoint,
extra: { platform: "Good Judgment",
superforecastercommentary: analysis || "", description: description,
}, options: options,
}; timestamp: new Date().toISOString(),
if (standardObj.title != undefined) { qualityindicators: {
results.push(standardObj); stars: calculateStars("Good Judgment", {}),
} },
extra: {
superforecastercommentary: analysis || "",
},
};
if (standardObj.title != undefined) {
results.push(standardObj);
} }
} }
// console.log(results.slice(0,10)) // console.log(results.slice(0,10))