quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: add proxy

Browse Source
This commit is contained in:
NunoSempere 2022-02-04 16:25:31 -05:00
parent e0bc00d6e9
commit 9bda1c2558
1 changed files with 81 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/platforms/goodjudgment-fetch.js
View File

@ -1,5 +1,6 @@
/* Imports */ /* Imports */
import axios from "axios"; import axios from "axios";
import tunnel from "tunnel";
import fs from "fs"; import fs from "fs";
import { Tabletojson } from "tabletojson"; import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js"; import toMarkdown from "../utils/toMarkdown.js";
@ -7,39 +8,47 @@ import { calculateStars } from "../utils/stars.js";
import { upsert } from "../utils/mongo-wrapper.js"; import { upsert } from "../utils/mongo-wrapper.js";
/* Definitions */ /* Definitions */
let endpoints = [ let endpoint = "https://goodjudgment.io/superforecasts/";
"https://goodjudgment.io/superforecasts/",
// "https://goodjudgment.io/economist/",
];
String.prototype.replaceAll = function replaceAll(search, replace) { String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace); return this.split(search).join(replace);
}; };
// Tunelling
/* Support functions */ /* Support functions */
/* Body */ /* Body */
export async function goodjudgment() { export async function goodjudgment() {
let results = []; // Proxy fuckery
let proxy = await axios let proxy = await axios
.get("http://pubproxy.com/api/proxy?cookies=true") .get("http://pubproxy.com/api/proxy")
.then((query) => query.data); .then((query) => query.data);
console.log(proxy); console.log(proxy);
for (let endpoint of endpoints) {
let content = await axios let agent = tunnel.httpsOverHttp({
.get(endpoint, {
headers: {
"User-Agent": "Axios",
},
proxy: { proxy: {
host: proxy.ip, host: proxy.ip,
port: proxy.port, port: proxy.port,
}, },
});
let content = await axios
.request({
url: "https://goodjudgment.io/superforecasts/",
method: "get",
headers: {
"User-Agent": "Chrome",
},
agent,
port: 80,
}) })
.then((query) => query.data); .then((query) => query.data);
// Processing
let results = [];
let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false });
jsonTable.shift(); // deletes first element jsonTable.shift(); // deletes first element
jsonTable.pop(); // deletes last element jsonTable.pop(); // deletes last element
if (endpoint == endpoints[1]) jsonTable.pop(); // pop again\
// console.log(jsonTable) // console.log(jsonTable)
for (let table of jsonTable) { for (let table of jsonTable) {
// console.log(table) // console.log(table)
@ -62,9 +71,7 @@ export async function goodjudgment() {
let options = table let options = table
.filter((row) => "4" in row) .filter((row) => "4" in row)
.map((row) => ({ .map((row) => ({
name: row["2"] name: row["2"].split('<span class="qTitle">')[1].replace("</span>", ""),
.split('<span class="qTitle">')[1]
.replace("</span>", ""),
probability: Number(row["3"].split("%")[0]) / 100, probability: Number(row["3"].split("%")[0]) / 100,
type: "PROBABILITY", type: "PROBABILITY",
})); }));
@ -94,7 +101,6 @@ export async function goodjudgment() {
results.push(standardObj); results.push(standardObj);
} }
} }
}
// console.log(results.slice(0,10)) // console.log(results.slice(0,10))
let string = JSON.stringify(results, null, 2); let string = JSON.stringify(results, null, 2);
// fs.writeFileSync('./data/goodjudgment-questions.json', string); // fs.writeFileSync('./data/goodjudgment-questions.json', string);