From 9bda1c2558fe3b6911276058a45b255202499c12 Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Fri, 4 Feb 2022 16:25:31 -0500 Subject: [PATCH] fix: add proxy --- src/platforms/goodjudgment-fetch.js | 156 +++++++++++++++------------- 1 file changed, 81 insertions(+), 75 deletions(-) diff --git a/src/platforms/goodjudgment-fetch.js b/src/platforms/goodjudgment-fetch.js index f5f649c..a34244b 100644 --- a/src/platforms/goodjudgment-fetch.js +++ b/src/platforms/goodjudgment-fetch.js @@ -1,5 +1,6 @@ /* Imports */ import axios from "axios"; +import tunnel from "tunnel"; import fs from "fs"; import { Tabletojson } from "tabletojson"; import toMarkdown from "../utils/toMarkdown.js"; @@ -7,92 +8,97 @@ import { calculateStars } from "../utils/stars.js"; import { upsert } from "../utils/mongo-wrapper.js"; /* Definitions */ -let endpoints = [ - "https://goodjudgment.io/superforecasts/", - // "https://goodjudgment.io/economist/", -]; +let endpoint = "https://goodjudgment.io/superforecasts/"; String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }; +// Tunelling /* Support functions */ /* Body */ export async function goodjudgment() { - let results = []; + // Proxy fuckery + let proxy = await axios - .get("http://pubproxy.com/api/proxy?cookies=true") + .get("http://pubproxy.com/api/proxy") .then((query) => query.data); console.log(proxy); - for (let endpoint of endpoints) { - let content = await axios - .get(endpoint, { - headers: { - "User-Agent": "Axios", - }, - proxy: { - host: proxy.ip, - port: proxy.port, - }, - }) - .then((query) => query.data); - let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); - jsonTable.shift(); // deletes first element - jsonTable.pop(); // deletes last element - if (endpoint == endpoints[1]) jsonTable.pop(); // pop again\ - // console.log(jsonTable) - for (let table of jsonTable) { - // console.log(table) - let title = table[0]["0"].split("\t\t\t").splice(3)[0]; - let description = table - .filter((row) => row["0"].includes("BACKGROUND:")) - .map((row) => row["0"]) - .map((text) => - text - .split("BACKGROUND:")[1] - .split("Examples of Superforecaster")[0] - .split("AT A GLANCE")[0] - .replaceAll("\n\n", "\n") - .split("\n") - .slice(3) - .join(" ") - .replaceAll(" ", "") - .replaceAll("
", "") - )[0]; - let options = table - .filter((row) => "4" in row) - .map((row) => ({ - name: row["2"] - .split('')[1] - .replace("", ""), - probability: Number(row["3"].split("%")[0]) / 100, - type: "PROBABILITY", - })); - let analysis = table.filter((row) => - row[0] ? row[0].toLowerCase().includes("commentary") : false - ); - // "Examples of Superforecaster Commentary" / Analysis - // The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list. - analysis = analysis ? analysis[0] : ""; - analysis = analysis ? analysis[0] : ""; - // console.log(analysis) - let standardObj = { - title: title, - url: endpoint, - platform: "Good Judgment", - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Good Judgment", {}), - }, - extra: { - superforecastercommentary: analysis || "", - }, - }; - if (standardObj.title != undefined) { - results.push(standardObj); - } + + let agent = tunnel.httpsOverHttp({ + proxy: { + host: proxy.ip, + port: proxy.port, + }, + }); + + let content = await axios + .request({ + url: "https://goodjudgment.io/superforecasts/", + method: "get", + headers: { + "User-Agent": "Chrome", + }, + agent, + port: 80, + }) + .then((query) => query.data); + + // Processing + let results = []; + let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); + jsonTable.shift(); // deletes first element + jsonTable.pop(); // deletes last element + // console.log(jsonTable) + for (let table of jsonTable) { + // console.log(table) + let title = table[0]["0"].split("\t\t\t").splice(3)[0]; + let description = table + .filter((row) => row["0"].includes("BACKGROUND:")) + .map((row) => row["0"]) + .map((text) => + text + .split("BACKGROUND:")[1] + .split("Examples of Superforecaster")[0] + .split("AT A GLANCE")[0] + .replaceAll("\n\n", "\n") + .split("\n") + .slice(3) + .join(" ") + .replaceAll(" ", "") + .replaceAll("
", "") + )[0]; + let options = table + .filter((row) => "4" in row) + .map((row) => ({ + name: row["2"].split('')[1].replace("", ""), + probability: Number(row["3"].split("%")[0]) / 100, + type: "PROBABILITY", + })); + let analysis = table.filter((row) => + row[0] ? row[0].toLowerCase().includes("commentary") : false + ); + // "Examples of Superforecaster Commentary" / Analysis + // The following is necessary twite, because we want to check if there is an empty list, and then get the first element of the first element of the list. + analysis = analysis ? analysis[0] : ""; + analysis = analysis ? analysis[0] : ""; + // console.log(analysis) + let standardObj = { + title: title, + url: endpoint, + platform: "Good Judgment", + description: description, + options: options, + timestamp: new Date().toISOString(), + qualityindicators: { + stars: calculateStars("Good Judgment", {}), + }, + extra: { + superforecastercommentary: analysis || "", + }, + }; + if (standardObj.title != undefined) { + results.push(standardObj); } } // console.log(results.slice(0,10))