fix: add proxy

src/platforms/goodjudgment-fetch.js

@@ -1,5 +1,6 @@
 /* Imports */
 import axios from "axios";
+import tunnel from "tunnel";
 import fs from "fs";
 import { Tabletojson } from "tabletojson";
 import toMarkdown from "../utils/toMarkdown.js";
@@ -7,39 +8,47 @@ import { calculateStars } from "../utils/stars.js";
 import { upsert } from "../utils/mongo-wrapper.js";
 
 /* Definitions */
-let endpoints = [
-  "https://goodjudgment.io/superforecasts/",
-  // "https://goodjudgment.io/economist/",
-];
+let endpoint = "https://goodjudgment.io/superforecasts/";
 String.prototype.replaceAll = function replaceAll(search, replace) {
   return this.split(search).join(replace);
 };
 
+// Tunelling
 /* Support functions */
 
 /* Body */
 export async function goodjudgment() {
-  let results = [];
+  // Proxy fuckery
+
   let proxy = await axios
-    .get("http://pubproxy.com/api/proxy?cookies=true")
+    .get("http://pubproxy.com/api/proxy")
     .then((query) => query.data);
   console.log(proxy);
-  for (let endpoint of endpoints) {
-    let content = await axios
-      .get(endpoint, {
-        headers: {
-          "User-Agent": "Axios",
-        },
+
+  let agent = tunnel.httpsOverHttp({
     proxy: {
       host: proxy.ip,
       port: proxy.port,
     },
+  });
+
+  let content = await axios
+    .request({
+      url: "https://goodjudgment.io/superforecasts/",
+      method: "get",
+      headers: {
+        "User-Agent": "Chrome",
+      },
+      agent,
+      port: 80,
     })
     .then((query) => query.data);
+
+  // Processing
+  let results = [];
   let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false });
   jsonTable.shift(); // deletes first element
   jsonTable.pop(); // deletes last element
-    if (endpoint == endpoints[1]) jsonTable.pop(); // pop again\
   // console.log(jsonTable)
   for (let table of jsonTable) {
     // console.log(table)
@@ -62,9 +71,7 @@ export async function goodjudgment() {
     let options = table
       .filter((row) => "4" in row)
       .map((row) => ({
-          name: row["2"]
-            .split('<span class="qTitle">')[1]
-            .replace("</span>", ""),
+        name: row["2"].split('<span class="qTitle">')[1].replace("</span>", ""),
         probability: Number(row["3"].split("%")[0]) / 100,
         type: "PROBABILITY",
       }));
@@ -94,7 +101,6 @@ export async function goodjudgment() {
       results.push(standardObj);
     }
   }
-  }
   // console.log(results.slice(0,10))
   let string = JSON.stringify(results, null, 2);
   // fs.writeFileSync('./data/goodjudgment-questions.json', string);