metaforecast/src/backend/platforms/goodjudmentopen.ts

/* Imports */
import axios from "axios";
import { Tabletojson } from "tabletojson";

import { applyIfSecretExists } from "../utils/getSecrets";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
import { Platform } from "./";

/* Definitions */
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
let annoyingPromptUrls = [
  "https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
  "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters",
  "https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
  "https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
];
const DEBUG_MODE: "on" | "off" = "off"; // "on"
const id = () => 0;

/* Support functions */

async function fetchPage(page, cookie) {
  let response = await axios({
    url: htmlEndPoint + page,
    method: "GET",
    headers: {
      "Content-Type": "text/html",
      Cookie: cookie,
    },
  }).then((res) => res.data);
  //console.log(response)
  return response;
}

async function fetchStats(questionUrl, cookie) {
  let response = await axios({
    url: questionUrl + "/stats",
    method: "GET",
    headers: {
      "Content-Type": "text/html",
      Cookie: cookie,
      Referer: questionUrl,
    },
  }).then((res) => res.data);
  //console.log(response)

  // Is binary?
  let isbinary = response.includes("binary?":true");

  let options = [];
  if (isbinary) {
    // Crowd percentage
    let htmlElements = response.split("\n");
    let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
    // console.log(h3Element)
    let crowdpercentage = h3Element.split(">")[1].split("<")[0];
    let probability = Number(crowdpercentage.replace("%", "")) / 100;
    options.push(
      {
        name: "Yes",
        probability: probability,
        type: "PROBABILITY",
      },
      {
        name: "No",
        probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
        type: "PROBABILITY",
      }
    );
  } else {
    let optionsHtmlElement = "<table" + response.split("tbody")[1] + "table>";
    let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
    let firstTable = tablesAsJson[0];
    options = firstTable.map((element) => ({
      name: element["0"],
      probability: Number(element["1"].replace("%", "")) / 100,
      type: "PROBABILITY",
    }));
    //console.log(optionsHtmlElement)
    //console.log(options)
  }

  // Description
  let descriptionraw = response.split(
    `<div id="question-background" class="collapse smb">`
  )[1];
  let descriptionprocessed1 = descriptionraw.split(`</div>`)[0];
  let descriptionprocessed2 = toMarkdown(descriptionprocessed1);
  let descriptionprocessed3 = descriptionprocessed2
    .split("\n")
    .filter((string) => !string.includes("Confused? Check our"))
    .join("\n");
  let description = descriptionprocessed3;

  // Number of forecasts
  let numforecasts = response
    .split("prediction_sets_count&quot;:")[1]
    .split(",")[0];
  //console.log(numforecasts)

  // Number of predictors
  let numforecasters = response
    .split("predictors_count&quot;:")[1]
    .split(",")[0];
  //console.log(numpredictors)

  // Calculate the stars
  let minProbability = Math.min(...options.map((option) => option.probability));
  let maxProbability = Math.max(...options.map((option) => option.probability));

  let result = {
    description: description,
    options: options,
    timestamp: new Date().toISOString(),
    qualityindicators: {
      numforecasts: Number(numforecasts),
      numforecasters: Number(numforecasters),
      stars: calculateStars("Good Judgment Open", {
        numforecasts,
        minProbability,
        maxProbability,
      }),
    },
    // this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now
  } as any;

  return result;
}

function isSignedIn(html) {
  let isSignedInBool = !(
    html.includes("You need to sign in or sign up before continuing") ||
    html.includes("Sign up")
  );
  // console.log(html)
  if (!isSignedInBool) {
    console.log("Error: Not signed in.");
  }
  console.log(`is signed in? ${isSignedInBool}`);
  return isSignedInBool;
}

function isEnd(html) {
  let isEndBool = html.includes("No questions match your filter");
  if (isEndBool) {
    //console.log(html)
  }
  console.log(`IsEnd? ${isEndBool}`);
  return isEndBool;
}

function sleep(ms: number) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

/* Body */

async function goodjudgmentopen_inner(cookie) {
  let i = 1;
  let response = await fetchPage(i, cookie);

  let results = [];
  let init = Date.now();
  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
  while (!isEnd(response) && isSignedIn(response)) {
    let htmlLines = response.split("\n");
    DEBUG_MODE == "on" ? htmlLines.forEach((line) => console.log(line)) : id();
    let h5elements = htmlLines.filter((str) => str.includes("<h5> <a href="));
    DEBUG_MODE == "on" ? console.log(h5elements) : id();
    let j = 0;
    for (let h5element of h5elements) {
      let h5elementSplit = h5element.split('"><span>');
      let url = h5elementSplit[0].split('<a href="')[1];
      if (!annoyingPromptUrls.includes(url)) {
        let title = h5elementSplit[1].replace("</span></a></h5>", "");
        await sleep(1000 + Math.random() * 1000); // don't be as noticeable
        try {
          let moreinfo = await fetchStats(url, cookie);
          if (moreinfo.isbinary) {
            if (!moreinfo.crowdpercentage) {
              // then request again.
              moreinfo = await fetchStats(url, cookie);
            }
          }
          let questionNumRegex = new RegExp("questions/([0-9]+)");
          let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
          let id = `goodjudmentopen-${questionNum}`;
          let question = {
            id: id,
            title: title,
            url: url,
            platform: "Good Judgment Open",
            ...moreinfo,
          };
          if (j % 30 == 0 || DEBUG_MODE == "on") {
            console.log(`Page #${i}`);
            console.log(question);
          }
          // console.log(question)
          results.push(question);
        } catch (error) {
          console.log(error);
          console.log(
            `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
          );
        }
      }
      j = j + 1;
    }
    i = i + 1;
    // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
    await sleep(5000 + Math.random() * 1000); // don't be a dick to gjopen server

    try {
      response = await fetchPage(i, cookie);
    } catch (error) {
      console.log(error);
      console.log(
        `We encountered some error when fetching page #${i}, so it won't appear on the final json`
      );
    }
  }

  if (results.length === 0) {
    console.log("Not updating results, as process was not signed in");
    return;
  }

  let end = Date.now();
  let difference = end - init;
  console.log(
    `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
  );

  return results;
}

export const goodjudmentopen: Platform = {
  name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen`
  async fetcher() {
    let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
    return await applyIfSecretExists(cookie, goodjudgmentopen_inner);
  },
};