Merge pull request #47 from QURIresearch/rootclaim-fix

Fix rootclaim fetcher

package.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@tailwindcss/forms": "^0.4.0",
     "@tailwindcss/typography": "^0.5.1",
+    "@types/jsdom": "^16.2.14",
     "@types/nprogress": "^0.2.0",
     "@types/react": "^17.0.39",
     "airtable": "^0.11.1",
@@ -48,6 +49,7 @@
     "html-to-image": "^1.7.0",
     "https": "^1.0.0",
     "isomorphic-fetch": "^3.0.0",
+    "jsdom": "^19.0.0",
     "json2csv": "^5.0.5",
     "multiselect-react-dropdown": "^2.0.17",
     "next": "12",
@@ -82,7 +84,7 @@
   "devDependencies": {
     "@netlify/plugin-nextjs": "^4.2.4",
     "@svgr/cli": "^6.2.1",
-    "netlify-cli": "^9.13.6",
+    "@types/pg": "^8.6.5",
-    "@types/pg": "^8.6.5"
+    "netlify-cli": "^9.13.6"
   }
 }

src/backend/platforms/rootclaim.ts

@@ -1,55 +1,82 @@
 import axios from "axios";
+import { JSDOM } from "jsdom";
 
 import { calculateStars } from "../utils/stars";
 import toMarkdown from "../utils/toMarkdown";
 import { Forecast, Platform } from "./";
 
-/* Definitions */
 const platformName = "rootclaim";
-let jsonEndpoint =
+const jsonEndpoint =
-  "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
+  "https://live-rootclaim-backend.herokuapp.com/analysis/public-list?limit=1000&offset=0";
 
-async function fetchAllRootclaims() {
+const fetchAllRootclaims = async () => {
-  // for info which the polymarket graphql API
+  console.log(`Fetching ${jsonEndpoint}`);
-  let response = await axios
+  const response = await axios
     .get(jsonEndpoint)
     .then((response) => response.data);
-  if (response.length != response[0] + 1) {
+
-    console.log(response.length);
+  const claims = response.result.main_page_stories;
-    console.log(response[0]);
+  if (typeof claims !== "object") {
-    //throw Error("Rootclaim's backend has changed.")
+    throw new Error("Expected result.main_page_stories field in API response");
   }
-  response.shift();
+  return claims;
-  return response;
+};
-}
+
+const fetchDescription = async (url: string, isclaim: boolean) => {
+  console.log(`Fetching description for ${url}`);
+  const response = await axios.get(url).then((response) => response.data);
+
+  const { document } = new JSDOM(response).window;
+  const nextDataEl = document.querySelector("#__NEXT_DATA__");
+  if (!nextDataEl) {
+    throw new Error(`Couldn't find __NEXT_DATA__ for ${url}`);
+  }
+
+  const data = JSON.parse(nextDataEl.innerHTML);
+  const mainData = data?.props?.pageProps?.initialReduxState?.main;
+  const info = isclaim
+    ? mainData?.claim?.background
+    : mainData?.analise?.background_info;
+
+  if (!info) {
+    throw new Error(`Couldn't find description for page ${url}`);
+  }
+  return info;
+};
 
 export const rootclaim: Platform = {
   name: platformName,
   label: "Rootclaim",
   color: "#0d1624",
   async fetcher() {
-    let claims = await fetchAllRootclaims();
+    const claims = await fetchAllRootclaims();
-    let results: Forecast[] = [];
+    const results: Forecast[] = [];
-    for (let claim of claims) {
+
-      let id = `${platformName}-${claim.slug.toLowerCase()}`;
+    for (const claim of claims) {
+      const id = `${platformName}-${claim.slug.toLowerCase()}`;
+
       let options = [];
       for (let scenario of claim.scenarios) {
-        //console.log(scenario)
         options.push({
-          name: toMarkdown(scenario.text)
+          name: toMarkdown(scenario.name || scenario.text)
             .replace("\n", "")
             .replace("'", "'"),
           probability: scenario.net_prob / 100,
           type: "PROBABILITY",
         });
       }
-      let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
+
+      let claimUrlPath = claim.isclaim ? "claims" : "analysis";
+      const url = `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`;
+
+      const description = await fetchDescription(url, claim.isclaim);
+
       let obj: Forecast = {
-        id: id,
+        id,
         title: toMarkdown(claim.question).replace("\n", ""),
-        url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
+        url,
         platform: platformName,
-        description: toMarkdown(claim.background).replace("&#39;", "'"),
+        description: toMarkdown(description).replace("&#39;", "'"),
         options: options,
         timestamp: new Date().toISOString(),
         qualityindicators: {