Merge pull request #47 from QURIresearch/rootclaim-fix

Fix rootclaim fetcher
This commit is contained in:
Nuño Sempere 2022-04-06 21:14:30 -04:00 committed by GitHub
commit 7c89a1c6ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 931 additions and 27 deletions

879
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -29,6 +29,7 @@
"dependencies": {
"@tailwindcss/forms": "^0.4.0",
"@tailwindcss/typography": "^0.5.1",
"@types/jsdom": "^16.2.14",
"@types/nprogress": "^0.2.0",
"@types/react": "^17.0.39",
"airtable": "^0.11.1",
@ -48,6 +49,7 @@
"html-to-image": "^1.7.0",
"https": "^1.0.0",
"isomorphic-fetch": "^3.0.0",
"jsdom": "^19.0.0",
"json2csv": "^5.0.5",
"multiselect-react-dropdown": "^2.0.17",
"next": "12",
@ -82,7 +84,7 @@
"devDependencies": {
"@netlify/plugin-nextjs": "^4.2.4",
"@svgr/cli": "^6.2.1",
"netlify-cli": "^9.13.6",
"@types/pg": "^8.6.5"
"@types/pg": "^8.6.5",
"netlify-cli": "^9.13.6"
}
}

View File

@ -1,55 +1,82 @@
import axios from "axios";
import { JSDOM } from "jsdom";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
import { Forecast, Platform } from "./";
/* Definitions */
const platformName = "rootclaim";
let jsonEndpoint =
"https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
const jsonEndpoint =
"https://live-rootclaim-backend.herokuapp.com/analysis/public-list?limit=1000&offset=0";
async function fetchAllRootclaims() {
// for info which the polymarket graphql API
let response = await axios
const fetchAllRootclaims = async () => {
console.log(`Fetching ${jsonEndpoint}`);
const response = await axios
.get(jsonEndpoint)
.then((response) => response.data);
if (response.length != response[0] + 1) {
console.log(response.length);
console.log(response[0]);
//throw Error("Rootclaim's backend has changed.")
const claims = response.result.main_page_stories;
if (typeof claims !== "object") {
throw new Error("Expected result.main_page_stories field in API response");
}
response.shift();
return response;
return claims;
};
const fetchDescription = async (url: string, isclaim: boolean) => {
console.log(`Fetching description for ${url}`);
const response = await axios.get(url).then((response) => response.data);
const { document } = new JSDOM(response).window;
const nextDataEl = document.querySelector("#__NEXT_DATA__");
if (!nextDataEl) {
throw new Error(`Couldn't find __NEXT_DATA__ for ${url}`);
}
const data = JSON.parse(nextDataEl.innerHTML);
const mainData = data?.props?.pageProps?.initialReduxState?.main;
const info = isclaim
? mainData?.claim?.background
: mainData?.analise?.background_info;
if (!info) {
throw new Error(`Couldn't find description for page ${url}`);
}
return info;
};
export const rootclaim: Platform = {
name: platformName,
label: "Rootclaim",
color: "#0d1624",
async fetcher() {
let claims = await fetchAllRootclaims();
let results: Forecast[] = [];
for (let claim of claims) {
let id = `${platformName}-${claim.slug.toLowerCase()}`;
const claims = await fetchAllRootclaims();
const results: Forecast[] = [];
for (const claim of claims) {
const id = `${platformName}-${claim.slug.toLowerCase()}`;
let options = [];
for (let scenario of claim.scenarios) {
//console.log(scenario)
options.push({
name: toMarkdown(scenario.text)
name: toMarkdown(scenario.name || scenario.text)
.replace("\n", "")
.replace("'", "'"),
probability: scenario.net_prob / 100,
type: "PROBABILITY",
});
}
let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis";
let claimUrlPath = claim.isclaim ? "claims" : "analysis";
const url = `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`;
const description = await fetchDescription(url, claim.isclaim);
let obj: Forecast = {
id: id,
id,
title: toMarkdown(claim.question).replace("\n", ""),
url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`,
url,
platform: platformName,
description: toMarkdown(claim.background).replace("&#39;", "'"),
description: toMarkdown(description).replace("&#39;", "'"),
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {