quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: platform names, dashboards, cleanups

Browse Source
This commit is contained in:
Vyacheslav Matyukhin 2022-04-02 02:25:12 +03:00
parent 1461002070
commit 1b6ff8d440
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
10 changed files with 92 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/backend/platforms/example-fetch.ts → src/backend/platforms/_example.ts
View File

@ -5,7 +5,8 @@ import { calculateStars } from "../utils/stars";
import { Platform } from "./";
/* Definitions */
let endpoint = "https://example.com/";
const platformName = "example";
const endpoint = "https://example.com/";
/* Support functions */
@ -23,7 +24,7 @@ async function fetchData() {
async function processPredictions(predictions) {
let results = await predictions.map((prediction) => {
let id = `example-${prediction.id}`;
let id = `${platformName}-${prediction.id}`;
let probability = prediction.probability;
let options = [
{
@ -40,12 +41,12 @@ async function processPredictions(predictions) {
let result = {
title: prediction.title,
url: `https://example.com`,
platform: "Example",
platform: platformName,
description: prediction.description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Example", {
stars: calculateStars(platformName, {
/* some: somex, factors: factors */
}),
other: prediction.otherx,
@ -60,7 +61,7 @@ async function processPredictions(predictions) {
/* Body */
export const example: Platform = {
name: "example",
name: platformName,
label: "Example platform",
color: "#ff0000",
async fetcher() {

2
src/backend/platforms/betfair.ts
View File

@ -84,7 +84,7 @@ async function processPredictions(data) {
/* if(Math.floor(Math.random() * 10) % 20 ==0){
console.log(JSON.stringify(prediction, null, 4))
} */
let id = `betfair-${prediction.marketId}`;
let id = `${platformName}-${prediction.marketId}`;
let normalizationFactor = prediction.options
.filter((option) => option.status == "ACTIVE" && option.totalMatched > 0)
.map((option) => option.lastPriceTraded)

1
src/backend/platforms/givewellopenphil.ts
View File

@ -79,6 +79,7 @@ export const givewellopenphil: Platform = {
const data = JSON.parse(rawdata);
const dataWithDate = data.map((datum: any) => ({
...datum,
platform: platformName,
timestamp: "2021-02-23",
}));
return dataWithDate;

137
src/backend/platforms/infer.ts
View File

@ -3,6 +3,7 @@ import axios from "axios";
import { Tabletojson } from "tabletojson";
import { applyIfSecretExists } from "../utils/getSecrets";
import { measureTime } from "../utils/measureTime";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
import { Forecast, Platform } from "./";
@ -180,96 +181,84 @@ function sleep(ms) {
/* Body */
async function infer_inner(cookie) {
async function infer_inner(cookie: string) {
let i = 1;
let response = await fetchPage(i, cookie);
let results: Forecast[] = [];
let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n");
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.infer-pub.com/questions/")
);
// console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log(response)
console.log("questionHrefs: ");
console.log(questionHrefs);
}
await measureTime(async () => {
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n");
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.infer-pub.com/questions/")
);
// console.log(questionHrefs)
//console.log("")
//console.log("")
//console.log(h4elements)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log("questionHrefs: ");
console.log(questionHrefs);
}
for (let questionHref of questionHrefs) {
//console.log(h4element)
for (let questionHref of questionHrefs) {
let elementSplit = questionHref.split('"><span>');
let url = elementSplit[0].split('<a href="')[1];
let title = elementSplit[1]
.replace("</h4>", "")
.replace("</h5>", "")
.replace("</span></a>", "");
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
let elementSplit = questionHref.split('"><span>');
let url = elementSplit[0].split('<a href="')[1];
let title = elementSplit[1]
.replace("</h4>", "")
.replace("</h5>", "")
.replace("</span></a>", "");
try {
let moreinfo = await fetchStats(url, cookie);
let questionNumRegex = new RegExp("questions/([0-9]+)");
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
let id = `${platformName}-${questionNum}`;
let question: Forecast = {
id: id,
title: title,
url: url,
platform: platformName,
...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(`Page #${i}`);
console.log(question);
}
results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url);
console.log(question);
}
} catch (error) {
console.log(error);
console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
}
}
i++;
console.log(
"Sleeping for ~5secs so as to not be as noticeable to the infer servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
let moreinfo = await fetchStats(url, cookie);
let questionNumRegex = new RegExp("questions/([0-9]+)");
let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0];
let id = `${platformName}-${questionNum}`;
let question = {
id: id,
title: title,
url: url,
platform: "Infer",
...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(`Page #${i}`);
console.log(question);
}
results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url);
console.log(question);
}
response = await fetchPage(i, cookie);
} catch (error) {
console.log(error);
console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
);
}
}
i++;
//i=Number(i)+1
console.log(
"Sleeping for ~5secs so as to not be as noticeable to the infer servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
response = await fetchPage(i, cookie);
} catch (error) {
console.log(error);
console.log(
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
);
}
}
let end = Date.now();
let difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
});
if (results.length === 0) {
console.log("Not updating results, as process was not signed in");

20
src/backend/platforms/manifold.ts
View File

@ -2,7 +2,7 @@
import axios from "axios";
import { calculateStars } from "../utils/stars";
import { Platform } from "./";
import { Forecast, Platform } from "./";
/* Definitions */
const platformName = "manifold";
@ -23,7 +23,7 @@ async function fetchData() {
return response;
}
function showStatistics(results) {
function showStatistics(results: Forecast[]) {
console.log(`Num unresolved markets: ${results.length}`);
let sum = (arr) => arr.reduce((tally, a) => tally + a, 0);
let num2StarsOrMore = results.filter(
@ -44,8 +44,8 @@ function showStatistics(results) {
}
async function processPredictions(predictions) {
let results = await predictions.map((prediction) => {
let id = `manifold-${prediction.id}`; // oops, doesn't match platform name
let results: Forecast[] = await predictions.map((prediction) => {
let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name
let probability = prediction.probability;
let options = [
{
@ -59,11 +59,11 @@ async function processPredictions(predictions) {
type: "PROBABILITY",
},
];
let result = {
const result: Forecast = {
id: id,
title: prediction.question,
url: prediction.url,
platform: "Manifold Markets",
platform: platformName,
description: prediction.description,
options: options,
timestamp: new Date().toISOString(),
@ -84,9 +84,11 @@ async function processPredictions(predictions) {
};
return result;
});
let unresolvedResults = results.filter((result) => !result.extra.isResolved);
// console.log(unresolvedResults);
return unresolvedResults; //resultsProcessed
const unresolvedResults = results.filter(
(result) => !result.extra.isResolved
);
return unresolvedResults;
}
export const manifold: Platform = {

2
src/backend/platforms/metaculus.ts
View File

@ -50,7 +50,7 @@ async function fetchMetaculusQuestions(next) {
return data;
}
function sleep(ms) {
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}

4
src/backend/platforms/smarkets.ts
View File

@ -1,7 +1,7 @@
import axios from "axios";
import { calculateStars } from "../utils/stars";
import { Platform } from "./";
import { Forecast, Platform } from "./";
/* Definitions */
const platformName = "smarkets";
@ -159,7 +159,7 @@ export const smarkets: Platform = {
name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`)
}
*/
let result = {
let result: Forecast = {
id: id,
title: name,
url: "https://smarkets.com/event/" + market.event_id + market.slug,

6
src/backend/utils/getSecrets.ts
View File

@ -1,10 +1,10 @@
export async function applyIfSecretExists<T>(
cookie,
fun: (...args: any[]) => T
cookie: string,
fun: (cookie: string) => T
) {
if (cookie) {
return await fun(cookie);
} else if (!cookie) {
} else {
console.log(
`Cannot proceed with ${fun.name} because cookie does not exist`
);

2
src/pages/dashboards.tsx
View File

@ -93,7 +93,7 @@ export default function Home({
let isGraubardEasterEgg = (name) => (name == "Clay Graubard" ? true : false);
return (
<Layout key="index" page={"dashboard"}>
<Layout page="dashboard">
{/* Display forecasts */}
<div className="mt-7 mb-7">
<h1

15
src/web/worker/getDashboardForecasts.ts
View File

@ -2,7 +2,7 @@ import axios from "axios";
export async function getDashboardForecastsByDashboardId({ dashboardId }) {
console.log("getDashboardForecastsByDashboardId: ");
let dashboardForecastCompatibleWithFuse = [];
let dashboardContents = [];
let dashboardItem = null;
try {
let { data } = await axios({
@ -13,22 +13,13 @@ export async function getDashboardForecastsByDashboardId({ dashboardId }) {
},
});
console.log(data);
let dashboardContents = data.dashboardContents;
dashboardContents = data.dashboardContents;
dashboardItem = data.dashboardItem;
// let { dashboardContents, dashboardItem } = data
if (!!dashboardContents && !!dashboardContents.map) {
dashboardForecastCompatibleWithFuse = dashboardContents.map((result) => ({
item: result,
score: 0,
}));
} else {
console.log("Error in getDashboardForecastsByDashboardId");
}
} catch (error) {
console.log(error);
} finally {
return {
dashboardForecasts: dashboardForecastCompatibleWithFuse,
dashboardForecasts: dashboardContents,
dashboardItem,
};
}