From 439a9045da6993be2abffe3701ec79e59f82a145 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Mon, 9 May 2022 21:37:28 +0400 Subject: [PATCH 1/6] refactor: some typescript improvements --- .../flow/rebuildNetliftySiteWithNewData.ts | 4 +- src/backend/platforms/betfair.ts | 7 ++- src/backend/platforms/givewellopenphil.ts | 4 +- src/backend/utils/algolia.ts | 5 +- src/web/display/DashboardCreator.tsx | 4 +- src/web/display/SliderElement.tsx | 36 ++++++------ .../components/QuestionCard/index.tsx | 2 +- src/web/questions/pages/QuestionPage.tsx | 2 +- src/web/search/components/QueryForm.tsx | 4 +- src/web/search/components/SearchScreen.tsx | 26 +++++---- src/web/urql.ts | 3 + src/web/worker/searchWithAlgolia.ts | 56 +++++++++---------- 12 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/backend/flow/rebuildNetliftySiteWithNewData.ts b/src/backend/flow/rebuildNetliftySiteWithNewData.ts index 85edd16..72ab28c 100644 --- a/src/backend/flow/rebuildNetliftySiteWithNewData.ts +++ b/src/backend/flow/rebuildNetliftySiteWithNewData.ts @@ -2,7 +2,7 @@ import axios from "axios"; import { applyIfSecretExists } from "../utils/getSecrets"; -async function rebuildNetlifySiteWithNewData_inner(cookie) { +async function rebuildNetlifySiteWithNewData_inner(cookie: string) { let payload = {}; let response = await axios.post(cookie, payload); let data = response.data; @@ -10,6 +10,6 @@ async function rebuildNetlifySiteWithNewData_inner(cookie) { } export async function rebuildNetlifySiteWithNewData() { - let cookie = process.env.REBUIDNETLIFYHOOKURL; + const cookie = process.env.REBUIDNETLIFYHOOKURL || ""; await applyIfSecretExists(cookie, rebuildNetlifySiteWithNewData_inner); } diff --git a/src/backend/platforms/betfair.ts b/src/backend/platforms/betfair.ts index 4af99fc..6e71ebe 100644 --- a/src/backend/platforms/betfair.ts +++ b/src/backend/platforms/betfair.ts @@ -8,10 +8,10 @@ import { FetchedQuestion, Platform } from "./"; const platformName = "betfair"; /* Definitions */ -let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; +const endpoint = process.env.SECRET_BETFAIR_ENDPOINT; /* Utilities */ -let arraysEqual = (a, b) => { +const arraysEqual = (a: string[], b: string[]) => { if (a === b) return true; if (a == null || b == null) return false; if (a.length !== b.length) return false; @@ -26,7 +26,8 @@ let arraysEqual = (a, b) => { } return true; }; -let mergeRunners = (runnerCatalog, runnerBook) => { + +const mergeRunners = (runnerCatalog, runnerBook) => { let keys = Object.keys(runnerCatalog); let result = []; for (let key of keys) { diff --git a/src/backend/platforms/givewellopenphil.ts b/src/backend/platforms/givewellopenphil.ts index 22c37b8..e8d9754 100644 --- a/src/backend/platforms/givewellopenphil.ts +++ b/src/backend/platforms/givewellopenphil.ts @@ -8,8 +8,8 @@ import { Platform } from "./"; const platformName = "givewellopenphil"; /* Support functions */ -async function fetchPage(url: string) { - let response = await axios({ +async function fetchPage(url: string): Promise { + const response = await axios({ url: url, method: "GET", headers: { diff --git a/src/backend/utils/algolia.ts b/src/backend/utils/algolia.ts index 5728fec..e50c2f6 100644 --- a/src/backend/utils/algolia.ts +++ b/src/backend/utils/algolia.ts @@ -5,13 +5,14 @@ import { Question } from "@prisma/client"; import { prisma } from "../database/prisma"; import { platforms } from "../platforms"; -let cookie = process.env.ALGOLIA_MASTER_API_KEY; -const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID; +let cookie = process.env.ALGOLIA_MASTER_API_KEY || ""; +const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || ""; const client = algoliasearch(algoliaAppId, cookie); const index = client.initIndex("metaforecast"); export type AlgoliaQuestion = Omit & { timestamp: string; + optionsstringforsearch?: string; }; const getoptionsstringforsearch = (record: Question): string => { diff --git a/src/web/display/DashboardCreator.tsx b/src/web/display/DashboardCreator.tsx index 4e40e15..48ddde5 100644 --- a/src/web/display/DashboardCreator.tsx +++ b/src/web/display/DashboardCreator.tsx @@ -1,4 +1,4 @@ -import React, { EventHandler, SyntheticEvent, useState } from "react"; +import React, { ChangeEvent, EventHandler, SyntheticEvent, useState } from "react"; import { Button } from "../common/Button"; import { InfoBox } from "../common/InfoBox"; @@ -18,7 +18,7 @@ export const DashboardCreator: React.FC = ({ handleSubmit }) => { const [value, setValue] = useState(exampleInput); const [acting, setActing] = useState(false); - const handleChange = (event) => { + const handleChange = (event: ChangeEvent) => { setValue(event.target.value); }; diff --git a/src/web/display/SliderElement.tsx b/src/web/display/SliderElement.tsx index c14da06..9112d0f 100644 --- a/src/web/display/SliderElement.tsx +++ b/src/web/display/SliderElement.tsx @@ -1,6 +1,7 @@ -/* Imports */ import React from "react"; -import { Handles, Rail, Slider, Tracks } from "react-compound-slider"; +import { + GetHandleProps, GetTrackProps, Handles, Rail, Slider, SliderItem, Tracks +} from "react-compound-slider"; // https://sghall.github.io/react-compound-slider/#/getting-started/tutorial @@ -24,12 +25,11 @@ const railStyle = { }; /* Support functions */ -function Handle({ - handle: { id, value, percent }, - getHandleProps, - displayFunction, - handleWidth, -}) { +const Handle: React.FC<{ + handle: SliderItem; + getHandleProps: GetHandleProps; + displayFunction: (value: number) => string; +}> = ({ handle: { id, value, percent }, getHandleProps, displayFunction }) => { return ( <>
@@ -53,9 +53,13 @@ function Handle({ >
); -} +}; -function Track({ source, target, getTrackProps }) { +const Track: React.FC<{ + source: SliderItem; + target: SliderItem; + getTrackProps: GetTrackProps; +}> = ({ source, target, getTrackProps }) => { return (
); -} +}; interface Props { value: number; - onChange: (event: any) => void; + onChange: (value: number) => void; displayFunction: (value: number) => string; } /* Body */ -// Two functions, essentially identical. export const SliderElement: React.FC = ({ onChange, value, @@ -96,21 +99,20 @@ export const SliderElement: React.FC = ({ } domain={[0, 200]} values={[value]} - onChange={onChange} + onChange={(values) => onChange(values[0])} > {({ getRailProps }) =>
} {({ handles, getHandleProps }) => ( -
+
{handles.map((handle) => ( ))}
@@ -118,7 +120,7 @@ export const SliderElement: React.FC = ({ {({ tracks, getTrackProps }) => ( -
+
{tracks.map(({ id, source, target }) => ( = ({
)} - {question.platform.id === "guesstimate" && ( + {question.platform.id === "guesstimate" && question.visualization && (
- {question.platform.id === "guesstimate" ? ( + {question.platform.id === "guesstimate" && question.visualization ? ( void; @@ -9,7 +11,7 @@ export const QueryForm: React.FC = ({ onChange, placeholder, }) => { - const handleInputChange = (event) => { + const handleInputChange = (event: ChangeEvent) => { event.preventDefault(); onChange(event.target.value); // In this case, the query, e.g. "COVID.19" }; diff --git a/src/web/search/components/SearchScreen.tsx b/src/web/search/components/SearchScreen.tsx index 63c54cd..31d09c7 100644 --- a/src/web/search/components/SearchScreen.tsx +++ b/src/web/search/components/SearchScreen.tsx @@ -1,5 +1,5 @@ import { useRouter } from "next/router"; -import React, { Fragment, useMemo, useState } from "react"; +import React, { useMemo, useState } from "react"; import { useQuery } from "urql"; import { PlatformConfig } from "../../../backend/platforms"; @@ -126,7 +126,8 @@ export const SearchScreen: React.FC = ({ }; const updateRoute = () => { - const stringify = (key: string, value: any) => { + const stringify = (key: string, obj: { [k: string]: any }) => { + const value = obj[key]; if (key === "forecastingPlatforms") { return value.join("|"); } else { @@ -134,15 +135,16 @@ export const SearchScreen: React.FC = ({ } }; - const query = {}; + const query: { [k: string]: string } = {}; for (const key of Object.keys(defaultQueryParameters)) { - const value = stringify(key, queryParameters[key]); - const defaultValue = stringify(key, defaultQueryParameters[key]); + const value = stringify(key, queryParameters); + const defaultValue = stringify(key, defaultQueryParameters); if (value === defaultValue) continue; query[key] = value; } - if (numDisplay !== defaultNumDisplay) query["numDisplay"] = numDisplay; + if (numDisplay !== defaultNumDisplay) + query["numDisplay"] = String(numDisplay); router.replace( { @@ -191,8 +193,8 @@ export const SearchScreen: React.FC = ({ (Math.round(value) === 1 ? "" : "s") ); }; - const onChangeSliderForNumDisplay = (event) => { - setNumDisplay(Math.round(event[0])); + const onChangeSliderForNumDisplay = (value: number) => { + setNumDisplay(Math.round(value)); setForceSearch(forceSearch + 1); // FIXME - force new search iff numDisplay is greater than last search limit }; @@ -200,10 +202,10 @@ export const SearchScreen: React.FC = ({ const displayFunctionNumForecasts = (value: number) => { return "# Forecasts > " + Math.round(value); }; - const onChangeSliderForNumForecasts = (event) => { + const onChangeSliderForNumForecasts = (value: number) => { setQueryParameters({ ...queryParameters, - forecastsThreshold: Math.round(event[0]), + forecastsThreshold: Math.round(value), }); }; @@ -230,7 +232,7 @@ export const SearchScreen: React.FC = ({ /* Final return */ return ( - + <> + ); }; diff --git a/src/web/urql.ts b/src/web/urql.ts index c083933..a691bdd 100644 --- a/src/web/urql.ts +++ b/src/web/urql.ts @@ -36,5 +36,8 @@ export const getUrqlClientOptions = (ssr: SSRExchange) => ({ export const ssrUrql = () => { const ssrCache = ssrExchange({ isClient: false }); const client = initUrqlClient(getUrqlClientOptions(ssrCache), false); + if (!client) { + throw new Error("Expected non-null client instance from initUrqlClient"); + } return [ssrCache, client] as const; }; diff --git a/src/web/worker/searchWithAlgolia.ts b/src/web/worker/searchWithAlgolia.ts index 9eba5b4..7b53b14 100644 --- a/src/web/worker/searchWithAlgolia.ts +++ b/src/web/worker/searchWithAlgolia.ts @@ -1,18 +1,31 @@ import algoliasearch from "algoliasearch"; +import { Hit } from "@algolia/client-search"; + import { AlgoliaQuestion } from "../../backend/utils/algolia"; const client = algoliasearch( - process.env.NEXT_PUBLIC_ALGOLIA_APP_ID, - process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_KEY + process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || "", + process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_KEY || "" ); const index = client.initIndex("metaforecast"); -let buildFilter = ({ +interface SearchOpts { + queryString: string; + hitsPerPage?: number; + starsThreshold: number; + filterByPlatforms: string[]; + forecastsThreshold: number; +} + +const buildFilter = ({ starsThreshold, filterByPlatforms, forecastsThreshold, -}) => { +}: Pick< + SearchOpts, + "starsThreshold" | "filterByPlatforms" | "forecastsThreshold" +>) => { const starsFilter = starsThreshold ? `qualityindicators.stars >= ${starsThreshold}` : null; @@ -35,26 +48,15 @@ let buildFilter = ({ return finalFilter; }; -let buildFacetFilter = ({ filterByPlatforms }) => { - let platformsFilter = []; - if (filterByPlatforms.length > 0) { - platformsFilter = [ - [filterByPlatforms.map((platform) => `platform:${platform}`)], - ]; - } - console.log(platformsFilter); - console.log( - "searchWithAlgolia.js/searchWithAlgolia/buildFacetFilter", - platformsFilter - ); - return platformsFilter; -}; - -let noExactMatch = (queryString, result) => { +const noExactMatch = (queryString: string, result: Hit) => { queryString = queryString.toLowerCase(); - let title = result.title.toLowerCase(); - let description = result.description.toLowerCase(); - let optionsstringforsearch = result.optionsstringforsearch.toLowerCase(); + + const title = result.title.toLowerCase(); + const description = result.description.toLowerCase(); + const optionsstringforsearch = ( + result.optionsstringforsearch || "" + ).toLowerCase(); + return !( title.includes(queryString) || description.includes(queryString) || @@ -62,14 +64,6 @@ let noExactMatch = (queryString, result) => { ); }; -interface SearchOpts { - queryString: string; - hitsPerPage?: number; - starsThreshold: number; - filterByPlatforms: string[]; - forecastsThreshold: number; -} - // only query string export default async function searchWithAlgolia({ queryString, From f6e2e8cfa1d8e3f2201a97beaa22311266868886 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Mon, 9 May 2022 23:27:51 +0400 Subject: [PATCH 2/6] refactor: more typescript --- package-lock.json | 23 +- package.json | 2 + src/backend/platforms/betfair.ts | 13 +- src/backend/platforms/goodjudgment.ts | 5 +- src/backend/platforms/infer.ts | 5 +- src/backend/utils/roughSize.ts | 25 -- src/backend/utils/stars.ts | 30 +-- src/backend/utils/toMarkdown.ts | 30 +-- src/graphql/schema/dashboards.ts | 1 + src/graphql/schema/questions.ts | 2 +- src/graphql/schema/search.ts | 14 +- src/pages/dashboards/embed/[id].tsx | 16 +- src/pages/index.tsx | 9 +- src/pages/secretEmbed.tsx | 23 +- src/web/common/Layout.tsx | 2 +- src/web/common/MultiSelectPlatform.tsx | 2 +- src/web/display/DashboardCreator.tsx | 4 +- src/web/icons/Favicon.tsx | 8 +- src/web/icons/Logo.tsx | 8 +- src/web/icons/Logo2.tsx | 8 +- src/web/icons/index.ts | 6 +- .../components/HistoryChart/InnerChart.tsx | 8 +- .../components/HistoryChart/Legend.tsx | 8 +- .../components/HistoryChart/index.tsx | 3 +- .../components/HistoryChart/utils.ts | 10 +- .../questions/components/IndicatorsTable.tsx | 10 +- .../QuestionCard/QuestionFooter.tsx | 49 ++-- .../components/QuestionCard/index.tsx | 53 +--- .../components/QuestionIndicators.tsx | 227 ------------------ .../questions/components/QuestionOptions.tsx | 30 ++- src/web/questions/utils.ts | 17 ++ src/web/search/components/SearchScreen.tsx | 2 +- src/web/utils.ts | 11 + src/web/worker/searchWithAlgolia.ts | 8 +- 34 files changed, 209 insertions(+), 463 deletions(-) delete mode 100644 src/backend/utils/roughSize.ts delete mode 100644 src/web/questions/components/QuestionIndicators.tsx diff --git a/package-lock.json b/package-lock.json index 1ae8f10..4560da0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,7 +5,6 @@ "requires": true, "packages": { "": { - "name": "metaforecast", "version": "2.0.0", "license": "MIT", "dependencies": { @@ -17,11 +16,13 @@ "@prisma/client": "^3.11.1", "@tailwindcss/forms": "^0.4.0", "@tailwindcss/typography": "^0.5.1", + "@types/chroma-js": "^2.1.3", "@types/dom-to-image": "^2.6.4", "@types/jsdom": "^16.2.14", "@types/nprogress": "^0.2.0", "@types/react": "^17.0.39", "@types/react-copy-to-clipboard": "^5.0.2", + "@types/textversionjs": "^1.1.1", "airtable": "^0.11.1", "algoliasearch": "^4.10.3", "autoprefixer": "^10.1.0", @@ -3170,6 +3171,11 @@ "@types/responselike": "*" } }, + "node_modules/@types/chroma-js": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@types/chroma-js/-/chroma-js-2.1.3.tgz", + "integrity": "sha512-1xGPhoSGY1CPmXLCBcjVZSQinFjL26vlR8ZqprsBWiFyED4JacJJ9zHhh5aaUXqbY9B37mKQ73nlydVAXmr1+g==" + }, "node_modules/@types/cross-spawn": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cross-spawn/-/cross-spawn-6.0.2.tgz", @@ -3354,6 +3360,11 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "license": "MIT" }, + "node_modules/@types/textversionjs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/textversionjs/-/textversionjs-1.1.1.tgz", + "integrity": "sha512-xXa08oZ76+J2rS36guKd6zJFAbkRtUnuDb0R6OFtY93VTuXdi94k0ycsIBrsq/Av8DmiQVfTYE3k7KCEUVYsag==" + }, "node_modules/@types/tough-cookie": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", @@ -42405,6 +42416,11 @@ "@types/responselike": "*" } }, + "@types/chroma-js": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@types/chroma-js/-/chroma-js-2.1.3.tgz", + "integrity": "sha512-1xGPhoSGY1CPmXLCBcjVZSQinFjL26vlR8ZqprsBWiFyED4JacJJ9zHhh5aaUXqbY9B37mKQ73nlydVAXmr1+g==" + }, "@types/cross-spawn": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cross-spawn/-/cross-spawn-6.0.2.tgz", @@ -42577,6 +42593,11 @@ "resolved": "https://registry.npmjs.org/@types%2fscheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, + "@types/textversionjs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/textversionjs/-/textversionjs-1.1.1.tgz", + "integrity": "sha512-xXa08oZ76+J2rS36guKd6zJFAbkRtUnuDb0R6OFtY93VTuXdi94k0ycsIBrsq/Av8DmiQVfTYE3k7KCEUVYsag==" + }, "@types/tough-cookie": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", diff --git a/package.json b/package.json index 07dbc88..2eecce6 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,13 @@ "@prisma/client": "^3.11.1", "@tailwindcss/forms": "^0.4.0", "@tailwindcss/typography": "^0.5.1", + "@types/chroma-js": "^2.1.3", "@types/dom-to-image": "^2.6.4", "@types/jsdom": "^16.2.14", "@types/nprogress": "^0.2.0", "@types/react": "^17.0.39", "@types/react-copy-to-clipboard": "^5.0.2", + "@types/textversionjs": "^1.1.1", "airtable": "^0.11.1", "algoliasearch": "^4.10.3", "autoprefixer": "^10.1.0", diff --git a/src/backend/platforms/betfair.ts b/src/backend/platforms/betfair.ts index 6e71ebe..2fec2cf 100644 --- a/src/backend/platforms/betfair.ts +++ b/src/backend/platforms/betfair.ts @@ -42,7 +42,7 @@ async function fetchPredictions() { const agent = new https.Agent({ rejectUnauthorized: false, }); - let response = await axios({ + const response = await axios({ url: endpoint, method: "GET", headers: { @@ -115,18 +115,19 @@ async function processPredictions(data) { if (rules == undefined) { // console.log(prediction.description) } + let title = rules.split("? ")[0] + "?"; let description = rules.split("? ")[1].trim(); if (title.includes("of the named")) { title = prediction.marketName + ": " + title; } - let result = { - id: id, - title: title, + const result = { + id, + title, url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`, platform: platformName, - description: description, - options: options, + description, + options, qualityindicators: { stars: calculateStars(platformName, { volume: prediction.totalMatched, diff --git a/src/backend/platforms/goodjudgment.ts b/src/backend/platforms/goodjudgment.ts index 499e8f0..3d191ea 100644 --- a/src/backend/platforms/goodjudgment.ts +++ b/src/backend/platforms/goodjudgment.ts @@ -9,10 +9,7 @@ import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "goodjudgment"; -let endpoint = "https://goodjudgment.io/superforecasts/"; -String.prototype.replaceAll = function replaceAll(search, replace) { - return this.split(search).join(replace); -}; +const endpoint = "https://goodjudgment.io/superforecasts/"; /* Body */ export const goodjudgment: Platform = { diff --git a/src/backend/platforms/infer.ts b/src/backend/platforms/infer.ts index cf62fd0..d6133c6 100644 --- a/src/backend/platforms/infer.ts +++ b/src/backend/platforms/infer.ts @@ -9,10 +9,7 @@ import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "infer"; -let htmlEndPoint = "https://www.infer-pub.com/questions"; -String.prototype.replaceAll = function replaceAll(search, replace) { - return this.split(search).join(replace); -}; +const htmlEndPoint = "https://www.infer-pub.com/questions"; const DEBUG_MODE: "on" | "off" = "off"; // "off" const SLEEP_TIME_RANDOM = 7000; // miliseconds const SLEEP_TIME_EXTRA = 2000; diff --git a/src/backend/utils/roughSize.ts b/src/backend/utils/roughSize.ts deleted file mode 100644 index 75fff92..0000000 --- a/src/backend/utils/roughSize.ts +++ /dev/null @@ -1,25 +0,0 @@ -export function roughSizeOfObject(object) { - var objectList = []; - var stack = [object]; - var bytes = 0; - - while (stack.length) { - var value = stack.pop(); - if (typeof value === "boolean") { - bytes += 4; - } else if (typeof value === "string") { - bytes += value.length * 2; - } else if (typeof value === "number") { - bytes += 8; - } else if (typeof value === "object" && objectList.indexOf(value) === -1) { - objectList.push(value); - - for (var i in value) { - stack.push(value[i]); - } - } - } - let megaBytes = bytes / 1024 ** 2; - let megaBytesRounded = Math.round(megaBytes * 10) / 10; - return megaBytesRounded; -} diff --git a/src/backend/utils/stars.ts b/src/backend/utils/stars.ts index c0a7dae..5150394 100644 --- a/src/backend/utils/stars.ts +++ b/src/backend/utils/stars.ts @@ -1,31 +1,5 @@ -export function getStarSymbols(numstars) { - let stars = "★★☆☆☆"; - switch (numstars) { - case 0: - stars = "☆☆☆☆☆"; - break; - case 1: - stars = "★☆☆☆☆"; - break; - case 2: - stars = "★★☆☆☆"; - break; - case 3: - stars = "★★★☆☆"; - break; - case 4: - stars = "★★★★☆"; - break; - case 5: - stars = "★★★★★"; - break; - default: - stars = "★★☆☆☆"; - } - return stars; -} - -let average = (array) => array.reduce((a, b) => a + b, 0) / array.length; +let average = (array: number[]) => + array.reduce((a, b) => a + b, 0) / array.length; function calculateStarsAstralCodexTen(data) { let nuno = (data) => 3; diff --git a/src/backend/utils/toMarkdown.ts b/src/backend/utils/toMarkdown.ts index 31b59ae..b227af6 100644 --- a/src/backend/utils/toMarkdown.ts +++ b/src/backend/utils/toMarkdown.ts @@ -1,26 +1,16 @@ -/* Imports */ import textVersion from "textversionjs"; -/* Definitions */ -String.prototype.replaceAll = function replaceAll(search, replace) { - return this.split(search).join(replace); -}; - -var styleConfig = { - linkProcess: function (href, linkText) { - let newHref = href ? href.replace(/\(/g, "%28").replace(/\)/g, "%29") : ""; - // Deal corretly in markdown with links that contain parenthesis - return `[${linkText}](${newHref})`; - }, -}; - -/* Support functions */ - -/* Body */ - -export default function toMarkdown(htmlText) { +export default function toMarkdown(htmlText: string) { let html2 = htmlText.replaceAll(`='`, `="`).replaceAll(`'>`, `">`); - return textVersion(html2, styleConfig); + return textVersion(html2, { + linkProcess: (href, linkText) => { + let newHref = href + ? href.replace(/\(/g, "%28").replace(/\)/g, "%29") + : ""; + // Deal correctly in markdown with links that contain parenthesis + return `[${linkText}](${newHref})`; + }, + }); } // toMarkdown() diff --git a/src/graphql/schema/dashboards.ts b/src/graphql/schema/dashboards.ts index f9837a6..8138fc2 100644 --- a/src/graphql/schema/dashboards.ts +++ b/src/graphql/schema/dashboards.ts @@ -36,6 +36,7 @@ const DashboardObj = builder.objectRef("Dashboard").implement({ builder.queryField("dashboard", (t) => t.field({ type: DashboardObj, + nullable: true, description: "Look up a single dashboard by its id", args: { id: t.arg({ type: "ID", required: true }), diff --git a/src/graphql/schema/questions.ts b/src/graphql/schema/questions.ts index f3ec7e2..fb28bed 100644 --- a/src/graphql/schema/questions.ts +++ b/src/graphql/schema/questions.ts @@ -140,6 +140,7 @@ builder.queryField("questions", (t) => builder.queryField("question", (t) => t.field({ type: QuestionObj, + nullable: true, description: "Look up a single question by its id", args: { id: t.arg({ type: "ID", required: true }), @@ -149,7 +150,6 @@ builder.queryField("question", (t) => const [platform, id] = [parts[0], parts.slice(1).join("-")]; if (platform === "guesstimate") { const q = await guesstimate.fetchQuestion(Number(id)); - console.log(q); return q; } return await prisma.question.findUnique({ diff --git a/src/graphql/schema/search.ts b/src/graphql/schema/search.ts index 4676b45..6039e21 100644 --- a/src/graphql/schema/search.ts +++ b/src/graphql/schema/search.ts @@ -32,19 +32,19 @@ builder.queryField("searchQuestions", (t) => // defs const query = input.query === undefined ? "" : input.query; if (query === "") return []; - const forecastsThreshold = input.forecastsThreshold; - const starsThreshold = input.starsThreshold; + const { forecastsThreshold, starsThreshold } = input; + const platformsIncludeGuesstimate = input.forecastingPlatforms?.includes("guesstimate") && - starsThreshold <= 1; + (!starsThreshold || starsThreshold <= 1); // preparation const unawaitedAlgoliaResponse = searchWithAlgolia({ queryString: query, - hitsPerPage: input.limit + 50, - starsThreshold, - filterByPlatforms: input.forecastingPlatforms, - forecastsThreshold, + hitsPerPage: input.limit ?? 50, + starsThreshold: starsThreshold ?? undefined, + filterByPlatforms: input.forecastingPlatforms ?? undefined, + forecastsThreshold: forecastsThreshold ?? undefined, }); let results: AlgoliaQuestion[] = []; diff --git a/src/pages/dashboards/embed/[id].tsx b/src/pages/dashboards/embed/[id].tsx index 09bbe79..d91e0ad 100644 --- a/src/pages/dashboards/embed/[id].tsx +++ b/src/pages/dashboards/embed/[id].tsx @@ -1,5 +1,5 @@ import { GetServerSideProps, NextPage } from "next"; -import Error from "next/error"; +import NextError from "next/error"; import { DashboardByIdDocument, DashboardFragment @@ -19,9 +19,13 @@ export const getServerSideProps: GetServerSideProps = async ( const dashboardId = context.query.id as string; const numCols = Number(context.query.numCols); - const dashboard = ( - await client.query(DashboardByIdDocument, { id: dashboardId }).toPromise() - ).data?.result; + const response = await client + .query(DashboardByIdDocument, { id: dashboardId }) + .toPromise(); + if (!response.data) { + throw new Error(`GraphQL query failed: ${response.error}`); + } + const dashboard = response.data.result; if (!dashboard) { context.res.statusCode = 404; @@ -32,14 +36,14 @@ export const getServerSideProps: GetServerSideProps = async ( // reduntant: page component doesn't do graphql requests, but it's still nice/more consistent to have data in cache urqlState: ssrCache.extractData(), dashboard, - numCols: !numCols ? null : numCols < 5 ? numCols : 4, + numCols: !numCols ? undefined : numCols < 5 ? numCols : 4, }, }; }; const EmbedDashboardPage: NextPage = ({ dashboard, numCols }) => { if (!dashboard) { - return ; + return ; } return ( diff --git a/src/pages/index.tsx b/src/pages/index.tsx index b8cb2ae..1ea9ad9 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -45,8 +45,11 @@ export const getServerSideProps: GetServerSideProps = async ( const defaultNumDisplay = 21; const initialNumDisplay = Number(urlQuery.numDisplay) || defaultNumDisplay; - const defaultResults = (await client.query(FrontpageDocument).toPromise()) - .data.result; + const response = await client.query(FrontpageDocument).toPromise(); + if (!response.data) { + throw new Error(`GraphQL query failed: ${response.error}`); + } + const defaultResults = response.data.result; if ( !!initialQueryParameters && @@ -58,7 +61,7 @@ export const getServerSideProps: GetServerSideProps = async ( .query(SearchDocument, { input: { ...initialQueryParameters, - limit: initialNumDisplay, + limit: initialNumDisplay + 50, }, }) .toPromise(); diff --git a/src/pages/secretEmbed.tsx b/src/pages/secretEmbed.tsx index fd7f222..7420734 100644 --- a/src/pages/secretEmbed.tsx +++ b/src/pages/secretEmbed.tsx @@ -29,16 +29,19 @@ export const getServerSideProps: GetServerSideProps = async ( let results: QuestionFragment[] = []; if (initialQueryParameters.query !== "") { - results = ( - await client - .query(SearchDocument, { - input: { - ...initialQueryParameters, - limit: 1, - }, - }) - .toPromise() - ).data.result; + const response = await client + .query(SearchDocument, { + input: { + ...initialQueryParameters, + limit: 1, + }, + }) + .toPromise(); + if (response.data?.result) { + results = response.data.result; + } else { + throw new Error("GraphQL request failed"); + } } return { diff --git a/src/web/common/Layout.tsx b/src/web/common/Layout.tsx index 4d1e84f..084daac 100644 --- a/src/web/common/Layout.tsx +++ b/src/web/common/Layout.tsx @@ -2,7 +2,7 @@ import Head from "next/head"; import Link from "next/link"; import React, { ErrorInfo } from "react"; -import { Logo2 } from "../icons/index"; +import { Logo2 } from "../icons"; interface MenuItem { page: string; diff --git a/src/web/common/MultiSelectPlatform.tsx b/src/web/common/MultiSelectPlatform.tsx index cc4436e..a458faf 100644 --- a/src/web/common/MultiSelectPlatform.tsx +++ b/src/web/common/MultiSelectPlatform.tsx @@ -87,7 +87,7 @@ export const MultiSelectPlatform: React.FC = ({ const selectValue = value.map((v) => id2option[v]).filter((v) => v); - const onSelectChange = (newValue: Option[]) => { + const onSelectChange = (newValue: readonly Option[]) => { onChange(newValue.map((o) => o.value)); }; diff --git a/src/web/display/DashboardCreator.tsx b/src/web/display/DashboardCreator.tsx index 48ddde5..7014caf 100644 --- a/src/web/display/DashboardCreator.tsx +++ b/src/web/display/DashboardCreator.tsx @@ -37,7 +37,9 @@ export const DashboardCreator: React.FC = ({ handleSubmit }) => { } } catch (error) { setActing(false); - const substituteText = `Error: ${error.message} + const substituteText = `Error: ${ + error instanceof Error ? error.message : "Unknown" + } Try something like: ${exampleInput} diff --git a/src/web/icons/Favicon.tsx b/src/web/icons/Favicon.tsx index 9bee8ac..412723f 100644 --- a/src/web/icons/Favicon.tsx +++ b/src/web/icons/Favicon.tsx @@ -1,6 +1,4 @@ -import * as React from "react"; - -function SvgFavicon(props) { +export const Favicon: React.FC> = (props) => { return ( ); -} - -export default SvgFavicon; +}; diff --git a/src/web/icons/Logo.tsx b/src/web/icons/Logo.tsx index 0633163..9fe8fa9 100644 --- a/src/web/icons/Logo.tsx +++ b/src/web/icons/Logo.tsx @@ -1,6 +1,4 @@ -import * as React from "react"; - -function SvgLogo(props) { +export const Logo: React.FC> = (props) => { return ( ); -} - -export default SvgLogo; +}; diff --git a/src/web/icons/Logo2.tsx b/src/web/icons/Logo2.tsx index 43709c6..dd2ce86 100644 --- a/src/web/icons/Logo2.tsx +++ b/src/web/icons/Logo2.tsx @@ -1,6 +1,4 @@ -import * as React from "react"; - -function SvgLogo2(props) { +export const Logo2: React.FC> = (props) => { return ( ); -} - -export default SvgLogo2; +}; diff --git a/src/web/icons/index.ts b/src/web/icons/index.ts index 7fb9e03..843cc09 100644 --- a/src/web/icons/index.ts +++ b/src/web/icons/index.ts @@ -1,3 +1,3 @@ -export { default as Favicon } from "./Favicon"; -export { default as Logo } from "./Logo"; -export { default as Logo2 } from "./Logo2"; +export { Favicon } from "./Favicon"; +export { Logo } from "./Logo"; +export { Logo2 } from "./Logo2"; diff --git a/src/web/questions/components/HistoryChart/InnerChart.tsx b/src/web/questions/components/HistoryChart/InnerChart.tsx index 826c46d..c25edb2 100644 --- a/src/web/questions/components/HistoryChart/InnerChart.tsx +++ b/src/web/questions/components/HistoryChart/InnerChart.tsx @@ -34,10 +34,12 @@ const getVictoryGroup = ({ ); }; -export const InnerChart: React.FC<{ +export type Props = { data: ChartData; highlight: number | undefined; -}> = ({ +}; + +export const InnerChart: React.FC = ({ data: { maxProbability, seriesList, minDate, maxDate }, highlight, }) => { @@ -120,7 +122,7 @@ export const InnerChart: React.FC<{ void }> = ({ onHighlight, }) => { const { x, y, reference, floating, strategy } = useFloating({ - // placement: "right", middleware: [shift()], }); const [showTooltip, setShowTooltip] = useState(false); - const textRef = useRef(); + const textRef = useRef(null); const onHover = () => { - if (textRef.current.scrollWidth > textRef.current.clientWidth) { + if ( + textRef.current && + textRef.current.scrollWidth > textRef.current.clientWidth + ) { setShowTooltip(true); } onHighlight(); diff --git a/src/web/questions/components/HistoryChart/index.tsx b/src/web/questions/components/HistoryChart/index.tsx index 0884adc..7d512dd 100644 --- a/src/web/questions/components/HistoryChart/index.tsx +++ b/src/web/questions/components/HistoryChart/index.tsx @@ -2,11 +2,12 @@ import dynamic from "next/dynamic"; import React, { useMemo, useState } from "react"; import { QuestionWithHistoryFragment } from "../../../fragments.generated"; +import { Props as InnerChartProps } from "./InnerChart"; // hopefully doesn't import code, just types - need to check import { InnerChartPlaceholder } from "./InnerChartPlaceholder"; import { Legend } from "./Legend"; import { buildChartData, chartColors } from "./utils"; -const InnerChart = dynamic( +const InnerChart = dynamic( () => import("./InnerChart").then((mod) => mod.InnerChart), { ssr: false, loading: () => } ); diff --git a/src/web/questions/components/HistoryChart/utils.ts b/src/web/questions/components/HistoryChart/utils.ts index 886bdfb..3072809 100644 --- a/src/web/questions/components/HistoryChart/utils.ts +++ b/src/web/questions/components/HistoryChart/utils.ts @@ -1,6 +1,8 @@ import { addDays, startOfDay, startOfToday, startOfTomorrow } from "date-fns"; import { QuestionWithHistoryFragment } from "../../../fragments.generated"; +import { isQuestionBinary } from "../../../utils"; +import { isFullQuestionOption } from "../../utils"; export type ChartSeries = { x: Date; y: number; name: string }[]; @@ -33,6 +35,7 @@ export const buildChartData = ( question: QuestionWithHistoryFragment ): ChartData => { let seriesNames = question.options + .filter(isFullQuestionOption) .sort((a, b) => { if (a.probability > b.probability) { return -1; @@ -44,9 +47,7 @@ export const buildChartData = ( .map((o) => o.name) .slice(0, MAX_LINES); - const isBinary = - (seriesNames[0] === "Yes" && seriesNames[1] === "No") || - (seriesNames[0] === "No" && seriesNames[1] === "Yes"); + const isBinary = isQuestionBinary(question); if (isBinary) { seriesNames = ["Yes"]; } @@ -69,6 +70,9 @@ export const buildChartData = ( const date = new Date(item.timestamp * 1000); for (const option of item.options) { + if (option.name == null || option.probability == null) { + continue; + } const idx = nameToIndex[option.name]; if (idx === undefined) { continue; diff --git a/src/web/questions/components/IndicatorsTable.tsx b/src/web/questions/components/IndicatorsTable.tsx index 6fce458..81099a0 100644 --- a/src/web/questions/components/IndicatorsTable.tsx +++ b/src/web/questions/components/IndicatorsTable.tsx @@ -45,18 +45,18 @@ export const IndicatorsTable: React.FC = ({ question }) => ( ) : null} {Object.keys(question.qualityIndicators) .filter( - (indicator) => - question.qualityIndicators[indicator] != null && - !!qualityIndicatorLabels[indicator] + (indicator): indicator is UsedIndicatorName => + (question.qualityIndicators as any)[indicator] != null && + indicator in qualityIndicatorLabels ) - .map((indicator: UsedIndicatorName) => { + .map((indicator) => { return ( {formatIndicatorValue( - question.qualityIndicators[indicator], + Number(question.qualityIndicators[indicator]), // must be non-null due to former check indicator, question.platform.id )} diff --git a/src/web/questions/components/QuestionCard/QuestionFooter.tsx b/src/web/questions/components/QuestionCard/QuestionFooter.tsx index 1b5a233..77d3168 100644 --- a/src/web/questions/components/QuestionCard/QuestionFooter.tsx +++ b/src/web/questions/components/QuestionCard/QuestionFooter.tsx @@ -30,13 +30,17 @@ export const qualityIndicatorLabels: { [k in UsedIndicatorName]: string } = { openInterest: "Interest", }; -const formatNumber = (num) => { - if (Number(num) < 1000) { - return Number(num).toFixed(0); +const isUsedIndicatorName = (name: string): name is UsedIndicatorName => { + return name in qualityIndicatorLabels; +}; + +const formatNumber = (num: number) => { + if (num < 1000) { + return num.toFixed(0); } else if (num < 10000) { - return (Number(num) / 1000).toFixed(1) + "k"; + return (num / 1000).toFixed(1) + "k"; } else { - return (Number(num) / 1000).toFixed(0) + "k"; + return (num / 1000).toFixed(0) + "k"; } }; @@ -100,7 +104,7 @@ const FirstQualityIndicator: React.FC<{ }; export const formatIndicatorValue = ( - value: any, + value: number, indicator: UsedIndicatorName, platform: string ): string => { @@ -119,21 +123,26 @@ const QualityIndicatorsList: React.FC<{ return (
- {Object.entries(question.qualityIndicators).map((entry, i) => { - const indicatorLabel = qualityIndicatorLabels[entry[0]]; - if (!indicatorLabel || entry[1] === null) return; - const indicator = entry[0] as UsedIndicatorName; // guaranteed by the previous line - const value = entry[1]; + {Object.entries(question.qualityIndicators).map( + ([indicator, value], i) => { + if (!isUsedIndicatorName(indicator)) return; + const indicatorLabel = qualityIndicatorLabels[indicator]; + if (!indicatorLabel || value === null) return; - return ( -
- {indicatorLabel}:  - - {formatIndicatorValue(value, indicator, question.platform.id)} - -
- ); - })} + return ( +
+ {indicatorLabel}:  + + {formatIndicatorValue( + Number(value), + indicator, + question.platform.id + )} + +
+ ); + } + )}
); }; diff --git a/src/web/questions/components/QuestionCard/index.tsx b/src/web/questions/components/QuestionCard/index.tsx index f1d9eef..0cba25b 100644 --- a/src/web/questions/components/QuestionCard/index.tsx +++ b/src/web/questions/components/QuestionCard/index.tsx @@ -13,62 +13,25 @@ const truncateText = (length: number, text: string): string => { if (!text) { return ""; } - if (!!text && text.length <= length) { + if (text.length <= length) { return text; } const breakpoints = " .!?"; - let lastLetter = null; - let lastIndex = null; + let lastLetter: string | undefined = undefined; + let lastIndex: number | undefined = undefined; for (let index = length; index > 0; index--) { - let letter = text[index]; + const letter = text[index]; if (breakpoints.includes(letter)) { lastLetter = letter; lastIndex = index; break; } } - let truncatedText = !!text.slice - ? text.slice(0, lastIndex) + (lastLetter != "." ? "..." : "..") - : ""; + let truncatedText = + text.slice(0, lastIndex) + (lastLetter != "." ? "..." : ".."); return truncatedText; }; -// replaceAll polyfill -function escapeRegExp(string) { - return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string -} - -function replaceAll( - originalString: string, - pattern: string | RegExp, - substitute -) { - return originalString.replace( - new RegExp(escapeRegExp(pattern), "g"), - substitute - ); -} - -if (!String.prototype.replaceAll) { - String.prototype.replaceAll = function ( - pattern: string | RegExp, - substitute - ) { - let originalString = this; - - // If a regex pattern - if ( - Object.prototype.toString.call(pattern).toLowerCase() === - "[object regexp]" - ) { - return originalString.replace(pattern, substitute); - } - - // If a string - return replaceAll(originalString, pattern, substitute); - }; -} - // Auxiliary components const DisplayMarkdown: React.FC<{ description: string }> = ({ @@ -153,14 +116,14 @@ export const QuestionCard: React.FC = ({
{isBinary ? (
- +
) : (
- +
diff --git a/src/web/questions/components/QuestionIndicators.tsx b/src/web/questions/components/QuestionIndicators.tsx deleted file mode 100644 index ecd37da..0000000 --- a/src/web/questions/components/QuestionIndicators.tsx +++ /dev/null @@ -1,227 +0,0 @@ -import { QuestionFragment } from "../../fragments.generated"; - -type QualityIndicator = QuestionFragment["qualityIndicators"]; -type IndicatorName = keyof QualityIndicator; - -// this duplication can probably be simplified with typescript magic, but this is good enough for now -type UsedIndicatorName = - | "volume" - | "numForecasters" - | "spread" - | "sharesVolume" - | "liquidity" - | "tradeVolume" - | "openInterest"; - -const qualityIndicatorLabels: { [k in UsedIndicatorName]: string } = { - // numForecasts: null, - // stars: null, - // yesBid: "Yes bid", - // yesAsk: "Yes ask", - volume: "Volume", - numForecasters: "Forecasters", - spread: "Spread", - sharesVolume: "Shares vol.", - liquidity: "Liquidity", - tradeVolume: "Volume", - openInterest: "Interest", -}; - -const formatNumber = (num) => { - if (Number(num) < 1000) { - return Number(num).toFixed(0); - } else if (num < 10000) { - return (Number(num) / 1000).toFixed(1) + "k"; - } else { - return (Number(num) / 1000).toFixed(0) + "k"; - } -}; - -/* Display functions*/ - -const getPercentageSymbolIfNeeded = ({ - indicator, - platform, -}: { - indicator: UsedIndicatorName; - platform: string; -}) => { - let indicatorsWhichNeedPercentageSymbol: IndicatorName[] = ["spread"]; - if (indicatorsWhichNeedPercentageSymbol.includes(indicator)) { - return "%"; - } else { - return ""; - } -}; - -const getCurrencySymbolIfNeeded = ({ - indicator, - platform, -}: { - indicator: UsedIndicatorName; - platform: string; -}) => { - const indicatorsWhichNeedCurrencySymbol: IndicatorName[] = [ - "volume", - "tradeVolume", - "openInterest", - "liquidity", - ]; - let dollarPlatforms = ["predictit", "kalshi", "polymarket"]; - if (indicatorsWhichNeedCurrencySymbol.includes(indicator)) { - if (dollarPlatforms.includes(platform)) { - return "$"; - } else { - return "£"; - } - } else { - return ""; - } -}; - -const FirstQualityIndicator: React.FC<{ - question: QuestionFragment; -}> = ({ question }) => { - if (question.qualityIndicators.numForecasts) { - return ( -
- Forecasts:  - - {Number(question.qualityIndicators.numForecasts).toFixed(0)} - -
- ); - } else { - return null; - } -}; - -const QualityIndicatorsList: React.FC<{ - question: QuestionFragment; -}> = ({ question }) => { - return ( -
- - {Object.entries(question.qualityIndicators).map((entry, i) => { - const indicatorLabel = qualityIndicatorLabels[entry[0]]; - if (!indicatorLabel || entry[1] === null) return; - const indicator = entry[0] as UsedIndicatorName; // guaranteed by the previous line - const value = entry[1]; - - return ( -
- {indicatorLabel}:  - - {`${getCurrencySymbolIfNeeded({ - indicator, - platform: question.platform.id, - })}${formatNumber(value)}${getPercentageSymbolIfNeeded({ - indicator, - platform: question.platform.id, - })}`} - -
- ); - })} -
- ); -}; - -// Database-like functions -export function getstars(numstars: number) { - let stars = "★★☆☆☆"; - switch (numstars) { - case 0: - stars = "☆☆☆☆☆"; - break; - case 1: - stars = "★☆☆☆☆"; - break; - case 2: - stars = "★★☆☆☆"; - break; - case 3: - stars = "★★★☆☆"; - break; - case 4: - stars = "★★★★☆"; - break; - case 5: - stars = "★★★★★"; - break; - default: - stars = "★★☆☆☆"; - } - return stars; -} - -function getStarsColor(numstars: number) { - let color = "text-yellow-400"; - switch (numstars) { - case 0: - color = "text-red-400"; - break; - case 1: - color = "text-red-400"; - break; - case 2: - color = "text-orange-400"; - break; - case 3: - color = "text-yellow-400"; - break; - case 4: - color = "text-green-400"; - break; - case 5: - color = "text-blue-400"; - break; - default: - color = "text-yellow-400"; - } - return color; -} - -interface Props { - question: QuestionFragment; - expandFooterToFullWidth: boolean; -} - -export const QuestionFooter: React.FC = ({ - question, - expandFooterToFullWidth, -}) => { - return ( -
-
- {getstars(question.qualityIndicators.stars)} -
-
- {question.platform.label - .replace("Good Judgment Open", "GJOpen") - .replace(/ /g, "\u00a0")} -
-
- -
-
- ); -}; diff --git a/src/web/questions/components/QuestionOptions.tsx b/src/web/questions/components/QuestionOptions.tsx index 833e121..c227f10 100644 --- a/src/web/questions/components/QuestionOptions.tsx +++ b/src/web/questions/components/QuestionOptions.tsx @@ -1,7 +1,6 @@ import { QuestionFragment } from "../../fragments.generated"; -import { formatProbability } from "../utils"; - -type Option = QuestionFragment["options"][0]; +import { isQuestionBinary } from "../../utils"; +import { formatProbability, FullQuestionOption, isFullQuestionOption } from "../utils"; const textColor = (probability: number) => { if (probability < 0.03) { @@ -89,7 +88,7 @@ const chooseColor = (probability: number) => { } }; -const OptionRow: React.FC<{ option: Option }> = ({ option }) => { +const OptionRow: React.FC<{ option: FullQuestionOption }> = ({ option }) => { return (
= ({ option }) => { ); }; -export const QuestionOptions: React.FC<{ options: Option[] }> = ({ - options, +export const QuestionOptions: React.FC<{ question: QuestionFragment }> = ({ + question, }) => { - const isBinary = - options.length === 2 && - (options[0].name === "Yes" || options[0].name === "No"); + const isBinary = isQuestionBinary(question); if (isBinary) { - const yesOption = options.find((o) => o.name === "Yes"); + const yesOption = question.options.find((o) => o.name === "Yes"); + if (!yesOption) { + return null; // shouldn't happen + } + if (!isFullQuestionOption(yesOption)) { + return null; // missing data + } return (
= ({
); } else { - const optionsSorted = options.sort((a, b) => b.probability - a.probability); - const optionsMax5 = !!optionsSorted.slice ? optionsSorted.slice(0, 5) : []; // display max 5 options. + const optionsSorted = question.options + .filter(isFullQuestionOption) + .sort((a, b) => b.probability - a.probability); + + const optionsMax5 = optionsSorted.slice(0, 5); // display max 5 options. return (
diff --git a/src/web/questions/utils.ts b/src/web/questions/utils.ts index f7f4f1c..36fa46b 100644 --- a/src/web/questions/utils.ts +++ b/src/web/questions/utils.ts @@ -8,3 +8,20 @@ export const formatProbability = (probability: number) => { : percentage.toFixed(0) + "%"; return percentageCapped; }; + +import { QuestionFragment } from "../fragments.generated"; + +export type QuestionOption = QuestionFragment["options"][0]; +export type FullQuestionOption = Exclude< + QuestionOption, + "name" | "probability" +> & { + name: NonNullable; + probability: NonNullable; +}; + +export const isFullQuestionOption = ( + option: QuestionOption +): option is FullQuestionOption => { + return option.name != null && option.probability != null; +}; diff --git a/src/web/search/components/SearchScreen.tsx b/src/web/search/components/SearchScreen.tsx index 31d09c7..56200ff 100644 --- a/src/web/search/components/SearchScreen.tsx +++ b/src/web/search/components/SearchScreen.tsx @@ -61,7 +61,7 @@ export const SearchScreen: React.FC = ({ variables: { input: { ...queryParameters, - limit: numDisplay, + limit: numDisplay + 50, }, }, pause: !isFirstRender, diff --git a/src/web/utils.ts b/src/web/utils.ts index f4c5c81..70f8945 100644 --- a/src/web/utils.ts +++ b/src/web/utils.ts @@ -1,3 +1,5 @@ +import { QuestionFragment } from "./fragments.generated"; + export const getBasePath = () => { if (process.env.NEXT_PUBLIC_VERCEL_URL) { return `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`; @@ -29,3 +31,12 @@ export const cleanText = (text: string): string => { //console.log(textString) return textString; }; + +export const isQuestionBinary = (question: QuestionFragment): boolean => { + const { options } = question; + return ( + options.length === 2 && + ((options[0].name === "Yes" && options[1].name === "No") || + (options[0].name === "No" && options[1].name === "Yes")) + ); +}; diff --git a/src/web/worker/searchWithAlgolia.ts b/src/web/worker/searchWithAlgolia.ts index 7b53b14..8f0d4ca 100644 --- a/src/web/worker/searchWithAlgolia.ts +++ b/src/web/worker/searchWithAlgolia.ts @@ -13,9 +13,9 @@ const index = client.initIndex("metaforecast"); interface SearchOpts { queryString: string; hitsPerPage?: number; - starsThreshold: number; - filterByPlatforms: string[]; - forecastsThreshold: number; + starsThreshold?: number; + filterByPlatforms?: string[]; + forecastsThreshold?: number; } const buildFilter = ({ @@ -33,7 +33,7 @@ const buildFilter = ({ ? filterByPlatforms.map((platform) => `platform:"${platform}"`).join(" OR ") : null; const numForecastsFilter = - forecastsThreshold > 0 + forecastsThreshold && forecastsThreshold > 0 ? `qualityindicators.numforecasts >= ${forecastsThreshold}` : null; const finalFilter = [starsFilter, platformsFilter, numForecastsFilter] From ac7b54189684f124441a12128ea979a3c3e5f697 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Mon, 9 May 2022 23:34:56 +0400 Subject: [PATCH 3/6] refactor: platforms don't need to pass in results --- src/backend/platforms/_example.ts | 3 +-- src/backend/platforms/fantasyscotus.ts | 1 - src/backend/platforms/foretold.ts | 3 +-- src/backend/platforms/goodjudgment.ts | 1 - src/backend/platforms/index.ts | 2 +- src/backend/platforms/infer.ts | 1 - src/backend/platforms/kalshi.ts | 1 - src/backend/platforms/manifold.ts | 1 - src/backend/platforms/metaculus.ts | 1 - src/backend/platforms/polymarket.ts | 1 - src/backend/platforms/predictit.ts | 1 - src/backend/platforms/rootclaim.ts | 1 - src/backend/platforms/smarkets.ts | 1 - src/backend/platforms/wildeford.ts | 1 - src/backend/platforms/xrisk.ts | 1 - 15 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/backend/platforms/_example.ts b/src/backend/platforms/_example.ts index 3c5c3fe..897a151 100644 --- a/src/backend/platforms/_example.ts +++ b/src/backend/platforms/_example.ts @@ -41,8 +41,7 @@ async function processPredictions(predictions) { const result: FetchedQuestion = { id, title: prediction.title, - url: `https://example.com`, - platform: platformName, + url: "https://example.com", description: prediction.description, options, qualityindicators: { diff --git a/src/backend/platforms/fantasyscotus.ts b/src/backend/platforms/fantasyscotus.ts index 2787373..4a74614 100644 --- a/src/backend/platforms/fantasyscotus.ts +++ b/src/backend/platforms/fantasyscotus.ts @@ -79,7 +79,6 @@ async function processData(data) { id: id, title: `In ${event.short_name}, the SCOTUS will affirm the lower court's decision`, url: `https://fantasyscotus.net/user-predictions${event.docket_url}`, - platform: platformName, description: `${(pAffirm * 100).toFixed(2)}% (${ predictionData.numAffirm } out of ${ diff --git a/src/backend/platforms/foretold.ts b/src/backend/platforms/foretold.ts index 9adce77..97e63fd 100644 --- a/src/backend/platforms/foretold.ts +++ b/src/backend/platforms/foretold.ts @@ -84,11 +84,10 @@ export const foretold: Platform = { }, ]; } - let result: FetchedQuestion = { + const result: FetchedQuestion = { id, title: question.name, url: `https://www.foretold.io/c/${community}/m/${question.id}`, - platform: platformName, description: "", options, qualityindicators: { diff --git a/src/backend/platforms/goodjudgment.ts b/src/backend/platforms/goodjudgment.ts index 3d191ea..5f4b0df 100644 --- a/src/backend/platforms/goodjudgment.ts +++ b/src/backend/platforms/goodjudgment.ts @@ -101,7 +101,6 @@ export const goodjudgment: Platform = { id, title, url: endpoint, - platform: platformName, description, options, qualityindicators: { diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index dcaa265..7016060 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -41,7 +41,7 @@ export interface QualityIndicators { export type FetchedQuestion = Omit< Question, - "extra" | "qualityindicators" | "timestamp" + "extra" | "qualityindicators" | "timestamp" | "platform" > & { timestamp?: Date; extra?: object; // required in DB but annoying to return empty; also this is slightly stricter than Prisma's JsonValue diff --git a/src/backend/platforms/infer.ts b/src/backend/platforms/infer.ts index d6133c6..1516a93 100644 --- a/src/backend/platforms/infer.ts +++ b/src/backend/platforms/infer.ts @@ -179,7 +179,6 @@ async function infer_inner(cookie: string) { title: title, description: moreinfo.description, url: url, - platform: platformName, options: moreinfo.options, ...moreinfo, }; diff --git a/src/backend/platforms/kalshi.ts b/src/backend/platforms/kalshi.ts index 6330298..c7e88be 100644 --- a/src/backend/platforms/kalshi.ts +++ b/src/backend/platforms/kalshi.ts @@ -40,7 +40,6 @@ async function processMarkets(markets) { id, title: market.title.replaceAll("*", ""), url: `https://kalshi.com/markets/${market.ticker_name}`, - platform: platformName, description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`, options, qualityindicators: { diff --git a/src/backend/platforms/manifold.ts b/src/backend/platforms/manifold.ts index ab09aa2..f221bc6 100644 --- a/src/backend/platforms/manifold.ts +++ b/src/backend/platforms/manifold.ts @@ -63,7 +63,6 @@ async function processPredictions(predictions) { id: id, title: prediction.question, url: prediction.url, - platform: platformName, description: prediction.description, options: options, qualityindicators: { diff --git a/src/backend/platforms/metaculus.ts b/src/backend/platforms/metaculus.ts index a567510..5d54b10 100644 --- a/src/backend/platforms/metaculus.ts +++ b/src/backend/platforms/metaculus.ts @@ -152,7 +152,6 @@ export const metaculus: Platform = { id, title: result.title, url: "https://www.metaculus.com" + result.page_url, - platform: platformName, description, options, qualityindicators: { diff --git a/src/backend/platforms/polymarket.ts b/src/backend/platforms/polymarket.ts index a91c42c..e00db69 100644 --- a/src/backend/platforms/polymarket.ts +++ b/src/backend/platforms/polymarket.ts @@ -106,7 +106,6 @@ export const polymarket: Platform = { id: id, title: marketInfo.question, url: "https://polymarket.com/market/" + marketInfo.slug, - platform: platformName, description: marketInfo.description, options: options, qualityindicators: { diff --git a/src/backend/platforms/predictit.ts b/src/backend/platforms/predictit.ts index 0233ec1..9f5fed2 100644 --- a/src/backend/platforms/predictit.ts +++ b/src/backend/platforms/predictit.ts @@ -100,7 +100,6 @@ export const predictit: Platform = { id, title: market["name"], url: market.url, - platform: platformName, description, options, qualityindicators: { diff --git a/src/backend/platforms/rootclaim.ts b/src/backend/platforms/rootclaim.ts index 1a1feff..0999e73 100644 --- a/src/backend/platforms/rootclaim.ts +++ b/src/backend/platforms/rootclaim.ts @@ -75,7 +75,6 @@ export const rootclaim: Platform = { id, title: toMarkdown(claim.question).replace("\n", ""), url, - platform: platformName, description: toMarkdown(description).replace("'", "'"), options: options, qualityindicators: { diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts index 1b95af8..19cb449 100644 --- a/src/backend/platforms/smarkets.ts +++ b/src/backend/platforms/smarkets.ts @@ -163,7 +163,6 @@ export const smarkets: Platform = { id: id, title: name, url: "https://smarkets.com/event/" + market.event_id + market.slug, - platform: platformName, description: market.description, options: options, timestamp: new Date(), diff --git a/src/backend/platforms/wildeford.ts b/src/backend/platforms/wildeford.ts index 7778b04..0b39eaa 100644 --- a/src/backend/platforms/wildeford.ts +++ b/src/backend/platforms/wildeford.ts @@ -92,7 +92,6 @@ async function processPredictions(predictions) { id, title, url: prediction["url"], - platform: platformName, description: prediction["Notes"] || "", options, timestamp: new Date(Date.parse(prediction["Prediction Date"] + "Z")), diff --git a/src/backend/platforms/xrisk.ts b/src/backend/platforms/xrisk.ts index c769ae9..768e2be 100644 --- a/src/backend/platforms/xrisk.ts +++ b/src/backend/platforms/xrisk.ts @@ -21,7 +21,6 @@ export const xrisk: Platform = { return { ...item, id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique - platform: platformName, }; }); return results; From da03fa8804133f9936a097ad1222d4b329d5ca61 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Tue, 10 May 2022 01:15:18 +0400 Subject: [PATCH 4/6] refactor: stars calculation moved to platforms --- src/backend/platforms/_example.ts | 9 +- src/backend/platforms/betfair.ts | 29 +- src/backend/platforms/fantasyscotus.ts | 5 +- src/backend/platforms/foretold.ts | 15 +- src/backend/platforms/givewellopenphil.ts | 14 +- src/backend/platforms/goodjudgment.ts | 14 +- src/backend/platforms/goodjudgmentopen.ts | 50 ++-- src/backend/platforms/guesstimate.ts | 19 +- src/backend/platforms/index.ts | 38 ++- src/backend/platforms/infer.ts | 13 +- src/backend/platforms/kalshi.ts | 44 ++- src/backend/platforms/manifold.ts | 30 +- src/backend/platforms/metaculus.ts | 17 +- src/backend/platforms/polymarket.ts | 40 ++- src/backend/platforms/predictit.ts | 11 +- src/backend/platforms/rootclaim.ts | 11 +- src/backend/platforms/smarkets.ts | 14 +- src/backend/platforms/wildeford.ts | 16 +- src/backend/platforms/xrisk.ts | 1 + src/backend/utils/getSecrets.ts | 2 +- src/backend/utils/stars.ts | 330 ---------------------- src/utils.ts | 3 + 22 files changed, 272 insertions(+), 453 deletions(-) delete mode 100644 src/backend/utils/stars.ts diff --git a/src/backend/platforms/_example.ts b/src/backend/platforms/_example.ts index 897a151..e41113c 100644 --- a/src/backend/platforms/_example.ts +++ b/src/backend/platforms/_example.ts @@ -1,7 +1,6 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -26,7 +25,7 @@ async function processPredictions(predictions) { let results = await predictions.map((prediction) => { const id = `${platformName}-${prediction.id}`; const probability = prediction.probability; - const options = [ + const options: FetchedQuestion["options"] = [ { name: "Yes", probability: probability, @@ -45,9 +44,6 @@ async function processPredictions(predictions) { description: prediction.description, options, qualityindicators: { - stars: calculateStars(platformName, { - /* some: somex, factors: factors */ - }), // other: prediction.otherx, // indicators: prediction.indicatorx, }, @@ -68,4 +64,7 @@ export const example: Platform = { let results = await processPredictions(data); // somehow needed return results; }, + calculateStars(data) { + return 2; + }, }; diff --git a/src/backend/platforms/betfair.ts b/src/backend/platforms/betfair.ts index 2fec2cf..aa950d8 100644 --- a/src/backend/platforms/betfair.ts +++ b/src/backend/platforms/betfair.ts @@ -2,7 +2,7 @@ import axios from "axios"; import https from "https"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; const platformName = "betfair"; @@ -121,17 +121,13 @@ async function processPredictions(data) { if (title.includes("of the named")) { title = prediction.marketName + ": " + title; } - const result = { + const result: FetchedQuestion = { id, title, url: `https://www.betfair.com/exchange/plus/politics/market/${prediction.marketId}`, - platform: platformName, description, options, qualityindicators: { - stars: calculateStars(platformName, { - volume: prediction.totalMatched, - }), volume: prediction.totalMatched, }, }; @@ -149,4 +145,25 @@ export const betfair: Platform = { const results = await processPredictions(data); // somehow needed return results; }, + calculateStars(data) { + const volume = data.qualityindicators.volume || 0; + let nuno = () => (volume > 10000 ? 4 : volume > 1000 ? 3 : 2); + let eli = () => (volume > 10000 ? null : null); + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + + const firstOption = data.options[0]; + + // Substract 1 star if probability is above 90% or below 10% + if ( + firstOption && + ((firstOption.probability || 0) < 0.1 || + (firstOption.probability || 0) > 0.9) + ) { + starsDecimal = starsDecimal - 1; + } + + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/fantasyscotus.ts b/src/backend/platforms/fantasyscotus.ts index 4a74614..7527599 100644 --- a/src/backend/platforms/fantasyscotus.ts +++ b/src/backend/platforms/fantasyscotus.ts @@ -1,7 +1,6 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; import { FetchedQuestion, Platform } from "./"; const platformName = "fantasyscotus"; @@ -100,7 +99,6 @@ async function processData(data) { ], qualityindicators: { numforecasts: Number(predictionData.numForecasts), - stars: calculateStars(platformName, {}), }, }; results.push(eventObject); @@ -120,4 +118,7 @@ export const fantasyscotus: Platform = { let results = await processData(rawData); return results; }, + calculateStars(data) { + return 2; + }, }; diff --git a/src/backend/platforms/foretold.ts b/src/backend/platforms/foretold.ts index 97e63fd..0673fae 100644 --- a/src/backend/platforms/foretold.ts +++ b/src/backend/platforms/foretold.ts @@ -1,7 +1,7 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -68,7 +68,8 @@ export const foretold: Platform = { questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions questions.forEach((question) => { let id = `${platformName}-${question.id}`; - let options = []; + + let options: FetchedQuestion["options"] = []; if (question.valueType == "PERCENTAGE") { let probability = question.previousAggregate.value.percentage; options = [ @@ -84,6 +85,7 @@ export const foretold: Platform = { }, ]; } + const result: FetchedQuestion = { id, title: question.name, @@ -92,7 +94,6 @@ export const foretold: Platform = { options, qualityindicators: { numforecasts: Math.floor(Number(question.measurementCount) / 2), - stars: calculateStars(platformName, {}), }, /*liquidity: liquidity.toFixed(2), tradevolume: tradevolume.toFixed(2), @@ -104,4 +105,12 @@ export const foretold: Platform = { } return results; }, + calculateStars(data) { + let nuno = () => 2; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/givewellopenphil.ts b/src/backend/platforms/givewellopenphil.ts index e8d9754..48db8ca 100644 --- a/src/backend/platforms/givewellopenphil.ts +++ b/src/backend/platforms/givewellopenphil.ts @@ -2,7 +2,7 @@ import axios from "axios"; import fs from "fs"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { Platform } from "./"; const platformName = "givewellopenphil"; @@ -53,9 +53,7 @@ async function main1() { platform: platformName, description, options: [], - qualityindicators: { - stars: calculateStars(platformName, {}), - }, + qualityindicators: {}, }; // Note: This requires some processing afterwards // console.log(result) results.push(result); @@ -84,4 +82,12 @@ export const givewellopenphil: Platform = { })); return dataWithDate; }, + calculateStars(data) { + let nuno = () => 2; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/goodjudgment.ts b/src/backend/platforms/goodjudgment.ts index 5f4b0df..ab487bf 100644 --- a/src/backend/platforms/goodjudgment.ts +++ b/src/backend/platforms/goodjudgment.ts @@ -3,8 +3,8 @@ import axios from "axios"; import { Tabletojson } from "tabletojson"; import tunnel from "tunnel"; +import { average } from "../../utils"; import { hash } from "../utils/hash"; -import { calculateStars } from "../utils/stars"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -103,9 +103,7 @@ export const goodjudgment: Platform = { url: endpoint, description, options, - qualityindicators: { - stars: calculateStars(platformName, {}), - }, + qualityindicators: {}, extra: { superforecastercommentary: analysis || "", }, @@ -120,4 +118,12 @@ export const goodjudgment: Platform = { return results; }, + calculateStars(data) { + let nuno = () => 4; + let eli = () => 4; + let misha = () => 3.5; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/goodjudgmentopen.ts b/src/backend/platforms/goodjudgmentopen.ts index a4e1087..05c7a8d 100644 --- a/src/backend/platforms/goodjudgmentopen.ts +++ b/src/backend/platforms/goodjudgmentopen.ts @@ -2,16 +2,16 @@ import axios from "axios"; import { Tabletojson } from "tabletojson"; +import { average } from "../../utils"; import { applyIfSecretExists } from "../utils/getSecrets"; -import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; -import { Platform } from "./"; +import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "goodjudgmentopen"; -let htmlEndPoint = "https://www.gjopen.com/questions?page="; -let annoyingPromptUrls = [ +const htmlEndPoint = "https://www.gjopen.com/questions?page="; +const annoyingPromptUrls = [ "https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters", "https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread", @@ -22,7 +22,7 @@ const id = () => 0; /* Support functions */ -async function fetchPage(page, cookie) { +async function fetchPage(page: number, cookie: string) { let response = await axios({ url: htmlEndPoint + page, method: "GET", @@ -35,7 +35,7 @@ async function fetchPage(page, cookie) { return response; } -async function fetchStats(questionUrl, cookie) { +async function fetchStats(questionUrl: string, cookie: string) { let response = await axios({ url: questionUrl + "/stats", method: "GET", @@ -50,7 +50,7 @@ async function fetchStats(questionUrl, cookie) { // Is binary? let isbinary = response.includes("binary?":true"); - let options = []; + let options: FetchedQuestion["options"] = []; if (isbinary) { // Crowd percentage let htmlElements = response.split("\n"); @@ -107,21 +107,12 @@ async function fetchStats(questionUrl, cookie) { .split(",")[0]; //console.log(numpredictors) - // Calculate the stars - let minProbability = Math.min(...options.map((option) => option.probability)); - let maxProbability = Math.max(...options.map((option) => option.probability)); - let result = { - description: description, - options: options, + description, + options, qualityindicators: { numforecasts: Number(numforecasts), numforecasters: Number(numforecasters), - stars: calculateStars("Good Judgment Open", { - numforecasts, - minProbability, - maxProbability, - }), }, // this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now } as any; @@ -129,7 +120,7 @@ async function fetchStats(questionUrl, cookie) { return result; } -function isSignedIn(html) { +function isSignedIn(html: string) { let isSignedInBool = !( html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") @@ -157,7 +148,7 @@ function sleep(ms: number) { /* Body */ -async function goodjudgmentopen_inner(cookie) { +async function goodjudgmentopen_inner(cookie: string) { let i = 1; let response = await fetchPage(i, cookie); @@ -243,6 +234,23 @@ export const goodjudgmentopen: Platform = { color: "#002455", async fetcher() { let cookie = process.env.GOODJUDGMENTOPENCOOKIE; - return await applyIfSecretExists(cookie, goodjudgmentopen_inner); + return (await applyIfSecretExists(cookie, goodjudgmentopen_inner)) || null; + }, + calculateStars(data) { + let minProbability = Math.min( + ...data.options.map((option) => option.probability || 0) + ); + let maxProbability = Math.max( + ...data.options.map((option) => option.probability || 0) + ); + + let nuno = () => ((data.qualityindicators.numforecasts || 0) > 100 ? 3 : 2); + let eli = () => 3; + let misha = () => + minProbability > 0.1 || maxProbability < 0.9 ? 3.1 : 2.5; + + let starsDecimal = average([nuno(), eli(), misha()]); + let starsInteger = Math.round(starsDecimal); + return starsInteger; }, }; diff --git a/src/backend/platforms/guesstimate.ts b/src/backend/platforms/guesstimate.ts index d61ed7e..b517215 100644 --- a/src/backend/platforms/guesstimate.ts +++ b/src/backend/platforms/guesstimate.ts @@ -1,12 +1,10 @@ import axios from "axios"; -import { parseISO } from "date-fns"; -/* Imports */ import { Question } from "@prisma/client"; import { AlgoliaQuestion } from "../../backend/utils/algolia"; import { prisma } from "../database/prisma"; -import { Platform } from "./"; +import { FetchedQuestion, Platform, prepareQuestion } from "./"; /* Definitions */ const searchEndpoint = @@ -14,25 +12,20 @@ const searchEndpoint = const apiEndpoint = "https://guesstimate.herokuapp.com"; -/* Body */ - const modelToQuestion = (model: any): Question => { const { description } = model; // const description = model.description // ? model.description.replace(/\n/g, " ").replace(/ /g, " ") // : ""; - const stars = description.length > 250 ? 2 : 1; - const timestamp = parseISO(model.created_at); - const q: Question = { + // const timestamp = parseISO(model.created_at); + const fq: FetchedQuestion = { id: `guesstimate-${model.id}`, title: model.name, url: `https://www.getguesstimate.com/models/${model.id}`, - timestamp, - platform: "guesstimate", + // timestamp, description, options: [], qualityindicators: { - stars, numforecasts: 1, numforecasters: 1, }, @@ -41,6 +34,7 @@ const modelToQuestion = (model: any): Question => { }, // ranking: 10 * (index + 1) - 0.5, //(model._rankingInfo - 1*index)// hack }; + const q = prepareQuestion(fq, guesstimate); return q; }; @@ -68,7 +62,7 @@ async function search(query: string): Promise { }); // filter for duplicates. Surprisingly common. - let uniqueTitles = []; + let uniqueTitles: string[] = []; let uniqueModels: AlgoliaQuestion[] = []; for (let model of mappedModels) { if (!uniqueTitles.includes(model.title) && !model.title.includes("copy")) { @@ -100,4 +94,5 @@ export const guesstimate: Platform & { color: "#223900", search, fetchQuestion, + calculateStars: (q) => (q.description.length > 250 ? 2 : 1), }; diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index 7016060..d9127ed 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -41,11 +41,16 @@ export interface QualityIndicators { export type FetchedQuestion = Omit< Question, - "extra" | "qualityindicators" | "timestamp" | "platform" + "extra" | "qualityindicators" | "timestamp" | "platform" | "options" > & { timestamp?: Date; extra?: object; // required in DB but annoying to return empty; also this is slightly stricter than Prisma's JsonValue - qualityindicators: QualityIndicators; // slightly stronger type than Prisma's JsonValue + options: { + name?: string; + probability?: number; + type: "PROBABILITY"; + }[]; // stronger type than Prisma's JsonValue + qualityindicators: Omit; // slightly stronger type than Prisma's JsonValue }; // fetcher should return null if platform failed to fetch questions for some reason @@ -56,6 +61,7 @@ export interface Platform { label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates" color: string; // used on frontend fetcher?: PlatformFetcher; + calculateStars: (question: FetchedQuestion) => number; } // draft for the future callback-based streaming/chunking API: @@ -86,6 +92,22 @@ export const platforms: Platform[] = [ xrisk, ]; +export const prepareQuestion = ( + q: FetchedQuestion, + platform: Platform +): Question => { + return { + extra: {}, + timestamp: new Date(), + ...q, + platform: platform.name, + qualityindicators: { + ...q.qualityindicators, + stars: platform.calculateStars(q), + }, + }; +}; + export const processPlatform = async (platform: Platform) => { if (!platform.fetcher) { console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`); @@ -97,16 +119,6 @@ export const processPlatform = async (platform: Platform) => { return; } - const prepareQuestion = (q: FetchedQuestion): Question => { - return { - extra: {}, - timestamp: new Date(), - ...q, - platform: platform.name, - qualityindicators: q.qualityindicators as object, // fighting typescript - }; - }; - const oldQuestions = await prisma.question.findMany({ where: { platform: platform.name, @@ -123,7 +135,7 @@ export const processPlatform = async (platform: Platform) => { const updatedQuestions: Question[] = []; const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id)); - for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) { + for (const q of fetchedQuestions.map((q) => prepareQuestion(q, platform))) { if (oldIdsSet.has(q.id)) { updatedQuestions.push(q); } else { diff --git a/src/backend/platforms/infer.ts b/src/backend/platforms/infer.ts index 1516a93..b004917 100644 --- a/src/backend/platforms/infer.ts +++ b/src/backend/platforms/infer.ts @@ -1,9 +1,9 @@ /* Imports */ import axios from "axios"; +import { average } from "../../utils"; import { applyIfSecretExists } from "../utils/getSecrets"; import { measureTime } from "../utils/measureTime"; -import { calculateStars } from "../utils/stars"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; @@ -106,7 +106,6 @@ async function fetchStats(questionUrl, cookie) { numforecasts: Number(numforecasts), numforecasters: Number(numforecasters), comments_count: Number(comments_count), - stars: calculateStars(platformName, { numforecasts }), }, }; // console.log(JSON.stringify(result, null, 4)); @@ -177,9 +176,7 @@ async function infer_inner(cookie: string) { let question: FetchedQuestion = { id: id, title: title, - description: moreinfo.description, url: url, - options: moreinfo.options, ...moreinfo, }; console.log(JSON.stringify(question, null, 4)); @@ -236,4 +233,12 @@ export const infer: Platform = { let cookie = process.env.INFER_COOKIE; return await applyIfSecretExists(cookie, infer_inner); }, + calculateStars(data) { + let nuno = () => 2; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/kalshi.ts b/src/backend/platforms/kalshi.ts index c7e88be..665f0e6 100644 --- a/src/backend/platforms/kalshi.ts +++ b/src/backend/platforms/kalshi.ts @@ -1,7 +1,7 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -23,7 +23,7 @@ async function processMarkets(markets) { markets = markets.filter((market) => market.close_date > dateNow); let results = await markets.map((market) => { const probability = market.last_price / 100; - const options = [ + const options: FetchedQuestion["options"] = [ { name: "Yes", probability: probability, @@ -43,29 +43,26 @@ async function processMarkets(markets) { description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`, options, qualityindicators: { - stars: calculateStars(platformName, { - shares_volume: market.volume, - interest: market.open_interest, - }), yes_bid: market.yes_bid, yes_ask: market.yes_ask, spread: Math.abs(market.yes_bid - market.yes_ask), shares_volume: market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if. // "open_interest": market.open_interest, also in shares }, + extra: { + open_interest: market.open_interest, + }, }; return result; }); - //console.log(results.length) - // console.log(results.map(result => result.title)) - // console.log(results.map(result => result.title).length) + console.log([...new Set(results.map((result) => result.title))]); console.log( "Number of unique questions: ", [...new Set(results.map((result) => result.title))].length ); - // console.log([...new Set(results.map(result => result.title))].length) - return results; //resultsProcessed + + return results; } export const kalshi: Platform = { @@ -76,4 +73,29 @@ export const kalshi: Platform = { let markets = await fetchAllMarkets(); return await processMarkets(markets); }, + calculateStars(data) { + let nuno = () => + ((data.extra as any)?.open_interest || 0) > 500 && + data.qualityindicators.shares_volume > 10000 + ? 4 + : data.qualityindicators.shares_volume > 2000 + ? 3 + : 2; + // let eli = (data) => data.interest > 10000 ? 5 : 4 + // let misha = (data) => 4 + let starsDecimal = average([nuno()]); //, eli(data), misha(data)]) + + // Substract 1 star if probability is above 90% or below 10% + if ( + data.options instanceof Array && + data.options[0] && + ((data.options[0].probability || 0) < 0.1 || + (data.options[0].probability || 0) > 0.9) + ) { + starsDecimal = starsDecimal - 1; + } + + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/manifold.ts b/src/backend/platforms/manifold.ts index f221bc6..b735470 100644 --- a/src/backend/platforms/manifold.ts +++ b/src/backend/platforms/manifold.ts @@ -1,7 +1,7 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -25,16 +25,16 @@ async function fetchData() { function showStatistics(results: FetchedQuestion[]) { console.log(`Num unresolved markets: ${results.length}`); - let sum = (arr) => arr.reduce((tally, a) => tally + a, 0); + let sum = (arr: number[]) => arr.reduce((tally, a) => tally + a, 0); let num2StarsOrMore = results.filter( - (result) => result.qualityindicators.stars >= 2 + (result) => manifold.calculateStars(result) >= 2 ); console.log( `Manifold has ${num2StarsOrMore.length} markets with 2 stars or more` ); console.log( `Mean volume: ${ - sum(results.map((result) => result.qualityindicators.volume7Days)) / + sum(results.map((result) => result.qualityindicators.volume7Days || 0)) / results.length }; mean pool: ${ sum(results.map((result) => result.qualityindicators.pool)) / @@ -47,7 +47,7 @@ async function processPredictions(predictions) { let results: FetchedQuestion[] = await predictions.map((prediction) => { let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name let probability = prediction.probability; - let options = [ + let options: FetchedQuestion["options"] = [ { name: "Yes", probability: probability, @@ -64,13 +64,8 @@ async function processPredictions(predictions) { title: prediction.question, url: prediction.url, description: prediction.description, - options: options, + options, qualityindicators: { - stars: calculateStars(platformName, { - volume7Days: prediction.volume7Days, - volume24Hours: prediction.volume24Hours, - pool: prediction.pool, - }), createdTime: prediction.createdTime, volume7Days: prediction.volume7Days, volume24Hours: prediction.volume24Hours, @@ -99,4 +94,17 @@ export const manifold: Platform = { showStatistics(results); return results; }, + calculateStars(data) { + let nuno = () => + (data.qualityindicators.volume7Days || 0) > 250 || + ((data.qualityindicators.pool || 0) > 500 && + (data.qualityindicators.volume7Days || 0) > 100) + ? 2 + : 1; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(data), misha(data)]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/metaculus.ts b/src/backend/platforms/metaculus.ts index 5d54b10..0327ca6 100644 --- a/src/backend/platforms/metaculus.ts +++ b/src/backend/platforms/metaculus.ts @@ -1,7 +1,7 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; @@ -131,7 +131,7 @@ export const metaculus: Platform = { let description = descriptionprocessed2; let isbinary = result.possibilities.type == "binary"; - let options = []; + let options: FetchedQuestion["options"] = []; if (isbinary) { let probability = Number(result.community_prediction.full.q2); options = [ @@ -156,9 +156,6 @@ export const metaculus: Platform = { options, qualityindicators: { numforecasts: Number(result.number_of_predictions), - stars: calculateStars(platformName, { - numforecasts: result.number_of_predictions, - }), }, extra: { resolution_data: { @@ -193,4 +190,14 @@ export const metaculus: Platform = { return all_questions; }, + calculateStars(data) { + const { numforecasts } = data.qualityindicators; + let nuno = () => + (numforecasts || 0) > 300 ? 4 : (numforecasts || 0) > 100 ? 3 : 2; + let eli = () => 3; + let misha = () => 3; + let starsDecimal = average([nuno(), eli(), misha()]); + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/polymarket.ts b/src/backend/platforms/polymarket.ts index e00db69..395b60e 100644 --- a/src/backend/platforms/polymarket.ts +++ b/src/backend/platforms/polymarket.ts @@ -1,7 +1,7 @@ /* Imports */ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -96,8 +96,8 @@ export const polymarket: Platform = { let options = []; for (let outcome in moreMarketInfo.outcomeTokenPrices) { options.push({ - name: marketInfo.outcomes[outcome], - probability: moreMarketInfo.outcomeTokenPrices[outcome], + name: String(marketInfo.outcomes[outcome]), + probability: Number(moreMarketInfo.outcomeTokenPrices[outcome]), type: "PROBABILITY", }); } @@ -112,11 +112,6 @@ export const polymarket: Platform = { numforecasts: numforecasts.toFixed(0), liquidity: liquidity.toFixed(2), tradevolume: tradevolume.toFixed(2), - stars: calculateStars(platformName, { - liquidity, - option: options[0], - volume: tradevolume, - }), }, extra: { address: marketInfo.address, @@ -132,4 +127,33 @@ export const polymarket: Platform = { } return results; }, + calculateStars(data) { + // let nuno = (data) => (data.volume > 10000 ? 4 : data.volume > 1000 ? 3 : 2); + // let eli = (data) => data.liquidity > 10000 ? 5 : 4 + // let misha = (data) => 4 + + const liquidity = data.qualityindicators.liquidity || 0; + const volume = data.qualityindicators.tradevolume || 0; + + let nuno = () => + liquidity > 1000 && volume > 10000 + ? 4 + : liquidity > 500 && volume > 1000 + ? 3 + : 2; + let starsDecimal = average([nuno()]); //, eli(data), misha(data)]) + + // Substract 1 star if probability is above 90% or below 10% + if ( + data.options instanceof Array && + data.options[0] && + ((data.options[0].probability || 0) < 0.1 || + (data.options[0].probability || 0) > 0.9) + ) { + starsDecimal = starsDecimal - 1; + } + + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/predictit.ts b/src/backend/platforms/predictit.ts index 9f5fed2..550a622 100644 --- a/src/backend/platforms/predictit.ts +++ b/src/backend/platforms/predictit.ts @@ -1,6 +1,6 @@ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; @@ -103,7 +103,6 @@ export const predictit: Platform = { description, options, qualityindicators: { - stars: calculateStars(platformName, {}), shares_volume, }, }; @@ -113,4 +112,12 @@ export const predictit: Platform = { return results; }, + calculateStars(data) { + let nuno = () => 3; + let eli = () => 3.5; + let misha = () => 2.5; + let starsDecimal = average([nuno(), eli(), misha()]); + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/rootclaim.ts b/src/backend/platforms/rootclaim.ts index 0999e73..374e38e 100644 --- a/src/backend/platforms/rootclaim.ts +++ b/src/backend/platforms/rootclaim.ts @@ -1,7 +1,7 @@ import axios from "axios"; import { JSDOM } from "jsdom"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; @@ -79,11 +79,18 @@ export const rootclaim: Platform = { options: options, qualityindicators: { numforecasts: 1, - stars: calculateStars(platformName, {}), }, }; results.push(obj); } return results; }, + calculateStars(data) { + let nuno = () => 4; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno() /*, eli(data), misha(data)*/]); + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts index 19cb449..52b6059 100644 --- a/src/backend/platforms/smarkets.ts +++ b/src/backend/platforms/smarkets.ts @@ -1,6 +1,6 @@ import axios from "axios"; -import { calculateStars } from "../utils/stars"; +import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -166,9 +166,7 @@ export const smarkets: Platform = { description: market.description, options: options, timestamp: new Date(), - qualityindicators: { - stars: calculateStars(platformName, {}), - }, + qualityindicators: {}, }; VERBOSE ? console.log(result) : empty(); results.push(result); @@ -176,4 +174,12 @@ export const smarkets: Platform = { VERBOSE ? console.log(results) : empty(); return results; }, + calculateStars(data) { + let nuno = () => 2; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/wildeford.ts b/src/backend/platforms/wildeford.ts index 0b39eaa..5dee63d 100644 --- a/src/backend/platforms/wildeford.ts +++ b/src/backend/platforms/wildeford.ts @@ -1,9 +1,9 @@ /* Imports */ import { GoogleSpreadsheet } from "google-spreadsheet"; +import { average } from "../../utils"; import { applyIfSecretExists } from "../utils/getSecrets"; import { hash } from "../utils/hash"; -import { calculateStars } from "../utils/stars"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ @@ -76,7 +76,7 @@ async function processPredictions(predictions) { let title = prediction["Prediction"].replace(" [update]", ""); let id = `${platformName}-${hash(title)}`; let probability = Number(prediction["Odds"].replace("%", "")) / 100; - let options = [ + let options: FetchedQuestion["options"] = [ { name: "Yes", probability: probability, @@ -95,9 +95,7 @@ async function processPredictions(predictions) { description: prediction["Notes"] || "", options, timestamp: new Date(Date.parse(prediction["Prediction Date"] + "Z")), - qualityindicators: { - stars: calculateStars(platformName, null), - }, + qualityindicators: {}, }; return result; }); @@ -125,4 +123,12 @@ export const wildeford: Platform = { const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); }, + calculateStars(data) { + let nuno = () => 3; + let eli = () => null; + let misha = () => null; + let starsDecimal = average([nuno()]); //, eli(), misha()]) + let starsInteger = Math.round(starsDecimal); + return starsInteger; + }, }; diff --git a/src/backend/platforms/xrisk.ts b/src/backend/platforms/xrisk.ts index 768e2be..d5ab360 100644 --- a/src/backend/platforms/xrisk.ts +++ b/src/backend/platforms/xrisk.ts @@ -25,4 +25,5 @@ export const xrisk: Platform = { }); return results; }, + calculateStars: () => 2, }; diff --git a/src/backend/utils/getSecrets.ts b/src/backend/utils/getSecrets.ts index 2a73260..37c6d06 100644 --- a/src/backend/utils/getSecrets.ts +++ b/src/backend/utils/getSecrets.ts @@ -1,5 +1,5 @@ export async function applyIfSecretExists( - cookie: string, + cookie: string | undefined, fun: (cookie: string) => T ) { if (cookie) { diff --git a/src/backend/utils/stars.ts b/src/backend/utils/stars.ts deleted file mode 100644 index 5150394..0000000 --- a/src/backend/utils/stars.ts +++ /dev/null @@ -1,330 +0,0 @@ -let average = (array: number[]) => - array.reduce((a, b) => a + b, 0) / array.length; - -function calculateStarsAstralCodexTen(data) { - let nuno = (data) => 3; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsBetfair(data) { - let nuno = (data) => (data.volume > 10000 ? 4 : data.volume > 1000 ? 3 : 2); - let eli = (data) => (data.volume > 10000 ? null : null); - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - // Substract 1 star if probability is above 90% or below 10% - if ( - data.option && - (data.option.probability < 0.1 || data.option.probability > 0.9) - ) { - starsDecimal = starsDecimal - 1; - } - - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsCoupCast(data) { - let nuno = (data) => 3; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsCSETForetell(data) { - let nuno = (data) => (data.numforecasts > 100 ? 3 : 2); - let eli = (data) => 3; - let misha = (data) => 2; - let starsDecimal = average([nuno(data), eli(data), misha(data)]); - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsElicit(data) { - let nuno = (data) => 1; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsEstimize(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsForetold(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsGiveWellOpenPhil(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsGoodJudgment(data) { - let nuno = (data) => 4; - let eli = (data) => 4; - let misha = (data) => 3.5; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsGoodJudgmentOpen(data) { - let nuno = (data) => (data.numforecasts > 100 ? 3 : 2); - let eli = (data) => 3; - let misha = (data) => - data.minProbability > 0.1 || data.maxProbability < 0.9 ? 3.1 : 2.5; - let starsDecimal = average([nuno(data), eli(data), misha(data)]); - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsHypermind(data) { - let nuno = (data) => 3; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsInfer(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsKalshi(data) { - let nuno = (data) => - data.interest > 500 && data.shares_volume > 10000 - ? 4 - : data.shares_volume > 2000 - ? 3 - : 2; - // let eli = (data) => data.interest > 10000 ? 5 : 4 - // let misha = (data) => 4 - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - // Substract 1 star if probability is above 90% or below 10% - if ( - data.option && - (data.option.probability < 0.1 || data.option.probability > 0.9) - ) { - starsDecimal = starsDecimal - 1; - } - - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsLadbrokes(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsManifold(data) { - let nuno = (data) => - data.volume7Days > 250 || (data.pool > 500 && data.volume7Days > 100) - ? 2 - : 1; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - // console.log(data); - // console.log(starsInteger); - return starsInteger; -} - -function calculateStarsMetaculus(data) { - let nuno = (data) => - data.numforecasts > 300 ? 4 : data.numforecasts > 100 ? 3 : 2; - let eli = (data) => 3; - let misha = (data) => 3; - let starsDecimal = average([nuno(data), eli(data), misha(data)]); - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsOmen(data) { - let nuno = (data) => 1; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsPolymarket(data) { - // let nuno = (data) => (data.volume > 10000 ? 4 : data.volume > 1000 ? 3 : 2); - // let eli = (data) => data.liquidity > 10000 ? 5 : 4 - // let misha = (data) => 4 - - let nuno = (data) => - data.liquidity > 1000 && data.volume > 10000 - ? 4 - : data.liquidity > 500 && data.volume > 1000 - ? 3 - : 2; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - // Substract 1 star if probability is above 90% or below 10% - if ( - data.option && - (data.option.probability < 0.1 || data.option.probability > 0.9) - ) { - starsDecimal = starsDecimal - 1; - } - - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsPredictIt(data) { - let nuno = (data) => 3; - let eli = (data) => 3.5; - let misha = (data) => 2.5; - let starsDecimal = average([nuno(data), eli(data), misha(data)]); - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsRootclaim(data) { - let nuno = (data) => 4; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data) /*, eli(data), misha(data)*/]); - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsSmarkets(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsWildeford(data) { - let nuno = (data) => 3; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -function calculateStarsWilliamHill(data) { - let nuno = (data) => 2; - let eli = (data) => null; - let misha = (data) => null; - let starsDecimal = average([nuno(data)]); //, eli(data), misha(data)]) - let starsInteger = Math.round(starsDecimal); - return starsInteger; -} - -export function calculateStars(platform: string, data) { - let stars = 2; - switch (platform) { - case "betfair": - stars = calculateStarsBetfair(data); - break; - case "infer": - stars = calculateStarsInfer(data); - break; - case "foretold": - stars = calculateStarsForetold(data); - break; - case "givewellopenphil": - stars = calculateStarsGiveWellOpenPhil(data); - break; - case "goodjudgment": - stars = calculateStarsGoodJudgment(data); - break; - case "goodjudgmentopen": - stars = calculateStarsGoodJudgmentOpen(data); - break; - case "kalshi": - stars = calculateStarsKalshi(data); - break; - case "manifold": - stars = calculateStarsManifold(data); - break; - case "metaculus": - stars = calculateStarsMetaculus(data); - break; - case "polymarket": - stars = calculateStarsPolymarket(data); - break; - case "predictit": - stars = calculateStarsPredictIt(data); - break; - case "rootclaim": - stars = calculateStarsRootclaim(data); - break; - case "smarkets": - stars = calculateStarsSmarkets(data); - break; - case "wildeford": - stars = calculateStarsWildeford(data); - break; - - // deprecated - case "AstralCodexTen": - stars = calculateStarsAstralCodexTen(data); - break; - case "CoupCast": - stars = calculateStarsCoupCast(data); - break; - case "CSET-foretell": - stars = calculateStarsCSETForetell(data); - break; - case "Elicit": - stars = calculateStarsElicit(data); - break; - case "Estimize": - stars = calculateStarsEstimize(data); - break; - case "Hypermind": - stars = calculateStarsHypermind(data); - break; - case "Ladbrokes": - stars = calculateStarsLadbrokes(data); - break; - case "Omen": - stars = calculateStarsOmen(data); - break; - case "WilliamHill": - stars = calculateStarsWilliamHill(data); - break; - default: - stars = 2; - } - return stars; -} diff --git a/src/utils.ts b/src/utils.ts index 649b2a7..5b42717 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -6,3 +6,6 @@ export const shuffleArray = (array: T[]): T[] => { } return array; }; + +export const average = (array: number[]) => + array.reduce((a, b) => a + b, 0) / array.length; From f37a49e398711386fe9e120454aedc2bdd2122c3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Wed, 11 May 2022 01:45:02 +0400 Subject: [PATCH 5/6] refactor: strict typescript also: - save history simultaneously with question data - update squiggle - minor refactorings --- package-lock.json | 734 ++++-------------- package.json | 4 +- src/backend/flow/doEverything.ts | 9 +- src/backend/flow/history/updateHistory.ts | 11 - src/backend/flow/jobs.ts | 17 +- .../flow/rebuildNetliftySiteWithNewData.ts | 15 - src/backend/index.ts | 15 +- src/backend/platforms/_example.ts | 2 +- src/backend/platforms/betfair.ts | 27 +- src/backend/platforms/fantasyscotus.ts | 6 +- src/backend/platforms/foretold.ts | 20 +- src/backend/platforms/goodjudgment.ts | 48 +- src/backend/platforms/goodjudgmentopen.ts | 21 +- src/backend/platforms/guesstimate.ts | 7 +- src/backend/platforms/index.ts | 33 +- src/backend/platforms/infer.ts | 49 +- src/backend/platforms/kalshi.ts | 7 +- src/backend/platforms/manifold.ts | 8 +- src/backend/platforms/metaculus.ts | 33 +- src/backend/platforms/polymarket.ts | 14 +- src/backend/platforms/predictit.ts | 33 +- src/backend/platforms/rootclaim.ts | 4 +- src/backend/platforms/smarkets.ts | 161 ++-- src/backend/platforms/wildeford.ts | 26 +- src/backend/platforms/xrisk.ts | 4 +- src/backend/utils/algolia.ts | 2 +- .../pullForecastsToCSVForRating.ts | 51 -- .../pullMetaculusForecastsToCSVForRating.ts | 48 -- .../misc/process-forecasts-into-elicit.ts | 2 +- .../utils/misc/process-forecasts-template.ts | 2 +- src/backend/utils/sleep.ts | 3 + src/common/types.ts | 22 + src/pages/api/squiggle.ts | 5 +- .../components/HistoryChart/utils.ts | 2 +- .../questions/components/QuestionOptions.tsx | 3 +- src/web/questions/utils.ts | 17 - tsconfig.json | 2 +- 37 files changed, 453 insertions(+), 1014 deletions(-) delete mode 100644 src/backend/flow/history/updateHistory.ts delete mode 100644 src/backend/flow/rebuildNetliftySiteWithNewData.ts delete mode 100644 src/backend/utils/evaluations/pullForecastsToCSVForRating.ts delete mode 100644 src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts create mode 100644 src/backend/utils/sleep.ts create mode 100644 src/common/types.ts diff --git a/package-lock.json b/package-lock.json index 4560da0..b0f9dcf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,15 +14,18 @@ "@pothos/plugin-prisma": "^3.4.0", "@pothos/plugin-relay": "^3.10.0", "@prisma/client": "^3.11.1", + "@quri/squiggle-lang": "^0.2.8", "@tailwindcss/forms": "^0.4.0", "@tailwindcss/typography": "^0.5.1", "@types/chroma-js": "^2.1.3", "@types/dom-to-image": "^2.6.4", + "@types/google-spreadsheet": "^3.2.1", "@types/jsdom": "^16.2.14", "@types/nprogress": "^0.2.0", "@types/react": "^17.0.39", "@types/react-copy-to-clipboard": "^5.0.2", "@types/textversionjs": "^1.1.1", + "@types/tunnel": "^0.0.3", "airtable": "^0.11.1", "algoliasearch": "^4.10.3", "autoprefixer": "^10.1.0", @@ -69,7 +72,6 @@ "react-safe": "^1.3.0", "react-select": "^5.2.2", "remark-gfm": "^3.0.1", - "squiggle-experimental": "^0.1.9", "tabletojson": "^2.0.4", "tailwindcss": "^3.0.22", "textversionjs": "^1.1.3", @@ -1044,10 +1046,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.17.7", - "resolved": "https://registry.npmjs.org/@babel%2fruntime/-/runtime-7.17.7.tgz", - "integrity": "sha512-L6rvG9GDxaLgFjg41K+5Yv9OMrU98sWe+Ykmc6FDJW/+vYZMhdOMKkISgzptMaERHvS2Y2lw9MDRm2gHhlQQoA==", - "license": "MIT", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", "dependencies": { "regenerator-runtime": "^0.13.4" }, @@ -1496,12 +1497,6 @@ "react-dom": ">=16.8.0" } }, - "node_modules/@glennsl/bs-json": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@glennsl%2fbs-json/-/bs-json-5.0.4.tgz", - "integrity": "sha512-Th9DetZjRlMZrb74kgGJ44oWcoFyOTE884WlSuXft0Cd+J09vHRxiB7eVyK7Gthb4cSevsBBJDHYAbGGL25wPw==", - "license": "(LGPL-3.0 OR MPL-2.0)" - }, "node_modules/@graphql-codegen/add": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@graphql-codegen/add/-/add-3.1.1.tgz", @@ -2706,36 +2701,22 @@ "cross-spawn": "7.0.3" } }, + "node_modules/@quri/squiggle-lang": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@quri/squiggle-lang/-/squiggle-lang-0.2.8.tgz", + "integrity": "sha512-uK+oXE0chYtEW3la0qIxg6ueXRm34jvkL930RWhazVWC0S4aU2lzPJQ9Oz1h+8qIZuQRs89ZqfeO5r5IcjUTNQ==", + "dependencies": { + "jstat": "^1.9.5", + "mathjs": "^10.5.0", + "pdfast": "^0.2.0", + "rescript": "^9.1.4" + } + }, "node_modules/@repeaterjs/repeater": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@repeaterjs/repeater/-/repeater-3.0.4.tgz", "integrity": "sha512-AW8PKd6iX3vAZ0vA43nOUOnbq/X5ihgU+mSXXqunMkeQADGiqw/PY0JNeYtD5sr0PAy51YPgAPbDoeapv9r8WA==" }, - "node_modules/@rescript/react": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@rescript%2freact/-/react-0.10.3.tgz", - "integrity": "sha512-Lf9rzrR3bQPKJjOK3PBRa/B3xrJ7CqQ1HYr9VHPVxJidarIJJFZBhj0Dg1uZURX+Wg/xiP0PHFxXmdj2bK8Vxw==", - "license": "MIT", - "peer": true, - "peerDependencies": { - "react": ">=16.8.1", - "react-dom": ">=16.8.1" - } - }, - "node_modules/@rescriptbr/reform": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/@rescriptbr%2freform/-/reform-11.0.2.tgz", - "integrity": "sha512-4C4iZQ5oQKhG1hLc0fJM2Dn47zQYegDGV7Qgp18nQX/3nVNg4JrlP3Fx9eD8NvqUmd3V5ew710Y+TU5FNS8n6A==", - "license": "MIT", - "dependencies": { - "reschema": "^2.0.3", - "rescript-react-update": "^3.0.1" - }, - "peerDependencies": { - "@rescript/react": "^0.10.1", - "rescript": "^9.1.1" - } - }, "node_modules/@samverschueren/stream-to-observable": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", @@ -3198,6 +3179,11 @@ "resolved": "https://registry.npmjs.org/@types/dom-to-image/-/dom-to-image-2.6.4.tgz", "integrity": "sha512-UddUdGF1qulrSDulkz3K2Ypq527MR6ixlgAzqLbxSiQ0icx0XDlIV+h4+edmjq/1dqn0KgN0xGSe1kI9t+vGuw==" }, + "node_modules/@types/google-spreadsheet": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/google-spreadsheet/-/google-spreadsheet-3.2.1.tgz", + "integrity": "sha512-k9fMPyRJuWFoRiD4ZU2xTJTBykeXIh+ittw59UHGvNVkyu0b82DorkEJe36LBlaLkqD2qPBAunVz6JZRMFvWIA==" + }, "node_modules/@types/hast": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/@types%2fhast/-/hast-2.3.4.tgz", @@ -3370,6 +3356,14 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==" }, + "node_modules/@types/tunnel": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", + "integrity": "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/unist": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types%2funist/-/unist-2.0.6.tgz", @@ -3555,6 +3549,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -3744,75 +3739,6 @@ "follow-redirects": "^1.14.7" } }, - "node_modules/babel-code-frame": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", - "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", - "license": "MIT", - "dependencies": { - "chalk": "^1.1.3", - "esutils": "^2.0.2", - "js-tokens": "^3.0.2" - } - }, - "node_modules/babel-code-frame/node_modules/ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/babel-code-frame/node_modules/chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "license": "MIT", - "dependencies": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/babel-code-frame/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/babel-code-frame/node_modules/js-tokens": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", - "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=", - "license": "MIT" - }, - "node_modules/babel-code-frame/node_modules/supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/babel-messages": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-messages/-/babel-messages-6.23.0.tgz", - "integrity": "sha1-8830cDhYA1sqKVHG7F7fbGLyYw4=", - "license": "MIT", - "dependencies": { - "babel-runtime": "^6.22.0" - } - }, "node_modules/babel-plugin-dynamic-import-node": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", @@ -3855,28 +3781,6 @@ "integrity": "sha512-Xj9XuRuz3nTSbaTXWv3itLOcxyF4oPD8douBBmj7U9BBC6nEBYfyOJYQMf/8PJAFotC62UY5dFfIGEPr7WswzQ==", "dev": true }, - "node_modules/babel-plugin-transform-es2015-modules-commonjs": { - "version": "6.26.2", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.26.2.tgz", - "integrity": "sha512-CV9ROOHEdrjcwhIaJNBGMBCodN+1cfkwtM1SbUHmvyy35KGT7fohbpOxkE2uLz1o6odKK2Ck/tz47z+VqQfi9Q==", - "license": "MIT", - "dependencies": { - "babel-plugin-transform-strict-mode": "^6.24.1", - "babel-runtime": "^6.26.0", - "babel-template": "^6.26.0", - "babel-types": "^6.26.0" - } - }, - "node_modules/babel-plugin-transform-strict-mode": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz", - "integrity": "sha1-1fr3qleKZbvlkc9e2uBKDGcCB1g=", - "license": "MIT", - "dependencies": { - "babel-runtime": "^6.22.0", - "babel-types": "^6.24.1" - } - }, "node_modules/babel-preset-fbjs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/babel-preset-fbjs/-/babel-preset-fbjs-3.4.0.tgz", @@ -3915,114 +3819,6 @@ "@babel/core": "^7.0.0" } }, - "node_modules/babel-runtime": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", - "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", - "license": "MIT", - "dependencies": { - "core-js": "^2.4.0", - "regenerator-runtime": "^0.11.0" - } - }, - "node_modules/babel-runtime/node_modules/core-js": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", - "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", - "deprecated": "core-js@<3 is no longer maintained and not recommended for usage due to the number of issues. Please, upgrade your dependencies to the actual version of core-js@3.", - "hasInstallScript": true, - "license": "MIT" - }, - "node_modules/babel-runtime/node_modules/regenerator-runtime": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==", - "license": "MIT" - }, - "node_modules/babel-template": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-template/-/babel-template-6.26.0.tgz", - "integrity": "sha1-3gPi0WOWsGn0bdn/+FIfsaDjXgI=", - "license": "MIT", - "dependencies": { - "babel-runtime": "^6.26.0", - "babel-traverse": "^6.26.0", - "babel-types": "^6.26.0", - "babylon": "^6.18.0", - "lodash": "^4.17.4" - } - }, - "node_modules/babel-traverse": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-traverse/-/babel-traverse-6.26.0.tgz", - "integrity": "sha1-RqnL1+3MYsjlwGTi0tjQ9ANXZu4=", - "license": "MIT", - "dependencies": { - "babel-code-frame": "^6.26.0", - "babel-messages": "^6.23.0", - "babel-runtime": "^6.26.0", - "babel-types": "^6.26.0", - "babylon": "^6.18.0", - "debug": "^2.6.8", - "globals": "^9.18.0", - "invariant": "^2.2.2", - "lodash": "^4.17.4" - } - }, - "node_modules/babel-traverse/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/babel-traverse/node_modules/globals": { - "version": "9.18.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-9.18.0.tgz", - "integrity": "sha512-S0nG3CLEQiY/ILxqtztTWH/3iRRdyBLw6KMDxnKMchrtbj2OFmehVh0WUCfW3DUrIgx/qFrJPICrq4Z4sTR9UQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/babel-traverse/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "license": "MIT" - }, - "node_modules/babel-types": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz", - "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=", - "license": "MIT", - "dependencies": { - "babel-runtime": "^6.26.0", - "esutils": "^2.0.2", - "lodash": "^4.17.4", - "to-fast-properties": "^1.0.3" - } - }, - "node_modules/babel-types/node_modules/to-fast-properties": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", - "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/babylon": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", - "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==", - "license": "MIT", - "bin": { - "babylon": "bin/babylon.js" - } - }, "node_modules/backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", @@ -4793,12 +4589,15 @@ } }, "node_modules/complex.js": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.0.11.tgz", - "integrity": "sha512-6IArJLApNtdg1P1dFtn3dnyzoZBEF0MwMnrfF1exSBRpZYoy4yieMkpZhQDC0uwctw48vii0CFVyHfpgZ/DfGw==", - "license": "MIT OR GPL-2.0", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.1.1.tgz", + "integrity": "sha512-8njCHOTtFFLtegk6zQo0kkVX1rngygb/KQI6z1qZxlFI3scluC+LVTCFbrkWjBv4vvLlbQ9t88IPMC6k95VTTg==", "engines": { "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://www.patreon.com/infusion" } }, "node_modules/concat-map": { @@ -5347,10 +5146,9 @@ } }, "node_modules/decimal.js": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.0.tgz", - "integrity": "sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw==", - "license": "MIT" + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" }, "node_modules/decode-named-character-reference": { "version": "1.0.1", @@ -5797,8 +5595,7 @@ "node_modules/escape-latex": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", - "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==", - "license": "MIT" + "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==" }, "node_modules/escape-string-regexp": { "version": "4.0.0", @@ -6718,6 +6515,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^2.0.0" @@ -7109,6 +6907,7 @@ "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dev": true, "license": "MIT", "dependencies": { "loose-envify": "^1.0.0" @@ -7503,8 +7302,7 @@ "node_modules/javascript-natural-sort": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", - "integrity": "sha1-+eIwPUUH9tdDVac2ZNFED7Wg71k=", - "license": "MIT" + "integrity": "sha1-+eIwPUUH9tdDVac2ZNFED7Wg71k=" }, "node_modules/js-tokens": { "version": "4.0.0", @@ -7580,11 +7378,6 @@ "node": ">=0.4.0" } }, - "node_modules/jsdom/node_modules/decimal.js": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", - "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" - }, "node_modules/jsdom/node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -7819,6 +7612,11 @@ "html2canvas": "^1.0.0-rc.5" } }, + "node_modules/jstat": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/jstat/-/jstat-1.9.5.tgz", + "integrity": "sha512-cWnp4vObF5GmB2XsIEzxI/1ZTcYlcfNqxQ/9Fp5KFUa0Jf/4tO0ZkGVnqoEHDisJvYgvn5n3eWZbd2xTVJJPUQ==" + }, "node_modules/jwa": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", @@ -8553,34 +8351,25 @@ } }, "node_modules/mathjs": { - "version": "5.10.3", - "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-5.10.3.tgz", - "integrity": "sha512-ySjg30BC3dYjQm73ILZtwcWzFJde0VU6otkXW/57IjjuYRa3Qaf0Kb8pydEuBZYtqW2OxreAtsricrAmOj3jIw==", - "license": "Apache-2.0", + "version": "10.5.2", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-10.5.2.tgz", + "integrity": "sha512-yLl2yA597A+07+xsOgy1+pq/m4ZokQceWQbB9OsIkK2guCpA2Px9k11zul93eJAA0WUFZiMK6VcruY9ZBbSKjA==", "dependencies": { - "complex.js": "2.0.11", - "decimal.js": "10.2.0", - "escape-latex": "1.2.0", - "fraction.js": "4.0.12", - "javascript-natural-sort": "0.7.1", - "seed-random": "2.2.0", - "tiny-emitter": "2.1.0", - "typed-function": "1.1.0" + "@babel/runtime": "^7.17.9", + "complex.js": "^2.1.1", + "decimal.js": "^10.3.1", + "escape-latex": "^1.2.0", + "fraction.js": "^4.2.0", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^2.1.0" }, "bin": { "mathjs": "bin/cli.js" }, "engines": { - "node": ">= 6" - } - }, - "node_modules/mathjs/node_modules/fraction.js": { - "version": "4.0.12", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.0.12.tgz", - "integrity": "sha512-8Z1K0VTG4hzYY7kA/1sj4/r1/RWLBD3xwReT/RCrUCbzPszjNQCCsy3ktkU/eaEqX3MYa4pY37a52eiBlPMlhA==", - "license": "MIT OR GPL-2.0", - "engines": { - "node": "*" + "node": ">= 12" } }, "node_modules/mdast-util-definitions": { @@ -36501,8 +36290,7 @@ "node_modules/pdfast": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/pdfast/-/pdfast-0.2.0.tgz", - "integrity": "sha1-jLxVbhvyUiF3eHwN4uDUNzuohck=", - "license": "MIT" + "integrity": "sha1-jLxVbhvyUiF3eHwN4uDUNzuohck=" }, "node_modules/performance-now": { "version": "2.1.0", @@ -37388,12 +37176,6 @@ "performance-now": "^2.1.0" } }, - "node_modules/rationale": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/rationale/-/rationale-0.2.0.tgz", - "integrity": "sha512-Pd8w5Inv1JhTfRyx03zs486CEAn6UKXvvOtxVRLsewngsBSffo3MQwUKYS75L/8vPt98wmf7iaZROx362/f7Bw==", - "license": "MIT" - }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -37810,19 +37592,11 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, - "node_modules/reschema": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/reschema/-/reschema-2.2.0.tgz", - "integrity": "sha512-vkFNYm2GEqrmoK+n1xomn/EXgltvCfqmowMxXkr2IBa6Zw0JItunzOap73PaS9DUGak0h6xHoIOciGcq+LC9cg==", - "license": "MIT" - }, "node_modules/rescript": { "version": "9.1.4", "resolved": "https://registry.npmjs.org/rescript/-/rescript-9.1.4.tgz", "integrity": "sha512-aXANK4IqecJzdnDpJUsU6pxMViCR5ogAxzuqS0mOr8TloMnzAjJFu63fjD6LCkWrKAhlMkFFzQvVQYaAaVkFXw==", "hasInstallScript": true, - "license": "SEE LICENSE IN LICENSE", - "peer": true, "bin": { "bsc": "bsc", "bsrefmt": "bsrefmt", @@ -37830,17 +37604,6 @@ "rescript": "rescript" } }, - "node_modules/rescript-react-update": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rescript-react-update/-/rescript-react-update-3.0.2.tgz", - "integrity": "sha512-oOOaeWma6XEhfZwlB/hB6u+UfYtuHjWS4RpoBeE8NbPX60GYLx5BLoVjEqLKoqXFPJv/2ZgghOKBn2LZo0VaJg==", - "license": "MIT", - "peerDependencies": { - "@rescript/react": "^0.10.1", - "react": "^16.8.0 || ^17.0.0", - "react-dom": "^16.8.0 || ^17.0.0" - } - }, "node_modules/resolve": { "version": "1.22.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz", @@ -38013,11 +37776,10 @@ "integrity": "sha512-MuCAyrGZcTLfQoH2XoBlQ8C6bzwN88XT/0slOGz0pn8+gIP85BOAfYa44ZXQUTOwRwPU0QvgU+V+OSajl/59Xg==", "dev": true }, - "node_modules/seed-random": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/seed-random/-/seed-random-2.2.0.tgz", - "integrity": "sha1-KpsZ4lCoFwmSMaW5mk2vgLf77VQ=", - "license": "MIT" + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "node_modules/selfsigned": { "version": "2.0.0", @@ -38335,27 +38097,6 @@ "tslib": "^2.0.3" } }, - "node_modules/squiggle-experimental": { - "version": "0.1.9", - "resolved": "https://registry.npmjs.org/squiggle-experimental/-/squiggle-experimental-0.1.9.tgz", - "integrity": "sha512-ObEKX+2/chpeRY51u27o41oggYRzHmrdslHLOQkqsiM+hslQAgHeulS5xE+md09DMTAgkUOZFtil3Sb3Vxty2Q==", - "license": "MIT", - "dependencies": { - "@glennsl/bs-json": "^5.0.2", - "@rescriptbr/reform": "^11.0.1", - "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2", - "lodash": "4.17.15", - "mathjs": "5.10.3", - "pdfast": "^0.2.0", - "rationale": "0.2.0" - } - }, - "node_modules/squiggle-experimental/node_modules/lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "license": "MIT" - }, "node_modules/stable": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", @@ -38429,6 +38170,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^2.0.0" @@ -38772,8 +38514,7 @@ "node_modules/tiny-emitter": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", - "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==", - "license": "MIT" + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" }, "node_modules/tiny-glob": { "version": "0.2.9", @@ -39018,11 +38759,11 @@ } }, "node_modules/typed-function": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-1.1.0.tgz", - "integrity": "sha512-TuQzwiT4DDg19beHam3E66oRXhyqlyfgjHB/5fcvsRXbfmWPJfto9B4a0TBdTrQAPGlGmXh/k7iUI+WsObgORA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-2.1.0.tgz", + "integrity": "sha512-bctQIOqx2iVbWGDGPWwIm18QScpu2XRmkC19D8rQGFsjKSgteq/o1hTZvIG/wuDq8fanpBDrLkLq+aEN/6y5XQ==", "engines": { - "node": ">= 6" + "node": ">= 10" } }, "node_modules/typescript": { @@ -40947,9 +40688,9 @@ } }, "@babel/runtime": { - "version": "7.17.7", - "resolved": "https://registry.npmjs.org/@babel%2fruntime/-/runtime-7.17.7.tgz", - "integrity": "sha512-L6rvG9GDxaLgFjg41K+5Yv9OMrU98sWe+Ykmc6FDJW/+vYZMhdOMKkISgzptMaERHvS2Y2lw9MDRm2gHhlQQoA==", + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", "requires": { "regenerator-runtime": "^0.13.4" } @@ -41255,11 +40996,6 @@ "use-isomorphic-layout-effect": "^1.1.1" } }, - "@glennsl/bs-json": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@glennsl%2fbs-json/-/bs-json-5.0.4.tgz", - "integrity": "sha512-Th9DetZjRlMZrb74kgGJ44oWcoFyOTE884WlSuXft0Cd+J09vHRxiB7eVyK7Gthb4cSevsBBJDHYAbGGL25wPw==" - }, "@graphql-codegen/add": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@graphql-codegen/add/-/add-3.1.1.tgz", @@ -42154,27 +41890,22 @@ "cross-spawn": "7.0.3" } }, + "@quri/squiggle-lang": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@quri/squiggle-lang/-/squiggle-lang-0.2.8.tgz", + "integrity": "sha512-uK+oXE0chYtEW3la0qIxg6ueXRm34jvkL930RWhazVWC0S4aU2lzPJQ9Oz1h+8qIZuQRs89ZqfeO5r5IcjUTNQ==", + "requires": { + "jstat": "^1.9.5", + "mathjs": "^10.5.0", + "pdfast": "^0.2.0", + "rescript": "^9.1.4" + } + }, "@repeaterjs/repeater": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@repeaterjs/repeater/-/repeater-3.0.4.tgz", "integrity": "sha512-AW8PKd6iX3vAZ0vA43nOUOnbq/X5ihgU+mSXXqunMkeQADGiqw/PY0JNeYtD5sr0PAy51YPgAPbDoeapv9r8WA==" }, - "@rescript/react": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@rescript%2freact/-/react-0.10.3.tgz", - "integrity": "sha512-Lf9rzrR3bQPKJjOK3PBRa/B3xrJ7CqQ1HYr9VHPVxJidarIJJFZBhj0Dg1uZURX+Wg/xiP0PHFxXmdj2bK8Vxw==", - "peer": true, - "requires": {} - }, - "@rescriptbr/reform": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/@rescriptbr%2freform/-/reform-11.0.2.tgz", - "integrity": "sha512-4C4iZQ5oQKhG1hLc0fJM2Dn47zQYegDGV7Qgp18nQX/3nVNg4JrlP3Fx9eD8NvqUmd3V5ew710Y+TU5FNS8n6A==", - "requires": { - "reschema": "^2.0.3", - "rescript-react-update": "^3.0.1" - } - }, "@samverschueren/stream-to-observable": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", @@ -42442,6 +42173,11 @@ "resolved": "https://registry.npmjs.org/@types/dom-to-image/-/dom-to-image-2.6.4.tgz", "integrity": "sha512-UddUdGF1qulrSDulkz3K2Ypq527MR6ixlgAzqLbxSiQ0icx0XDlIV+h4+edmjq/1dqn0KgN0xGSe1kI9t+vGuw==" }, + "@types/google-spreadsheet": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/google-spreadsheet/-/google-spreadsheet-3.2.1.tgz", + "integrity": "sha512-k9fMPyRJuWFoRiD4ZU2xTJTBykeXIh+ittw59UHGvNVkyu0b82DorkEJe36LBlaLkqD2qPBAunVz6JZRMFvWIA==" + }, "@types/hast": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/@types%2fhast/-/hast-2.3.4.tgz", @@ -42603,6 +42339,14 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==" }, + "@types/tunnel": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", + "integrity": "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==", + "requires": { + "@types/node": "*" + } + }, "@types/unist": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types%2funist/-/unist-2.0.6.tgz", @@ -42749,7 +42493,8 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true }, "ansi-styles": { "version": "4.3.0", @@ -42861,58 +42606,6 @@ "follow-redirects": "^1.14.7" } }, - "babel-code-frame": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", - "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", - "requires": { - "chalk": "^1.1.3", - "esutils": "^2.0.2", - "js-tokens": "^3.0.2" - }, - "dependencies": { - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" - }, - "js-tokens": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", - "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=" - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, - "babel-messages": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-messages/-/babel-messages-6.23.0.tgz", - "integrity": "sha1-8830cDhYA1sqKVHG7F7fbGLyYw4=", - "requires": { - "babel-runtime": "^6.22.0" - } - }, "babel-plugin-dynamic-import-node": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", @@ -42952,26 +42645,6 @@ "integrity": "sha512-Xj9XuRuz3nTSbaTXWv3itLOcxyF4oPD8douBBmj7U9BBC6nEBYfyOJYQMf/8PJAFotC62UY5dFfIGEPr7WswzQ==", "dev": true }, - "babel-plugin-transform-es2015-modules-commonjs": { - "version": "6.26.2", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.26.2.tgz", - "integrity": "sha512-CV9ROOHEdrjcwhIaJNBGMBCodN+1cfkwtM1SbUHmvyy35KGT7fohbpOxkE2uLz1o6odKK2Ck/tz47z+VqQfi9Q==", - "requires": { - "babel-plugin-transform-strict-mode": "^6.24.1", - "babel-runtime": "^6.26.0", - "babel-template": "^6.26.0", - "babel-types": "^6.26.0" - } - }, - "babel-plugin-transform-strict-mode": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz", - "integrity": "sha1-1fr3qleKZbvlkc9e2uBKDGcCB1g=", - "requires": { - "babel-runtime": "^6.22.0", - "babel-types": "^6.24.1" - } - }, "babel-preset-fbjs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/babel-preset-fbjs/-/babel-preset-fbjs-3.4.0.tgz", @@ -43007,98 +42680,6 @@ "babel-plugin-syntax-trailing-function-commas": "^7.0.0-beta.0" } }, - "babel-runtime": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", - "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", - "requires": { - "core-js": "^2.4.0", - "regenerator-runtime": "^0.11.0" - }, - "dependencies": { - "core-js": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", - "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==" - }, - "regenerator-runtime": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" - } - } - }, - "babel-template": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-template/-/babel-template-6.26.0.tgz", - "integrity": "sha1-3gPi0WOWsGn0bdn/+FIfsaDjXgI=", - "requires": { - "babel-runtime": "^6.26.0", - "babel-traverse": "^6.26.0", - "babel-types": "^6.26.0", - "babylon": "^6.18.0", - "lodash": "^4.17.4" - } - }, - "babel-traverse": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-traverse/-/babel-traverse-6.26.0.tgz", - "integrity": "sha1-RqnL1+3MYsjlwGTi0tjQ9ANXZu4=", - "requires": { - "babel-code-frame": "^6.26.0", - "babel-messages": "^6.23.0", - "babel-runtime": "^6.26.0", - "babel-types": "^6.26.0", - "babylon": "^6.18.0", - "debug": "^2.6.8", - "globals": "^9.18.0", - "invariant": "^2.2.2", - "lodash": "^4.17.4" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "globals": { - "version": "9.18.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-9.18.0.tgz", - "integrity": "sha512-S0nG3CLEQiY/ILxqtztTWH/3iRRdyBLw6KMDxnKMchrtbj2OFmehVh0WUCfW3DUrIgx/qFrJPICrq4Z4sTR9UQ==" - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "babel-types": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz", - "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=", - "requires": { - "babel-runtime": "^6.26.0", - "esutils": "^2.0.2", - "lodash": "^4.17.4", - "to-fast-properties": "^1.0.3" - }, - "dependencies": { - "to-fast-properties": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", - "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=" - } - } - }, - "babylon": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", - "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==" - }, "backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", @@ -43642,9 +43223,9 @@ "dev": true }, "complex.js": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.0.11.tgz", - "integrity": "sha512-6IArJLApNtdg1P1dFtn3dnyzoZBEF0MwMnrfF1exSBRpZYoy4yieMkpZhQDC0uwctw48vii0CFVyHfpgZ/DfGw==" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.1.1.tgz", + "integrity": "sha512-8njCHOTtFFLtegk6zQo0kkVX1rngygb/KQI6z1qZxlFI3scluC+LVTCFbrkWjBv4vvLlbQ9t88IPMC6k95VTTg==" }, "concat-map": { "version": "0.0.1", @@ -44073,9 +43654,9 @@ "dev": true }, "decimal.js": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.0.tgz", - "integrity": "sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw==" + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", + "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" }, "decode-named-character-reference": { "version": "1.0.1", @@ -45059,6 +44640,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, "requires": { "ansi-regex": "^2.0.0" } @@ -45334,6 +44916,7 @@ "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dev": true, "requires": { "loose-envify": "^1.0.0" } @@ -45663,11 +45246,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.0.tgz", "integrity": "sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==" }, - "decimal.js": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", - "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==" - }, "form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -45850,6 +45428,11 @@ "html2canvas": "^1.0.0-rc.5" } }, + "jstat": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/jstat/-/jstat-1.9.5.tgz", + "integrity": "sha512-cWnp4vObF5GmB2XsIEzxI/1ZTcYlcfNqxQ/9Fp5KFUa0Jf/4tO0ZkGVnqoEHDisJvYgvn5n3eWZbd2xTVJJPUQ==" + }, "jwa": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", @@ -46430,25 +46013,19 @@ "integrity": "sha512-y8j3a5/DkJCmS5x4dMCQL+OR0+2EAq3DOtio1COSHsmW2BGXnNCK3v12hJt1LrUz5iZH5g0LmuYOjDdI+czghA==" }, "mathjs": { - "version": "5.10.3", - "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-5.10.3.tgz", - "integrity": "sha512-ySjg30BC3dYjQm73ILZtwcWzFJde0VU6otkXW/57IjjuYRa3Qaf0Kb8pydEuBZYtqW2OxreAtsricrAmOj3jIw==", + "version": "10.5.2", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-10.5.2.tgz", + "integrity": "sha512-yLl2yA597A+07+xsOgy1+pq/m4ZokQceWQbB9OsIkK2guCpA2Px9k11zul93eJAA0WUFZiMK6VcruY9ZBbSKjA==", "requires": { - "complex.js": "2.0.11", - "decimal.js": "10.2.0", - "escape-latex": "1.2.0", - "fraction.js": "4.0.12", - "javascript-natural-sort": "0.7.1", - "seed-random": "2.2.0", - "tiny-emitter": "2.1.0", - "typed-function": "1.1.0" - }, - "dependencies": { - "fraction.js": { - "version": "4.0.12", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.0.12.tgz", - "integrity": "sha512-8Z1K0VTG4hzYY7kA/1sj4/r1/RWLBD3xwReT/RCrUCbzPszjNQCCsy3ktkU/eaEqX3MYa4pY37a52eiBlPMlhA==" - } + "@babel/runtime": "^7.17.9", + "complex.js": "^2.1.1", + "decimal.js": "^10.3.1", + "escape-latex": "^1.2.0", + "fraction.js": "^4.2.0", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^2.1.0" } }, "mdast-util-definitions": { @@ -68043,11 +67620,6 @@ "performance-now": "^2.1.0" } }, - "rationale": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/rationale/-/rationale-0.2.0.tgz", - "integrity": "sha512-Pd8w5Inv1JhTfRyx03zs486CEAn6UKXvvOtxVRLsewngsBSffo3MQwUKYS75L/8vPt98wmf7iaZROx362/f7Bw==" - }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -68359,22 +67931,10 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, - "reschema": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/reschema/-/reschema-2.2.0.tgz", - "integrity": "sha512-vkFNYm2GEqrmoK+n1xomn/EXgltvCfqmowMxXkr2IBa6Zw0JItunzOap73PaS9DUGak0h6xHoIOciGcq+LC9cg==" - }, "rescript": { "version": "9.1.4", "resolved": "https://registry.npmjs.org/rescript/-/rescript-9.1.4.tgz", - "integrity": "sha512-aXANK4IqecJzdnDpJUsU6pxMViCR5ogAxzuqS0mOr8TloMnzAjJFu63fjD6LCkWrKAhlMkFFzQvVQYaAaVkFXw==", - "peer": true - }, - "rescript-react-update": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rescript-react-update/-/rescript-react-update-3.0.2.tgz", - "integrity": "sha512-oOOaeWma6XEhfZwlB/hB6u+UfYtuHjWS4RpoBeE8NbPX60GYLx5BLoVjEqLKoqXFPJv/2ZgghOKBn2LZo0VaJg==", - "requires": {} + "integrity": "sha512-aXANK4IqecJzdnDpJUsU6pxMViCR5ogAxzuqS0mOr8TloMnzAjJFu63fjD6LCkWrKAhlMkFFzQvVQYaAaVkFXw==" }, "resolve": { "version": "1.22.0", @@ -68484,10 +68044,10 @@ "integrity": "sha512-MuCAyrGZcTLfQoH2XoBlQ8C6bzwN88XT/0slOGz0pn8+gIP85BOAfYa44ZXQUTOwRwPU0QvgU+V+OSajl/59Xg==", "dev": true }, - "seed-random": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/seed-random/-/seed-random-2.2.0.tgz", - "integrity": "sha1-KpsZ4lCoFwmSMaW5mk2vgLf77VQ=" + "seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "selfsigned": { "version": "2.0.0", @@ -68704,27 +68264,6 @@ "tslib": "^2.0.3" } }, - "squiggle-experimental": { - "version": "0.1.9", - "resolved": "https://registry.npmjs.org/squiggle-experimental/-/squiggle-experimental-0.1.9.tgz", - "integrity": "sha512-ObEKX+2/chpeRY51u27o41oggYRzHmrdslHLOQkqsiM+hslQAgHeulS5xE+md09DMTAgkUOZFtil3Sb3Vxty2Q==", - "requires": { - "@glennsl/bs-json": "^5.0.2", - "@rescriptbr/reform": "^11.0.1", - "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2", - "lodash": "4.17.15", - "mathjs": "5.10.3", - "pdfast": "^0.2.0", - "rationale": "0.2.0" - }, - "dependencies": { - "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" - } - } - }, "stable": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", @@ -68786,6 +68325,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, "requires": { "ansi-regex": "^2.0.0" } @@ -69195,9 +68735,9 @@ "integrity": "sha512-EUV9jo4sffrwlg8s0zDhP0T2WD3pru5Xi0+HTE3zTUmBaZNhfkite9PdSJwdXLwPVW0jnAHT56pZHIOYckPEiw==" }, "typed-function": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-1.1.0.tgz", - "integrity": "sha512-TuQzwiT4DDg19beHam3E66oRXhyqlyfgjHB/5fcvsRXbfmWPJfto9B4a0TBdTrQAPGlGmXh/k7iUI+WsObgORA==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-2.1.0.tgz", + "integrity": "sha512-bctQIOqx2iVbWGDGPWwIm18QScpu2XRmkC19D8rQGFsjKSgteq/o1hTZvIG/wuDq8fanpBDrLkLq+aEN/6y5XQ==" }, "typescript": { "version": "4.6.2", diff --git a/package.json b/package.json index 2eecce6..f141b9e 100644 --- a/package.json +++ b/package.json @@ -33,15 +33,18 @@ "@pothos/plugin-prisma": "^3.4.0", "@pothos/plugin-relay": "^3.10.0", "@prisma/client": "^3.11.1", + "@quri/squiggle-lang": "^0.2.8", "@tailwindcss/forms": "^0.4.0", "@tailwindcss/typography": "^0.5.1", "@types/chroma-js": "^2.1.3", "@types/dom-to-image": "^2.6.4", + "@types/google-spreadsheet": "^3.2.1", "@types/jsdom": "^16.2.14", "@types/nprogress": "^0.2.0", "@types/react": "^17.0.39", "@types/react-copy-to-clipboard": "^5.0.2", "@types/textversionjs": "^1.1.1", + "@types/tunnel": "^0.0.3", "airtable": "^0.11.1", "algoliasearch": "^4.10.3", "autoprefixer": "^10.1.0", @@ -88,7 +91,6 @@ "react-safe": "^1.3.0", "react-select": "^5.2.2", "remark-gfm": "^3.0.1", - "squiggle-experimental": "^0.1.9", "tabletojson": "^2.0.4", "tailwindcss": "^3.0.22", "textversionjs": "^1.1.3", diff --git a/src/backend/flow/doEverything.ts b/src/backend/flow/doEverything.ts index c6e0bab..64835d0 100644 --- a/src/backend/flow/doEverything.ts +++ b/src/backend/flow/doEverything.ts @@ -3,14 +3,7 @@ import { executeJobByName } from "./jobs"; /* Do everything */ export async function doEverything() { - let jobNames = [ - ...platforms.map((platform) => platform.name), - "merge", - "algolia", - "history", - "netlify", - ]; - // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku. + let jobNames = [...platforms.map((platform) => platform.name), "algolia"]; console.log(""); console.log(""); diff --git a/src/backend/flow/history/updateHistory.ts b/src/backend/flow/history/updateHistory.ts deleted file mode 100644 index 8b56de3..0000000 --- a/src/backend/flow/history/updateHistory.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { prisma } from "../../database/prisma"; - -export async function updateHistory() { - const questions = await prisma.question.findMany({}); - await prisma.history.createMany({ - data: questions.map((q) => ({ - ...q, - idref: q.id, - })), - }); -} diff --git a/src/backend/flow/jobs.ts b/src/backend/flow/jobs.ts index 01a5a81..099e3c6 100644 --- a/src/backend/flow/jobs.ts +++ b/src/backend/flow/jobs.ts @@ -1,9 +1,8 @@ import { doEverything } from "../flow/doEverything"; -import { updateHistory } from "../flow/history/updateHistory"; -import { rebuildNetlifySiteWithNewData } from "../flow/rebuildNetliftySiteWithNewData"; import { rebuildFrontpage } from "../frontpage"; import { platforms, processPlatform } from "../platforms"; import { rebuildAlgoliaDatabase } from "../utils/algolia"; +import { sleep } from "../utils/sleep"; interface Job { name: string; @@ -23,16 +22,6 @@ export const jobs: Job[] = [ message: 'Rebuild algolia database ("index")', run: rebuildAlgoliaDatabase, }, - { - name: "history", - message: "Update history", - run: updateHistory, - }, - { - name: "netlify", - message: `Rebuild netlify site with new data`, - run: rebuildNetlifySiteWithNewData, - }, { name: "frontpage", message: "Rebuild frontpage", @@ -46,10 +35,6 @@ export const jobs: Job[] = [ }, ]; -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - async function tryCatchTryAgain(fun: () => Promise) { try { console.log("Initial try"); diff --git a/src/backend/flow/rebuildNetliftySiteWithNewData.ts b/src/backend/flow/rebuildNetliftySiteWithNewData.ts deleted file mode 100644 index 72ab28c..0000000 --- a/src/backend/flow/rebuildNetliftySiteWithNewData.ts +++ /dev/null @@ -1,15 +0,0 @@ -import axios from "axios"; - -import { applyIfSecretExists } from "../utils/getSecrets"; - -async function rebuildNetlifySiteWithNewData_inner(cookie: string) { - let payload = {}; - let response = await axios.post(cookie, payload); - let data = response.data; - console.log(data); -} - -export async function rebuildNetlifySiteWithNewData() { - const cookie = process.env.REBUIDNETLIFYHOOKURL || ""; - await applyIfSecretExists(cookie, rebuildNetlifySiteWithNewData_inner); -} diff --git a/src/backend/index.ts b/src/backend/index.ts index 2d9edfa..bb4234d 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -2,11 +2,10 @@ import "dotenv/config"; import readline from "readline"; -import util from "util"; import { executeJobByName, jobs } from "./flow/jobs"; -let generateWhatToDoMessage = () => { +const generateWhatToDoMessage = () => { const color = "\x1b[36m"; const resetColor = "\x1b[0m"; let completeMessages = [ @@ -23,10 +22,10 @@ let generateWhatToDoMessage = () => { return completeMessages; }; -let whattodoMessage = generateWhatToDoMessage(); +const whattodoMessage = generateWhatToDoMessage(); /* BODY */ -let commandLineUtility = async () => { +const commandLineUtility = async () => { const pickOption = async () => { if (process.argv.length === 3) { return process.argv[2]; // e.g., npm run cli polymarket @@ -37,9 +36,15 @@ let commandLineUtility = async () => { output: process.stdout, }); - const question = util.promisify(rl.question).bind(rl); + const question = (query: string) => { + return new Promise((resolve: (s: string) => void) => { + rl.question(query, resolve); + }); + }; + const answer = await question(whattodoMessage); rl.close(); + return answer; }; diff --git a/src/backend/platforms/_example.ts b/src/backend/platforms/_example.ts index e41113c..336bbc9 100644 --- a/src/backend/platforms/_example.ts +++ b/src/backend/platforms/_example.ts @@ -21,7 +21,7 @@ async function fetchData() { return response; } -async function processPredictions(predictions) { +async function processPredictions(predictions: any[]) { let results = await predictions.map((prediction) => { const id = `${platformName}-${prediction.id}`; const probability = prediction.probability; diff --git a/src/backend/platforms/betfair.ts b/src/backend/platforms/betfair.ts index aa950d8..d9495a1 100644 --- a/src/backend/platforms/betfair.ts +++ b/src/backend/platforms/betfair.ts @@ -27,7 +27,7 @@ const arraysEqual = (a: string[], b: string[]) => { return true; }; -const mergeRunners = (runnerCatalog, runnerBook) => { +const mergeRunners = (runnerCatalog: any, runnerBook: any) => { let keys = Object.keys(runnerCatalog); let result = []; for (let key of keys) { @@ -45,16 +45,13 @@ async function fetchPredictions() { const response = await axios({ url: endpoint, method: "GET", - headers: { - "Content-Type": "text/html", - }, httpsAgent: agent, }).then((response) => response.data); return response; } -async function whipIntoShape(data) { +async function whipIntoShape(data: any) { let catalogues = data.market_catalogues; let books = data.market_books; let keys1 = Object.keys(catalogues).sort(); @@ -78,7 +75,7 @@ async function whipIntoShape(data) { return results; } -async function processPredictions(data) { +async function processPredictions(data: any) { let predictions = await whipIntoShape(data); // console.log(JSON.stringify(predictions, null, 4)) let results: FetchedQuestion[] = predictions.map((prediction) => { @@ -87,13 +84,17 @@ async function processPredictions(data) { } */ let id = `${platformName}-${prediction.marketId}`; let normalizationFactor = prediction.options - .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0) - .map((option) => option.lastPriceTraded) - .map((x) => 1 / x) - .reduce((a, b) => a + b, 0); + .filter( + (option: any) => option.status == "ACTIVE" && option.totalMatched > 0 + ) + .map((option: any) => option.lastPriceTraded) + .map((x: any) => 1 / x) + .reduce((a: any, b: any) => a + b, 0); let options = prediction.options - .filter((option) => option.status == "ACTIVE" && option.totalMatched > 0) - .map((option) => ({ + .filter( + (option: any) => option.status == "ACTIVE" && option.totalMatched > 0 + ) + .map((option: any) => ({ name: option.runnerName, probability: option.lastPriceTraded != 0 @@ -142,7 +143,7 @@ export const betfair: Platform = { color: "#3d674a", async fetcher() { const data = await fetchPredictions(); - const results = await processPredictions(data); // somehow needed + const results = await processPredictions(data); return results; }, calculateStars(data) { diff --git a/src/backend/platforms/fantasyscotus.ts b/src/backend/platforms/fantasyscotus.ts index 7527599..8181afe 100644 --- a/src/backend/platforms/fantasyscotus.ts +++ b/src/backend/platforms/fantasyscotus.ts @@ -29,7 +29,7 @@ async function fetchData() { return response; } -async function getPredictionsData(caseUrl) { +async function getPredictionsData(caseUrl: string) { let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`; //console.log(newCaseUrl) let predictions = await axios({ @@ -49,7 +49,7 @@ async function getPredictionsData(caseUrl) { }).then((res) => res.data); let predictionsAffirm = predictions.filter( - (prediction) => prediction.percent_affirm > 50 + (prediction: any) => prediction.percent_affirm > 50 ); //console.log(predictions) //console.log(predictionsAffirm.length/predictions.length) @@ -61,7 +61,7 @@ async function getPredictionsData(caseUrl) { }; } -async function processData(data) { +async function processData(data: any) { let events = data.object_list; let historicalPercentageCorrect = data.stats.pcnt_correct; let historicalProbabilityCorrect = diff --git a/src/backend/platforms/foretold.ts b/src/backend/platforms/foretold.ts index 0673fae..55692e3 100644 --- a/src/backend/platforms/foretold.ts +++ b/src/backend/platforms/foretold.ts @@ -18,8 +18,10 @@ let highQualityCommunities = [ ]; /* Support functions */ -async function fetchAllCommunityQuestions(communityId) { - let response = await axios({ +async function fetchAllCommunityQuestions(communityId: string) { + // TODO - fetch foretold graphql schema to type the result properly? + // (should be doable with graphql-code-generator, why not) + const response = await axios({ url: graphQLendpoint, method: "POST", headers: { "Content-Type": "application/json" }, @@ -30,10 +32,10 @@ async function fetchAllCommunityQuestions(communityId) { channelId: "${communityId}", states: OPEN, first: 500 - ){ + ) { total - edges{ - node{ + edges { + node { id name valueType @@ -52,8 +54,8 @@ async function fetchAllCommunityQuestions(communityId) { }) .then((res) => res.data) .then((res) => res.data.measurables.edges); - //console.log(response) - return response; + + return response as any[]; } export const foretold: Platform = { @@ -67,11 +69,11 @@ export const foretold: Platform = { questions = questions.map((question) => question.node); questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions questions.forEach((question) => { - let id = `${platformName}-${question.id}`; + const id = `${platformName}-${question.id}`; let options: FetchedQuestion["options"] = []; if (question.valueType == "PERCENTAGE") { - let probability = question.previousAggregate.value.percentage; + const probability = question.previousAggregate.value.percentage; options = [ { name: "Yes", diff --git a/src/backend/platforms/goodjudgment.ts b/src/backend/platforms/goodjudgment.ts index ab487bf..be78ebe 100644 --- a/src/backend/platforms/goodjudgment.ts +++ b/src/backend/platforms/goodjudgment.ts @@ -1,7 +1,6 @@ /* Imports */ import axios from "axios"; import { Tabletojson } from "tabletojson"; -import tunnel from "tunnel"; import { average } from "../../utils"; import { hash } from "../utils/hash"; @@ -18,7 +17,7 @@ export const goodjudgment: Platform = { color: "#7d4f1b", async fetcher() { // Proxy fuckery - let proxy; + // let proxy; /* * try { proxy = await axios @@ -29,19 +28,19 @@ export const goodjudgment: Platform = { console.log("Proxy generation failed; using backup proxy instead"); // hard-coded backup proxy */ - proxy = { - ip: process.env.BACKUP_PROXY_IP, - port: process.env.BACKUP_PROXY_PORT, - }; - // } - let agent = tunnel.httpsOverHttp({ - proxy: { - host: proxy.ip, - port: proxy.port, - }, - }); + // proxy = { + // ip: process.env.BACKUP_PROXY_IP, + // port: process.env.BACKUP_PROXY_PORT, + // }; + // // } + // let agent = tunnel.httpsOverHttp({ + // proxy: { + // host: proxy.ip, + // port: proxy.port, + // }, + // }); - let content = await axios + const content = await axios .request({ url: "https://goodjudgment.io/superforecasts/", method: "get", @@ -58,17 +57,16 @@ export const goodjudgment: Platform = { let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); jsonTable.shift(); // deletes first element jsonTable.pop(); // deletes last element - // console.log(jsonTable) + for (let table of jsonTable) { - // console.log(table) let title = table[0]["0"].split("\t\t\t").splice(3)[0]; if (title != undefined) { title = title.replaceAll("", ""); - let id = `${platformName}-${hash(title)}`; - let description = table - .filter((row) => row["0"].includes("BACKGROUND:")) - .map((row) => row["0"]) - .map((text) => + const id = `${platformName}-${hash(title)}`; + const description = table + .filter((row: any) => row["0"].includes("BACKGROUND:")) + .map((row: any) => row["0"]) + .map((text: any) => text .split("BACKGROUND:")[1] .split("Examples of Superforecaster")[0] @@ -80,16 +78,16 @@ export const goodjudgment: Platform = { .replaceAll(" ", "") .replaceAll("
", "") )[0]; - let options = table - .filter((row) => "4" in row) - .map((row) => ({ + const options = table + .filter((row: any) => "4" in row) + .map((row: any) => ({ name: row["2"] .split('')[1] .replace("", ""), probability: Number(row["3"].split("%")[0]) / 100, type: "PROBABILITY", })); - let analysis = table.filter((row) => + let analysis = table.filter((row: any) => row[0] ? row[0].toLowerCase().includes("commentary") : false ); // "Examples of Superforecaster Commentary" / Analysis diff --git a/src/backend/platforms/goodjudgmentopen.ts b/src/backend/platforms/goodjudgmentopen.ts index 05c7a8d..a22fbea 100644 --- a/src/backend/platforms/goodjudgmentopen.ts +++ b/src/backend/platforms/goodjudgmentopen.ts @@ -4,6 +4,7 @@ import { Tabletojson } from "tabletojson"; import { average } from "../../utils"; import { applyIfSecretExists } from "../utils/getSecrets"; +import { sleep } from "../utils/sleep"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; @@ -23,11 +24,10 @@ const id = () => 0; /* Support functions */ async function fetchPage(page: number, cookie: string) { - let response = await axios({ + const response: string = await axios({ url: htmlEndPoint + page, method: "GET", headers: { - "Content-Type": "text/html", Cookie: cookie, }, }).then((res) => res.data); @@ -36,11 +36,10 @@ async function fetchPage(page: number, cookie: string) { } async function fetchStats(questionUrl: string, cookie: string) { - let response = await axios({ + let response: string = await axios({ url: questionUrl + "/stats", method: "GET", headers: { - "Content-Type": "text/html", Cookie: cookie, Referer: questionUrl, }, @@ -74,7 +73,7 @@ async function fetchStats(questionUrl: string, cookie: string) { let optionsHtmlElement = ""; let tablesAsJson = Tabletojson.convert(optionsHtmlElement); let firstTable = tablesAsJson[0]; - options = firstTable.map((element) => ({ + options = firstTable.map((element: any) => ({ name: element["0"], probability: Number(element["1"].replace("%", "")) / 100, type: "PROBABILITY", @@ -133,7 +132,7 @@ function isSignedIn(html: string) { return isSignedInBool; } -function reachedEnd(html) { +function reachedEnd(html: string) { let reachedEndBool = html.includes("No questions match your filter"); if (reachedEndBool) { //console.log(html) @@ -142,10 +141,6 @@ function reachedEnd(html) { return reachedEndBool; } -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - /* Body */ async function goodjudgmentopen_inner(cookie: string) { @@ -176,7 +171,11 @@ async function goodjudgmentopen_inner(cookie: string) { } } let questionNumRegex = new RegExp("questions/([0-9]+)"); - let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; + const questionNumMatch = url.match(questionNumRegex); + if (!questionNumMatch) { + throw new Error(`Couldn't find question num in ${url}`); + } + let questionNum = questionNumMatch[1]; let id = `${platformName}-${questionNum}`; let question = { id: id, diff --git a/src/backend/platforms/guesstimate.ts b/src/backend/platforms/guesstimate.ts index b517215..432a5fd 100644 --- a/src/backend/platforms/guesstimate.ts +++ b/src/backend/platforms/guesstimate.ts @@ -2,8 +2,8 @@ import axios from "axios"; import { Question } from "@prisma/client"; -import { AlgoliaQuestion } from "../../backend/utils/algolia"; import { prisma } from "../database/prisma"; +import { AlgoliaQuestion } from "../utils/algolia"; import { FetchedQuestion, Platform, prepareQuestion } from "./"; /* Definitions */ @@ -12,7 +12,7 @@ const searchEndpoint = const apiEndpoint = "https://guesstimate.herokuapp.com"; -const modelToQuestion = (model: any): Question => { +const modelToQuestion = (model: any): ReturnType => { const { description } = model; // const description = model.description // ? model.description.replace(/\n/g, " ").replace(/ /g, " ") @@ -77,12 +77,11 @@ async function search(query: string): Promise { const fetchQuestion = async (id: number): Promise => { const response = await axios({ url: `${apiEndpoint}/spaces/${id}` }); let q = modelToQuestion(response.data); - q = await prisma.question.upsert({ + return await prisma.question.upsert({ where: { id: q.id }, create: q, update: q, }); - return q; }; export const guesstimate: Platform & { diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index d9127ed..87c6139 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -1,5 +1,6 @@ import { Question } from "@prisma/client"; +import { QuestionOption } from "../../common/types"; import { prisma } from "../database/prisma"; import { betfair } from "./betfair"; import { fantasyscotus } from "./fantasyscotus"; @@ -45,11 +46,7 @@ export type FetchedQuestion = Omit< > & { timestamp?: Date; extra?: object; // required in DB but annoying to return empty; also this is slightly stricter than Prisma's JsonValue - options: { - name?: string; - probability?: number; - type: "PROBABILITY"; - }[]; // stronger type than Prisma's JsonValue + options: QuestionOption[]; // stronger type than Prisma's JsonValue qualityindicators: Omit; // slightly stronger type than Prisma's JsonValue }; @@ -92,10 +89,23 @@ export const platforms: Platform[] = [ xrisk, ]; +// Typing notes: +// There's a difference between prisma's Question type (type returned from `find` and `findMany`) and its input types due to JsonValue vs InputJsonValue mismatch. +// On the other hand, we can't use Prisma.QuestionUpdateInput or Prisma.QuestionCreateManyInput either, because we use this question in guesstimate's code for preparing questions from guesstimate models... +// So here we build a new type which should be ok to use both in place of prisma's Question type and as an input to its update or create methods. +type PreparedQuestion = Omit< + Question, + "extra" | "qualityindicators" | "options" +> & { + extra: NonNullable; + qualityindicators: NonNullable; + options: NonNullable; +}; + export const prepareQuestion = ( q: FetchedQuestion, platform: Platform -): Question => { +): PreparedQuestion => { return { extra: {}, timestamp: new Date(), @@ -131,8 +141,8 @@ export const processPlatform = async (platform: Platform) => { const fetchedIdsSet = new Set(fetchedIds); const oldIdsSet = new Set(oldIds); - const createdQuestions: Question[] = []; - const updatedQuestions: Question[] = []; + const createdQuestions: PreparedQuestion[] = []; + const updatedQuestions: PreparedQuestion[] = []; const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id)); for (const q of fetchedQuestions.map((q) => prepareQuestion(q, platform))) { @@ -163,6 +173,13 @@ export const processPlatform = async (platform: Platform) => { }, }); + await prisma.history.createMany({ + data: [...createdQuestions, ...updatedQuestions].map((q) => ({ + ...q, + idref: q.id, + })), + }); + console.log( `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created` ); diff --git a/src/backend/platforms/infer.ts b/src/backend/platforms/infer.ts index b004917..9d22048 100644 --- a/src/backend/platforms/infer.ts +++ b/src/backend/platforms/infer.ts @@ -1,9 +1,11 @@ /* Imports */ import axios from "axios"; +import { FullQuestionOption } from "../../common/types"; import { average } from "../../utils"; import { applyIfSecretExists } from "../utils/getSecrets"; import { measureTime } from "../utils/measureTime"; +import { sleep } from "../utils/sleep"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; @@ -16,20 +18,20 @@ const SLEEP_TIME_EXTRA = 2000; /* Support functions */ -function cleanDescription(text) { +function cleanDescription(text: string) { let md = toMarkdown(text); let result = md.replaceAll("---", "-").replaceAll(" ", " "); return result; } -async function fetchPage(page, cookie) { +async function fetchPage(page: number, cookie: string) { console.log(`Page #${page}`); if (page == 1) { cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :( } let urlEndpoint = `${htmlEndPoint}/?page=${page}`; console.log(urlEndpoint); - let response = await axios({ + const response: string = await axios({ url: urlEndpoint, method: "GET", headers: { @@ -41,8 +43,8 @@ async function fetchPage(page, cookie) { return response; } -async function fetchStats(questionUrl, cookie) { - let response = await axios({ +async function fetchStats(questionUrl: string, cookie: string) { + let response: string = await axios({ url: questionUrl + "/stats", method: "GET", headers: { @@ -56,7 +58,7 @@ async function fetchStats(questionUrl, cookie) { throw Error("Not logged in"); } // Init - let options = []; + let options: FullQuestionOption[] = []; // Parse the embedded json let htmlElements = response.split("\n"); @@ -81,7 +83,7 @@ async function fetchStats(questionUrl, cookie) { questionType.includes("Forecast::Question") || !questionType.includes("Forecast::MultiTimePeriodQuestion") ) { - options = firstEmbeddedJson.question.answers.map((answer) => ({ + options = firstEmbeddedJson.question.answers.map((answer: any) => ({ name: answer.name, probability: answer.normalized_probability, type: "PROBABILITY", @@ -91,12 +93,11 @@ async function fetchStats(questionUrl, cookie) { options[0].probability > 1 ? 1 - options[0].probability / 100 : 1 - options[0].probability; - let optionNo = { + options.push({ name: "No", probability: probabilityNo, type: "PROBABILITY", - }; - options.push(optionNo); + }); } } let result = { @@ -112,7 +113,7 @@ async function fetchStats(questionUrl, cookie) { return result; } -function isSignedIn(html) { +function isSignedIn(html: string) { let isSignedInBool = !( html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") @@ -124,7 +125,7 @@ function isSignedIn(html) { return isSignedInBool; } -function reachedEnd(html) { +function reachedEnd(html: string) { let reachedEndBool = html.includes("No questions match your filter"); if (reachedEndBool) { //console.log(html) @@ -133,10 +134,6 @@ function reachedEnd(html) { return reachedEndBool; } -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - /* Body */ async function infer_inner(cookie: string) { @@ -169,14 +166,18 @@ async function infer_inner(cookie: string) { await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable try { - let moreinfo = await fetchStats(url, cookie); - let questionNumRegex = new RegExp("questions/([0-9]+)"); - let questionNum = url.match(questionNumRegex)[1]; //.split("questions/")[1].split("-")[0]; - let id = `${platformName}-${questionNum}`; + const moreinfo = await fetchStats(url, cookie); + const questionNumRegex = new RegExp("questions/([0-9]+)"); + const questionNumMatch = url.match(questionNumRegex); + if (!questionNumMatch) { + throw new Error(`Couldn't find question num in ${url}`); + } + let questionNum = questionNumMatch[1]; + const id = `${platformName}-${questionNum}`; let question: FetchedQuestion = { - id: id, - title: title, - url: url, + id, + title, + url, ...moreinfo, }; console.log(JSON.stringify(question, null, 4)); @@ -231,7 +232,7 @@ export const infer: Platform = { color: "#223900", async fetcher() { let cookie = process.env.INFER_COOKIE; - return await applyIfSecretExists(cookie, infer_inner); + return (await applyIfSecretExists(cookie, infer_inner)) || null; }, calculateStars(data) { let nuno = () => 2; diff --git a/src/backend/platforms/kalshi.ts b/src/backend/platforms/kalshi.ts index 665f0e6..7b76686 100644 --- a/src/backend/platforms/kalshi.ts +++ b/src/backend/platforms/kalshi.ts @@ -6,18 +6,17 @@ import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "kalshi"; -let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' +let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; async function fetchAllMarkets() { - // for info which the polymarket graphql API let response = await axios .get(jsonEndpoint) .then((response) => response.data.markets); - // console.log(response) + return response; } -async function processMarkets(markets) { +async function processMarkets(markets: any[]) { let dateNow = new Date().toISOString(); // console.log(markets) markets = markets.filter((market) => market.close_date > dateNow); diff --git a/src/backend/platforms/manifold.ts b/src/backend/platforms/manifold.ts index b735470..6a035a4 100644 --- a/src/backend/platforms/manifold.ts +++ b/src/backend/platforms/manifold.ts @@ -6,7 +6,7 @@ import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "manifold"; -let endpoint = "https://manifold.markets/api/v0/markets"; +const endpoint = "https://manifold.markets/api/v0/markets"; // See https://manifoldmarkets.notion.site/Manifold-Markets-API-5e7d0aef4dcf452bb04b319e178fabc5 /* Support functions */ @@ -43,8 +43,8 @@ function showStatistics(results: FetchedQuestion[]) { ); } -async function processPredictions(predictions) { - let results: FetchedQuestion[] = await predictions.map((prediction) => { +function processPredictions(predictions: any[]): FetchedQuestion[] { + let results: FetchedQuestion[] = predictions.map((prediction) => { let id = `${platformName}-${prediction.id}`; // oops, doesn't match platform name let probability = prediction.probability; let options: FetchedQuestion["options"] = [ @@ -90,7 +90,7 @@ export const manifold: Platform = { color: "#793466", async fetcher() { let data = await fetchData(); - let results = await processPredictions(data); // somehow needed + let results = processPredictions(data); // somehow needed showStatistics(results); return results; }, diff --git a/src/backend/platforms/metaculus.ts b/src/backend/platforms/metaculus.ts index 0327ca6..fc14057 100644 --- a/src/backend/platforms/metaculus.ts +++ b/src/backend/platforms/metaculus.ts @@ -2,17 +2,18 @@ import axios from "axios"; import { average } from "../../utils"; +import { sleep } from "../utils/sleep"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "metaculus"; -let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; let now = new Date().toISOString(); let DEBUG_MODE = "off"; let SLEEP_TIME = 5000; + /* Support functions */ -async function fetchMetaculusQuestions(next) { +async function fetchMetaculusQuestions(next: string) { // Numbers about a given address: how many, how much, at what price, etc. let response; let data; @@ -24,15 +25,17 @@ async function fetchMetaculusQuestions(next) { }); data = response.data; } catch (error) { - console.log(`Error in async function fetchMetaculusQuestions(next)`); - if (!!error.response.headers["retry-after"]) { - let timeout = error.response.headers["retry-after"]; - console.log(`Timeout: ${timeout}`); - await sleep(Number(timeout) * 1000 + SLEEP_TIME); - } else { - await sleep(SLEEP_TIME); - } + console.log(`Error in async function fetchMetaculusQuestions(next)`); console.log(error); + if (axios.isAxiosError(error)) { + if (error.response?.headers["retry-after"]) { + const timeout = error.response.headers["retry-after"]; + console.log(`Timeout: ${timeout}`); + await sleep(Number(timeout) * 1000 + SLEEP_TIME); + } else { + await sleep(SLEEP_TIME); + } + } } finally { try { response = await axios({ @@ -50,11 +53,7 @@ async function fetchMetaculusQuestions(next) { return data; } -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function fetchMetaculusQuestionDescription(slug) { +async function fetchMetaculusQuestionDescription(slug: string) { try { let response = await axios({ method: "get", @@ -67,11 +66,12 @@ async function fetchMetaculusQuestionDescription(slug) { `We encountered some error when attempting to fetch a metaculus page. Trying again` ); if ( + axios.isAxiosError(error) && typeof error.response != "undefined" && typeof error.response.headers != "undefined" && typeof error.response.headers["retry-after"] != "undefined" ) { - let timeout = error.response.headers["retry-after"]; + const timeout = error.response.headers["retry-after"]; console.log(`Timeout: ${timeout}`); await sleep(Number(timeout) * 1000 + SLEEP_TIME); } else { @@ -190,6 +190,7 @@ export const metaculus: Platform = { return all_questions; }, + calculateStars(data) { const { numforecasts } = data.qualityindicators; let nuno = () => diff --git a/src/backend/platforms/polymarket.ts b/src/backend/platforms/polymarket.ts index 395b60e..b263050 100644 --- a/src/backend/platforms/polymarket.ts +++ b/src/backend/platforms/polymarket.ts @@ -6,8 +6,8 @@ import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "polymarket"; -let graphQLendpoint = - "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; // "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-4"// "https://api.thegraph.com/subgraphs/name/tokenunion/polymarket-matic"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3' +const graphQLendpoint = + "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; let units = 10 ** 6; async function fetchAllContractInfo() { @@ -18,11 +18,11 @@ async function fetchAllContractInfo() { // "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1" to get all markets, including closed ones ) .then((query) => query.data); - response = response.filter((res) => res.closed != true); + response = response.filter((res: any) => res.closed != true); return response; } -async function fetchIndividualContractData(marketMakerAddress) { +async function fetchIndividualContractData(marketMakerAddress: string) { let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week let response = await axios({ url: graphQLendpoint, @@ -59,7 +59,7 @@ async function fetchIndividualContractData(marketMakerAddress) { }) .then((res) => res.data) .then((res) => res.data.fixedProductMarketMakers); - // console.log(response) + return response; } @@ -93,7 +93,7 @@ export const polymarket: Platform = { // let isbinary = Number(moreMarketInfo.conditions[0].outcomeSlotCount) == 2 // let percentage = Number(moreMarketInfo.outcomeTokenPrices[0]) * 100 // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none" - let options = []; + let options: FetchedQuestion["options"] = []; for (let outcome in moreMarketInfo.outcomeTokenPrices) { options.push({ name: String(marketInfo.outcomes[outcome]), @@ -107,7 +107,7 @@ export const polymarket: Platform = { title: marketInfo.question, url: "https://polymarket.com/market/" + marketInfo.slug, description: marketInfo.description, - options: options, + options, qualityindicators: { numforecasts: numforecasts.toFixed(0), liquidity: liquidity.toFixed(2), diff --git a/src/backend/platforms/predictit.ts b/src/backend/platforms/predictit.ts index 550a622..6b94956 100644 --- a/src/backend/platforms/predictit.ts +++ b/src/backend/platforms/predictit.ts @@ -1,24 +1,25 @@ import axios from "axios"; import { average } from "../../utils"; +import { sleep } from "../utils/sleep"; import toMarkdown from "../utils/toMarkdown"; import { FetchedQuestion, Platform } from "./"; const platformName = "predictit"; /* Support functions */ -async function fetchmarkets() { - let response = await axios({ +async function fetchmarkets(): Promise { + const response = await axios({ method: "get", url: "https://www.predictit.org/api/marketdata/all/", }); - let openMarkets = response.data.markets.filter( - (market) => market.status == "Open" + const openMarkets = response.data.markets.filter( + (market: any) => market.status == "Open" ); return openMarkets; } -async function fetchmarketrules(market_id) { +async function fetchmarketrules(market_id: string | number) { let response = await axios({ method: "get", url: "https://www.predictit.org/api/Market/" + market_id, @@ -34,10 +35,6 @@ async function fetchmarketvolumes() { return response.data; } -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - /* Body */ export const predictit: Platform = { name: platformName, @@ -65,13 +62,15 @@ export const predictit: Platform = { let shares_volume = market["TotalSharesTraded"]; // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none" - let options = market.contracts.map((contract) => ({ - name: contract.name, - probability: contract.lastTradePrice, - type: "PROBABILITY", - })); + let options: FetchedQuestion["options"] = (market.contracts as any[]).map( + (contract) => ({ + name: String(contract.name), + probability: Number(contract.lastTradePrice), + type: "PROBABILITY", + }) + ); let totalValue = options - .map((element) => Number(element.probability)) + .map((element: any) => Number(element.probability)) .reduce((a, b) => a + b, 0); if (options.length != 1 && totalValue > 1) { @@ -81,7 +80,7 @@ export const predictit: Platform = { })); } else if (options.length == 1) { let option = options[0]; - let probability = option["probability"]; + let probability = option.probability; options = [ { name: "Yes", @@ -90,7 +89,7 @@ export const predictit: Platform = { }, { name: "No", - probability: 1 - probability, + probability: 1 - (probability || 0), type: "PROBABILITY", }, ]; diff --git a/src/backend/platforms/rootclaim.ts b/src/backend/platforms/rootclaim.ts index 374e38e..4d58d4a 100644 --- a/src/backend/platforms/rootclaim.ts +++ b/src/backend/platforms/rootclaim.ts @@ -55,7 +55,7 @@ export const rootclaim: Platform = { for (const claim of claims) { const id = `${platformName}-${claim.slug.toLowerCase()}`; - let options = []; + let options: FetchedQuestion["options"] = []; for (let scenario of claim.scenarios) { options.push({ name: toMarkdown(scenario.name || scenario.text) @@ -76,7 +76,7 @@ export const rootclaim: Platform = { title: toMarkdown(claim.question).replace("\n", ""), url, description: toMarkdown(description).replace("'", "'"), - options: options, + options, qualityindicators: { numforecasts: 1, }, diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts index 52b6059..1d3fa19 100644 --- a/src/backend/platforms/smarkets.ts +++ b/src/backend/platforms/smarkets.ts @@ -1,5 +1,6 @@ import axios from "axios"; +import { QuestionOption } from "../../common/types"; import { average } from "../../utils"; import { FetchedQuestion, Platform } from "./"; @@ -7,57 +8,51 @@ import { FetchedQuestion, Platform } from "./"; const platformName = "smarkets"; let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; let VERBOSE = false; -let empty = () => 0; /* Support functions */ -async function fetchEvents(url) { - let response = await axios({ +async function fetchEvents(url: string) { + const response = await axios({ url: htmlEndPointEntrance + url, method: "GET", - headers: { - "Content-Type": "text/html", - }, }).then((res) => res.data); - VERBOSE ? console.log(response) : empty(); + VERBOSE && console.log(response); return response; } -async function fetchMarkets(eventid) { - let response = await axios({ +async function fetchMarkets(eventid: string) { + const response = await axios({ url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, method: "GET", - headers: { - "Content-Type": "text/json", - }, }) .then((res) => res.data) .then((res) => res.markets); return response; } -async function fetchContracts(marketid) { - let response = await axios({ +async function fetchContracts(marketid: string) { + const response = await axios({ url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, method: "GET", - headers: { - "Content-Type": "text/html", - }, }).then((res) => res.data); - VERBOSE ? console.log(response) : empty(); - return response; + VERBOSE && console.log(response); + + if (!(response.contracts instanceof Array)) { + throw new Error("Invalid response while fetching contracts"); + } + return response.contracts as any[]; } -async function fetchPrices(marketid) { - let response = await axios({ +async function fetchPrices(marketid: string) { + const response = await axios({ url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, method: "GET", - headers: { - "Content-Type": "text/html", - }, }).then((res) => res.data); - VERBOSE ? console.log(response) : empty(); - return response; + VERBOSE && console.log(response); + if (!response.last_executed_prices) { + throw new Error("Invalid response while fetching prices"); + } + return response.last_executed_prices; } export const smarkets: Platform = { @@ -70,77 +65,91 @@ export const smarkets: Platform = { let events = []; while (htmlPath) { - let data = await fetchEvents(htmlPath); + const data = await fetchEvents(htmlPath); events.push(...data.events); htmlPath = data.pagination.next_page; } - VERBOSE ? console.log(events) : empty(); + VERBOSE && console.log(events); + let markets = []; - for (let event of events) { - VERBOSE ? console.log(Date.now()) : empty(); - VERBOSE ? console.log(event.name) : empty(); + for (const event of events) { + VERBOSE && console.log(Date.now()); + VERBOSE && console.log(event.name); + let eventMarkets = await fetchMarkets(event.id); - eventMarkets = eventMarkets.map((market) => ({ + eventMarkets = eventMarkets.map((market: any) => ({ ...market, + // smarkets doesn't have separate urls for different markets in a single event + // we could use anchors (e.g. https://smarkets.com/event/886716/politics/uk/uk-party-leaders/next-conservative-leader#contract-collapse-9815728-control), but it's unclear if they aren't going to change slug: event.full_slug, })); - VERBOSE ? console.log("Markets fetched") : empty(); - VERBOSE ? console.log(event.id) : empty(); - VERBOSE ? console.log(eventMarkets) : empty(); + VERBOSE && console.log("Markets fetched"); + VERBOSE && console.log(event.id); + VERBOSE && console.log(eventMarkets); markets.push(...eventMarkets); - //let lastPrices = await fetchPrices(market.id) } - VERBOSE ? console.log(markets) : empty(); + VERBOSE && console.log(markets); let results = []; for (let market of markets) { - VERBOSE ? console.log("================") : empty(); - VERBOSE ? console.log("Market: ", market) : empty(); - let id = `${platformName}-${market.id}`; - let name = market.name; + VERBOSE && console.log("================"); + VERBOSE && console.log("Market: ", market); let contracts = await fetchContracts(market.id); - VERBOSE ? console.log("Contracts: ", contracts) : empty(); + VERBOSE && console.log("Contracts: ", contracts); let prices = await fetchPrices(market.id); - VERBOSE - ? console.log("Prices: ", prices["last_executed_prices"][market.id]) - : empty(); + VERBOSE && console.log("Prices: ", prices[market.id]); - let optionsObj = {}; - for (let contract of contracts["contracts"]) { - optionsObj[contract.id] = { name: contract.name }; - } - for (let price of prices["last_executed_prices"][market.id]) { + let optionsObj: { + [k: string]: QuestionOption; + } = {}; + + const contractIdToName = Object.fromEntries( + contracts.map((c) => [c.id as string, c.name as string]) + ); + + for (const price of prices[market.id]) { + const contractName = contractIdToName[price.contract_id]; + if (!contractName) { + console.warn( + `Couldn't find contract ${price.contract_id} in contracts data, skipping` + ); + continue; + } optionsObj[price.contract_id] = { - ...optionsObj[price.contract_id], + name: contractName, probability: price.last_executed_price ? Number(price.last_executed_price) - : null, + : undefined, type: "PROBABILITY", }; } - let options: any[] = Object.values(optionsObj); + let options: QuestionOption[] = Object.values(optionsObj); // monkey patch the case where there are only two options and only one has traded. if ( options.length == 2 && - options.map((option) => option.probability).includes(null) + options.map((option) => option.probability).includes(undefined) ) { - let nonNullPrice = + const nonNullPrice = options[0].probability == null ? options[1].probability : options[0].probability; - options = options.map((option) => { - let probability = option.probability; - return { - ...option, - probability: probability == null ? 100 - nonNullPrice : probability, - // yes, 100, because prices are not yet normalized. - }; - }); + + if (nonNullPrice != null) { + options = options.map((option) => { + let probability = option.probability; + return { + ...option, + probability: + probability == null ? 100 - nonNullPrice : probability, + // yes, 100, because prices are not yet normalized. + }; + }); + } } // Normalize normally - let totalValue = options + const totalValue = options .map((element) => Number(element.probability)) .reduce((a, b) => a + b, 0); @@ -148,30 +157,32 @@ export const smarkets: Platform = { ...element, probability: Number(element.probability) / totalValue, })); - VERBOSE ? console.log(options) : empty(); + VERBOSE && console.log(options); /* - if(contracts["contracts"].length == 2){ + if(contracts.length == 2){ isBinary = true - percentage = ( Number(prices["last_executed_prices"][market.id][0].last_executed_price) + (100 - Number(prices["last_executed_prices"][market.id][1].last_executed_price)) ) / 2 + percentage = ( Number(prices[market.id][0].last_executed_price) + (100 - Number(prices[market.id][1].last_executed_price)) ) / 2 percentage = Math.round(percentage)+"%" - let contractName = contracts["contracts"][0].name - name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) + let contractName = contracts[0].name + name = name+ (contractName=="Yes"?'':` (${contracts[0].name})`) } */ - let result: FetchedQuestion = { - id: id, - title: name, + const id = `${platformName}-${market.id}`; + const title = market.name; + const result: FetchedQuestion = { + id, + title, url: "https://smarkets.com/event/" + market.event_id + market.slug, description: market.description, - options: options, + options, timestamp: new Date(), qualityindicators: {}, }; - VERBOSE ? console.log(result) : empty(); + VERBOSE && console.log(result); results.push(result); } - VERBOSE ? console.log(results) : empty(); + VERBOSE && console.log(results); return results; }, calculateStars(data) { diff --git a/src/backend/platforms/wildeford.ts b/src/backend/platforms/wildeford.ts index 5dee63d..532858d 100644 --- a/src/backend/platforms/wildeford.ts +++ b/src/backend/platforms/wildeford.ts @@ -13,7 +13,7 @@ const endpoint = `https://docs.google.com/spreadsheets/d/${SHEET_ID}/edit#gid=0` // https://docs.google.com/spreadsheets/d/1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0/edit#gid=0 /* Support functions */ -const formatRow = (row) => { +const formatRow = (row: string[]) => { let colNames = [ "Prediction Date", "Prediction", @@ -23,15 +23,15 @@ const formatRow = (row) => { "Prediction Right?", "Brier Score", "Notes", - ]; - let result = {}; - row.forEach((col, i) => { + ] as const; + let result: Partial<{ [k in typeof colNames[number]]: string }> = {}; + row.forEach((col: string, i) => { result[colNames[i]] = col; }); - return result; + return result as Required; }; -async function fetchGoogleDoc(google_api_key) { +async function fetchGoogleDoc(google_api_key: string) { // https://gist.github.com/micalevisk/9bc831bd4b3e5a3f62b9810330129c59 let results = []; const doc = new GoogleSpreadsheet(SHEET_ID); @@ -41,7 +41,7 @@ async function fetchGoogleDoc(google_api_key) { console.log(">>", doc.title); const sheet = doc.sheetsByIndex[0]; - const rows = await sheet.getRows({ offset: 0 }); + const rows = await sheet.getRows(); console.log("# " + rows[0]._sheet.headerValues.join(",")); let isEnd = false; @@ -68,7 +68,9 @@ async function fetchGoogleDoc(google_api_key) { return results; } -async function processPredictions(predictions) { +async function processPredictions( + predictions: Awaited> +) { let currentPredictions = predictions.filter( (prediction) => prediction["Actual"] == "Unknown" ); @@ -101,8 +103,8 @@ async function processPredictions(predictions) { }); results = results.reverse(); - let uniqueTitles = []; - let uniqueResults = []; + let uniqueTitles: string[] = []; + let uniqueResults: FetchedQuestion[] = []; results.forEach((result) => { if (!uniqueTitles.includes(result.title)) uniqueResults.push(result); uniqueTitles.push(result.title); @@ -110,7 +112,7 @@ async function processPredictions(predictions) { return uniqueResults; } -export async function wildeford_inner(google_api_key) { +export async function wildeford_inner(google_api_key: string) { let predictions = await fetchGoogleDoc(google_api_key); return await processPredictions(predictions); } @@ -121,7 +123,7 @@ export const wildeford: Platform = { color: "#984158", async fetcher() { const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey - return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); + return (await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner)) || null; }, calculateStars(data) { let nuno = () => 3; diff --git a/src/backend/platforms/xrisk.ts b/src/backend/platforms/xrisk.ts index d5ab360..0b2e02b 100644 --- a/src/backend/platforms/xrisk.ts +++ b/src/backend/platforms/xrisk.ts @@ -14,8 +14,8 @@ export const xrisk: Platform = { let fileRaw = fs.readFileSync("./input/xrisk-questions.json", { encoding: "utf-8", }); - let results = JSON.parse(fileRaw); - results = results.map((item) => { + let parsedData = JSON.parse(fileRaw); + const results = parsedData.map((item: any) => { item.extra = item.moreoriginsdata; delete item.moreoriginsdata; return { diff --git a/src/backend/utils/algolia.ts b/src/backend/utils/algolia.ts index e50c2f6..40efeb7 100644 --- a/src/backend/utils/algolia.ts +++ b/src/backend/utils/algolia.ts @@ -44,7 +44,7 @@ export async function rebuildAlgoliaDatabase() { }) ); - if (index.exists()) { + if (await index.exists()) { console.log("Index exists"); await index.replaceAllObjects(records, { safe: true }); console.log( diff --git a/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts b/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts deleted file mode 100644 index c02315e..0000000 --- a/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* Imports */ -import fs from "fs"; - -import { prisma } from "../../database/prisma"; - -/* Definitions */ - -/* Utilities */ - -/* Support functions */ -const getQualityIndicators = (question) => - Object.entries(question.qualityindicators) - .map((entry) => `${entry[0]}: ${entry[1]}`) - .join("; "); - -/* Body */ - -const main = async () => { - let highQualityPlatforms = [ - "CSET-foretell", - "Foretold", - "Good Judgment Open", - "Metaculus", - "PredictIt", - "Rootclaim", - ]; - const json = await prisma.question.findMany({}); - console.log(json.length); - //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))] - //console.log(uniquePlatforms) - - const questionsFromGoodPlatforms = json.filter((question) => - highQualityPlatforms.includes(question.platform) - ); - const tsv = - "index\ttitle\turl\tqualityindicators\n" + - questionsFromGoodPlatforms - .map((question, index) => { - let row = `${index}\t${question.title}\t${ - question.url - }\t${getQualityIndicators(question)}`; - console.log(row); - return row; - }) - .join("\n"); - //console.log(tsv) - - // let string = JSON.stringify(json, null, 2) - fs.writeFileSync("metaforecasts.tsv", tsv); -}; -main(); diff --git a/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts b/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts deleted file mode 100644 index cbc048d..0000000 --- a/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* Imports */ -import fs from "fs"; - -import { shuffleArray } from "../../../utils"; -import { prisma } from "../../database/prisma"; - -/* Definitions */ - -/* Utilities */ - -/* Support functions */ -let getQualityIndicators = (question) => - Object.entries(question.qualityindicators) - .map((entry) => `${entry[0]}: ${entry[1]}`) - .join("; "); - -/* Body */ - -let main = async () => { - let highQualityPlatforms = ["Metaculus"]; // ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim'] - let json = await prisma.question.findMany({}); - console.log(json.length); - //let uniquePlatforms = [...new Set(json.map(question => question.platform))] - //console.log(uniquePlatforms) - - let questionsFromGoodPlatforms = json.filter((question) => - highQualityPlatforms.includes(question.platform) - ); - let questionsFromGoodPlatformsShuffled = shuffleArray( - questionsFromGoodPlatforms - ); - let tsv = - "index\ttitle\turl\tqualityindicators\n" + - questionsFromGoodPlatforms - .map((question, index) => { - let row = `${index}\t${question.title}\t${ - question.url - }\t${getQualityIndicators(question)}`; - console.log(row); - return row; - }) - .join("\n"); - //console.log(tsv) - - // let string = JSON.stringify(json, null, 2) - fs.writeFileSync("metaforecasts_metaculus_v2.tsv", tsv); -}; -main(); diff --git a/src/backend/utils/misc/process-forecasts-into-elicit.ts b/src/backend/utils/misc/process-forecasts-into-elicit.ts index 8cf28b7..737d76e 100644 --- a/src/backend/utils/misc/process-forecasts-into-elicit.ts +++ b/src/backend/utils/misc/process-forecasts-into-elicit.ts @@ -11,7 +11,7 @@ let locationData = "./data/"; // let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src async function main() { const data = await prisma.question.findMany({}); - const processDescription = (description) => { + const processDescription = (description: string | null | undefined) => { if (description == null || description == undefined || description == "") { return ""; } else { diff --git a/src/backend/utils/misc/process-forecasts-template.ts b/src/backend/utils/misc/process-forecasts-template.ts index 98323b4..04a8ea8 100644 --- a/src/backend/utils/misc/process-forecasts-template.ts +++ b/src/backend/utils/misc/process-forecasts-template.ts @@ -10,7 +10,7 @@ let rawdata = fs.readFileSync("../data/merged-questions.json", { }); let data = JSON.parse(rawdata); -let results = []; +let results: any[] = []; for (let datum of data) { // do something } diff --git a/src/backend/utils/sleep.ts b/src/backend/utils/sleep.ts new file mode 100644 index 0000000..0d7f188 --- /dev/null +++ b/src/backend/utils/sleep.ts @@ -0,0 +1,3 @@ +export function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/common/types.ts b/src/common/types.ts new file mode 100644 index 0000000..9756fb2 --- /dev/null +++ b/src/common/types.ts @@ -0,0 +1,22 @@ +import { QuestionFragment } from "../web/fragments.generated"; + +// this type is good both for backend (e.g. FetchedQuestion["options"]) and for graphql shapes +export type QuestionOption = { + name?: string; + probability?: number; + type: "PROBABILITY"; +}; + +export type FullQuestionOption = Exclude< + QuestionOption, + "name" | "probability" +> & { + name: NonNullable; + probability: NonNullable; +}; + +export const isFullQuestionOption = ( + option: QuestionOption | QuestionFragment["options"][0] +): option is FullQuestionOption => { + return option.name != null && option.probability != null; +}; diff --git a/src/pages/api/squiggle.ts b/src/pages/api/squiggle.ts index 76539a5..549e3fa 100644 --- a/src/pages/api/squiggle.ts +++ b/src/pages/api/squiggle.ts @@ -1,5 +1,6 @@ import { NextApiRequest, NextApiResponse } from "next/types"; -import { runMePlease } from "squiggle-experimental/dist/index.js"; + +import { run } from "@quri/squiggle-lang"; export default async function handler( req: NextApiRequest, @@ -24,6 +25,6 @@ $ curl -X POST -H "Content-Type: application/json" -d '{"model": "1 to 4"}' }); } else { console.log(body.model); - res.status(200).send(runMePlease(body.model)); + res.status(200).send(run(body.model)); } } diff --git a/src/web/questions/components/HistoryChart/utils.ts b/src/web/questions/components/HistoryChart/utils.ts index 3072809..5b1e920 100644 --- a/src/web/questions/components/HistoryChart/utils.ts +++ b/src/web/questions/components/HistoryChart/utils.ts @@ -1,8 +1,8 @@ import { addDays, startOfDay, startOfToday, startOfTomorrow } from "date-fns"; +import { isFullQuestionOption } from "../../../../common/types"; import { QuestionWithHistoryFragment } from "../../../fragments.generated"; import { isQuestionBinary } from "../../../utils"; -import { isFullQuestionOption } from "../../utils"; export type ChartSeries = { x: Date; y: number; name: string }[]; diff --git a/src/web/questions/components/QuestionOptions.tsx b/src/web/questions/components/QuestionOptions.tsx index c227f10..b6e77cb 100644 --- a/src/web/questions/components/QuestionOptions.tsx +++ b/src/web/questions/components/QuestionOptions.tsx @@ -1,6 +1,7 @@ +import { FullQuestionOption, isFullQuestionOption } from "../../../common/types"; import { QuestionFragment } from "../../fragments.generated"; import { isQuestionBinary } from "../../utils"; -import { formatProbability, FullQuestionOption, isFullQuestionOption } from "../utils"; +import { formatProbability } from "../utils"; const textColor = (probability: number) => { if (probability < 0.03) { diff --git a/src/web/questions/utils.ts b/src/web/questions/utils.ts index 36fa46b..f7f4f1c 100644 --- a/src/web/questions/utils.ts +++ b/src/web/questions/utils.ts @@ -8,20 +8,3 @@ export const formatProbability = (probability: number) => { : percentage.toFixed(0) + "%"; return percentageCapped; }; - -import { QuestionFragment } from "../fragments.generated"; - -export type QuestionOption = QuestionFragment["options"][0]; -export type FullQuestionOption = Exclude< - QuestionOption, - "name" | "probability" -> & { - name: NonNullable; - probability: NonNullable; -}; - -export const isFullQuestionOption = ( - option: QuestionOption -): option is FullQuestionOption => { - return option.name != null && option.probability != null; -}; diff --git a/tsconfig.json b/tsconfig.json index 246f41d..a38329d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,7 @@ "dom.iterable", "esnext" ], - "strict": false, + "strict": true, "noEmit": true, "incremental": true, "moduleResolution": "node", From fcbc627d1dce0a70fcd956e16964effbf09a6ce4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 12 May 2022 17:58:56 +0400 Subject: [PATCH 6/6] feat: improve smarkets; cli args and partial fetchers --- src/backend/flow/jobs.ts | 38 ++- src/backend/index.ts | 65 +++-- src/backend/platforms/_example.ts | 1 + src/backend/platforms/betfair.ts | 1 + src/backend/platforms/fantasyscotus.ts | 1 + src/backend/platforms/foretold.ts | 1 + src/backend/platforms/givewellopenphil.ts | 1 + src/backend/platforms/goodjudgment.ts | 1 + src/backend/platforms/goodjudgmentopen.ts | 1 + src/backend/platforms/guesstimate.ts | 1 + src/backend/platforms/index.ts | 88 +++++-- src/backend/platforms/infer.ts | 1 + src/backend/platforms/kalshi.ts | 1 + src/backend/platforms/manifold.ts | 1 + src/backend/platforms/metaculus.ts | 1 + src/backend/platforms/polymarket.ts | 1 + src/backend/platforms/predictit.ts | 1 + src/backend/platforms/rootclaim.ts | 1 + src/backend/platforms/smarkets.ts | 302 ++++++++++++---------- src/backend/platforms/wildeford.ts | 1 + src/backend/platforms/xrisk.ts | 1 + 21 files changed, 311 insertions(+), 199 deletions(-) diff --git a/src/backend/flow/jobs.ts b/src/backend/flow/jobs.ts index 099e3c6..58d289c 100644 --- a/src/backend/flow/jobs.ts +++ b/src/backend/flow/jobs.ts @@ -4,18 +4,20 @@ import { platforms, processPlatform } from "../platforms"; import { rebuildAlgoliaDatabase } from "../utils/algolia"; import { sleep } from "../utils/sleep"; -interface Job { +interface Job { name: string; message: string; - run: () => Promise; + args?: ArgNames[]; + run: (args?: { [k in ArgNames]: string }) => Promise; separate?: boolean; } -export const jobs: Job[] = [ +export const jobs: Job[] = [ ...platforms.map((platform) => ({ name: platform.name, message: `Download predictions from ${platform.name}`, - run: () => processPlatform(platform), + ...(platform.version === "v2" ? { args: platform.fetcherArgs } : {}), + run: (args: any) => processPlatform(platform, args), })), { name: "algolia", @@ -35,27 +37,39 @@ export const jobs: Job[] = [ }, ]; -async function tryCatchTryAgain(fun: () => Promise) { +async function tryCatchTryAgain( + fun: (args: T) => Promise, + args: T +) { try { console.log("Initial try"); - await fun(); + await fun(args); } catch (error) { sleep(10000); console.log("Second try"); console.log(error); try { - await fun(); + await fun(args); } catch (error) { console.log(error); } } } -export const executeJobByName = async (option: string) => { - const job = jobs.find((job) => job.name === option); +export const executeJobByName = async ( + jobName: string, + jobArgs: { [k: string]: string } = {} +) => { + const job = jobs.find((job) => job.name === jobName); if (!job) { - console.log(`Error, job ${option} not found`); - } else { - await tryCatchTryAgain(job.run); + console.log(`Error, job ${jobName} not found`); + return; } + for (const key of Object.keys(jobArgs)) { + if (!job.args || job.args.indexOf(key) < 0) { + throw new Error(`Job ${jobName} doesn't accept ${key} argument`); + } + } + + await tryCatchTryAgain(job.run, jobArgs); }; diff --git a/src/backend/index.ts b/src/backend/index.ts index bb4234d..e6aea4c 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -24,31 +24,54 @@ const generateWhatToDoMessage = () => { const whattodoMessage = generateWhatToDoMessage(); -/* BODY */ -const commandLineUtility = async () => { - const pickOption = async () => { - if (process.argv.length === 3) { - return process.argv[2]; // e.g., npm run cli polymarket - } +const askForJobName = async () => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, + const question = (query: string) => { + return new Promise((resolve: (s: string) => void) => { + rl.question(query, resolve); }); - - const question = (query: string) => { - return new Promise((resolve: (s: string) => void) => { - rl.question(query, resolve); - }); - }; - - const answer = await question(whattodoMessage); - rl.close(); - - return answer; }; - await executeJobByName(await pickOption()); + const answer = await question(whattodoMessage); + rl.close(); + + return answer; +}; + +const pickJob = async (): Promise<[string, { [k: string]: string }]> => { + if (process.argv.length < 3) { + const jobName = await askForJobName(); + return [jobName, {}]; // e.g., npm run cli polymarket + } + + const jobName = process.argv[2]; + if ((process.argv.length - 3) % 2) { + throw new Error("Number of extra arguments must be even"); + } + + const args: { [k: string]: string } = {}; + for (let i = 3; i < process.argv.length; i += 2) { + let argName = process.argv[i]; + const argValue = process.argv[i + 1]; + if (argName.slice(0, 2) !== "--") { + throw new Error(`${argName} should start with --`); + } + argName = argName.slice(2); + args[argName] = argValue; + } + + return [jobName, args]; +}; + +/* BODY */ +const commandLineUtility = async () => { + const [jobName, jobArgs] = await pickJob(); + + await executeJobByName(jobName, jobArgs); process.exit(); }; diff --git a/src/backend/platforms/_example.ts b/src/backend/platforms/_example.ts index 336bbc9..8d28226 100644 --- a/src/backend/platforms/_example.ts +++ b/src/backend/platforms/_example.ts @@ -59,6 +59,7 @@ export const example: Platform = { name: platformName, label: "Example platform", color: "#ff0000", + version: "v1", async fetcher() { let data = await fetchData(); let results = await processPredictions(data); // somehow needed diff --git a/src/backend/platforms/betfair.ts b/src/backend/platforms/betfair.ts index d9495a1..3aa8098 100644 --- a/src/backend/platforms/betfair.ts +++ b/src/backend/platforms/betfair.ts @@ -141,6 +141,7 @@ export const betfair: Platform = { name: platformName, label: "Betfair", color: "#3d674a", + version: "v1", async fetcher() { const data = await fetchPredictions(); const results = await processPredictions(data); diff --git a/src/backend/platforms/fantasyscotus.ts b/src/backend/platforms/fantasyscotus.ts index 8181afe..522085c 100644 --- a/src/backend/platforms/fantasyscotus.ts +++ b/src/backend/platforms/fantasyscotus.ts @@ -113,6 +113,7 @@ export const fantasyscotus: Platform = { name: platformName, label: "FantasySCOTUS", color: "#231149", + version: "v1", async fetcher() { let rawData = await fetchData(); let results = await processData(rawData); diff --git a/src/backend/platforms/foretold.ts b/src/backend/platforms/foretold.ts index 55692e3..0f36dea 100644 --- a/src/backend/platforms/foretold.ts +++ b/src/backend/platforms/foretold.ts @@ -62,6 +62,7 @@ export const foretold: Platform = { name: platformName, label: "Foretold", color: "#62520b", + version: "v1", async fetcher() { let results: FetchedQuestion[] = []; for (let community of highQualityCommunities) { diff --git a/src/backend/platforms/givewellopenphil.ts b/src/backend/platforms/givewellopenphil.ts index 48db8ca..b3e667e 100644 --- a/src/backend/platforms/givewellopenphil.ts +++ b/src/backend/platforms/givewellopenphil.ts @@ -68,6 +68,7 @@ export const givewellopenphil: Platform = { name: platformName, label: "GiveWell/OpenPhilanthropy", color: "#32407e", + version: "v1", async fetcher() { // main1() return; // not necessary to refill the DB every time diff --git a/src/backend/platforms/goodjudgment.ts b/src/backend/platforms/goodjudgment.ts index be78ebe..91742c5 100644 --- a/src/backend/platforms/goodjudgment.ts +++ b/src/backend/platforms/goodjudgment.ts @@ -15,6 +15,7 @@ export const goodjudgment: Platform = { name: platformName, label: "Good Judgment", color: "#7d4f1b", + version: "v1", async fetcher() { // Proxy fuckery // let proxy; diff --git a/src/backend/platforms/goodjudgmentopen.ts b/src/backend/platforms/goodjudgmentopen.ts index a22fbea..cf17e72 100644 --- a/src/backend/platforms/goodjudgmentopen.ts +++ b/src/backend/platforms/goodjudgmentopen.ts @@ -231,6 +231,7 @@ export const goodjudgmentopen: Platform = { name: platformName, label: "Good Judgment Open", color: "#002455", + version: "v1", async fetcher() { let cookie = process.env.GOODJUDGMENTOPENCOOKIE; return (await applyIfSecretExists(cookie, goodjudgmentopen_inner)) || null; diff --git a/src/backend/platforms/guesstimate.ts b/src/backend/platforms/guesstimate.ts index 432a5fd..becb351 100644 --- a/src/backend/platforms/guesstimate.ts +++ b/src/backend/platforms/guesstimate.ts @@ -92,6 +92,7 @@ export const guesstimate: Platform & { label: "Guesstimate", color: "#223900", search, + version: "v1", fetchQuestion, calculateStars: (q) => (q.description.length > 250 ? 2 : 1), }; diff --git a/src/backend/platforms/index.ts b/src/backend/platforms/index.ts index 87c6139..cba1a58 100644 --- a/src/backend/platforms/index.ts +++ b/src/backend/platforms/index.ts @@ -51,25 +51,42 @@ export type FetchedQuestion = Omit< }; // fetcher should return null if platform failed to fetch questions for some reason -export type PlatformFetcher = () => Promise; +type PlatformFetcherV1 = () => Promise; -export interface Platform { +type PlatformFetcherV2Result = { + questions: FetchedQuestion[]; + // if partial is true then we won't cleanup old questions from the database; this is useful when manually invoking a fetcher with arguments for updating a single question + partial: boolean; +} | null; + +type PlatformFetcherV2 = (opts: { + args?: { [k in ArgNames]: string }; +}) => Promise; + +export type PlatformFetcher = + | PlatformFetcherV1 + | PlatformFetcherV2; + +// using "" as ArgNames default is technically incorrect, but shouldn't cause any real issues +// (I couldn't find a better solution for signifying an empty value, though there probably is one) +export type Platform = { name: string; // short name for ids and `platform` db column, e.g. "xrisk" label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates" color: string; // used on frontend - fetcher?: PlatformFetcher; calculateStars: (question: FetchedQuestion) => number; -} +} & ( + | { + version: "v1"; + fetcher?: PlatformFetcherV1; + } + | { + version: "v2"; + fetcherArgs?: ArgNames[]; + fetcher?: PlatformFetcherV2; + } +); -// draft for the future callback-based streaming/chunking API: -// interface FetchOptions { -// since?: string; // some kind of cursor, Date object or opaque string? -// save: (questions: Question[]) => Promise; -// } - -// export type PlatformFetcher = (options: FetchOptions) => Promise; - -export const platforms: Platform[] = [ +export const platforms: Platform[] = [ betfair, fantasyscotus, foretold, @@ -104,7 +121,7 @@ type PreparedQuestion = Omit< export const prepareQuestion = ( q: FetchedQuestion, - platform: Platform + platform: Platform ): PreparedQuestion => { return { extra: {}, @@ -118,12 +135,26 @@ export const prepareQuestion = ( }; }; -export const processPlatform = async (platform: Platform) => { +export const processPlatform = async ( + platform: Platform, + args?: { [k in T]: string } +) => { if (!platform.fetcher) { console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`); return; } - const fetchedQuestions = await platform.fetcher(); + const result = + platform.version === "v1" + ? { questions: await platform.fetcher(), partial: false } // this is not exactly PlatformFetcherV2Result, since `questions` can be null + : await platform.fetcher({ args }); + + if (!result) { + console.log(`Platform ${platform.name} didn't return any results`); + return; + } + + const { questions: fetchedQuestions, partial } = result; + if (!fetchedQuestions || !fetchedQuestions.length) { console.log(`Platform ${platform.name} didn't return any results`); return; @@ -154,24 +185,32 @@ export const processPlatform = async (platform: Platform) => { } } + const stats: { created?: number; updated?: number; deleted?: number } = {}; + await prisma.question.createMany({ data: createdQuestions, }); + stats.created = createdQuestions.length; for (const q of updatedQuestions) { await prisma.question.update({ where: { id: q.id }, data: q, }); + stats.updated ??= 0; + stats.updated++; } - await prisma.question.deleteMany({ - where: { - id: { - in: deletedIds, + if (!partial) { + await prisma.question.deleteMany({ + where: { + id: { + in: deletedIds, + }, }, - }, - }); + }); + stats.deleted = deletedIds.length; + } await prisma.history.createMany({ data: [...createdQuestions, ...updatedQuestions].map((q) => ({ @@ -181,7 +220,10 @@ export const processPlatform = async (platform: Platform) => { }); console.log( - `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created` + "Done, " + + Object.entries(stats) + .map(([k, v]) => `${v} ${k}`) + .join(", ") ); }; diff --git a/src/backend/platforms/infer.ts b/src/backend/platforms/infer.ts index 9d22048..5658368 100644 --- a/src/backend/platforms/infer.ts +++ b/src/backend/platforms/infer.ts @@ -230,6 +230,7 @@ export const infer: Platform = { name: platformName, label: "Infer", color: "#223900", + version: "v1", async fetcher() { let cookie = process.env.INFER_COOKIE; return (await applyIfSecretExists(cookie, infer_inner)) || null; diff --git a/src/backend/platforms/kalshi.ts b/src/backend/platforms/kalshi.ts index 7b76686..3d77de3 100644 --- a/src/backend/platforms/kalshi.ts +++ b/src/backend/platforms/kalshi.ts @@ -68,6 +68,7 @@ export const kalshi: Platform = { name: platformName, label: "Kalshi", color: "#615691", + version: "v1", fetcher: async function () { let markets = await fetchAllMarkets(); return await processMarkets(markets); diff --git a/src/backend/platforms/manifold.ts b/src/backend/platforms/manifold.ts index 6a035a4..fe8a140 100644 --- a/src/backend/platforms/manifold.ts +++ b/src/backend/platforms/manifold.ts @@ -88,6 +88,7 @@ export const manifold: Platform = { name: platformName, label: "Manifold Markets", color: "#793466", + version: "v1", async fetcher() { let data = await fetchData(); let results = processPredictions(data); // somehow needed diff --git a/src/backend/platforms/metaculus.ts b/src/backend/platforms/metaculus.ts index fc14057..710f3d3 100644 --- a/src/backend/platforms/metaculus.ts +++ b/src/backend/platforms/metaculus.ts @@ -98,6 +98,7 @@ export const metaculus: Platform = { name: platformName, label: "Metaculus", color: "#006669", + version: "v1", async fetcher() { // let metaculusQuestionsInit = await fetchMetaculusQuestions(1) // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20) diff --git a/src/backend/platforms/polymarket.ts b/src/backend/platforms/polymarket.ts index b263050..1732d14 100644 --- a/src/backend/platforms/polymarket.ts +++ b/src/backend/platforms/polymarket.ts @@ -67,6 +67,7 @@ export const polymarket: Platform = { name: platformName, label: "PolyMarket", color: "#00314e", + version: "v1", async fetcher() { let results: FetchedQuestion[] = []; let webpageEndpointData = await fetchAllContractInfo(); diff --git a/src/backend/platforms/predictit.ts b/src/backend/platforms/predictit.ts index 6b94956..1186707 100644 --- a/src/backend/platforms/predictit.ts +++ b/src/backend/platforms/predictit.ts @@ -40,6 +40,7 @@ export const predictit: Platform = { name: platformName, label: "PredictIt", color: "#460c00", + version: "v1", async fetcher() { let markets = await fetchmarkets(); let marketVolumes = await fetchmarketvolumes(); diff --git a/src/backend/platforms/rootclaim.ts b/src/backend/platforms/rootclaim.ts index 4d58d4a..cde52b8 100644 --- a/src/backend/platforms/rootclaim.ts +++ b/src/backend/platforms/rootclaim.ts @@ -48,6 +48,7 @@ export const rootclaim: Platform = { name: platformName, label: "Rootclaim", color: "#0d1624", + version: "v1", async fetcher() { const claims = await fetchAllRootclaims(); const results: FetchedQuestion[] = []; diff --git a/src/backend/platforms/smarkets.ts b/src/backend/platforms/smarkets.ts index 1d3fa19..b61f952 100644 --- a/src/backend/platforms/smarkets.ts +++ b/src/backend/platforms/smarkets.ts @@ -6,23 +6,45 @@ import { FetchedQuestion, Platform } from "./"; /* Definitions */ const platformName = "smarkets"; -let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; -let VERBOSE = false; +const apiEndpoint = "https://api.smarkets.com/v3"; // documented at https://docs.smarkets.com/ + +type Context = { + verbose: boolean; +}; /* Support functions */ -async function fetchEvents(url: string) { - const response = await axios({ - url: htmlEndPointEntrance + url, - method: "GET", - }).then((res) => res.data); - VERBOSE && console.log(response); - return response; +async function fetchEvents(ctx: Context) { + let queryString = + "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; + + let events = []; + while (queryString) { + const data = await axios({ + url: `${apiEndpoint}/events/${queryString}`, + method: "GET", + }).then((res) => res.data); + + events.push(...data.events); + queryString = data.pagination.next_page; + } + ctx.verbose && console.log(events); + + return events; } -async function fetchMarkets(eventid: string) { +async function fetchSingleEvent(id: string, ctx: Context) { + const events = await fetchEvents(ctx); + const event = events.find((event) => event.id === id); + if (!event) { + throw new Error(`Event ${id} not found`); + } + return event; +} + +async function fetchMarkets(eventId: string) { const response = await axios({ - url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, + url: `${apiEndpoint}/events/${eventId}/markets/`, method: "GET", }) .then((res) => res.data) @@ -30,12 +52,12 @@ async function fetchMarkets(eventid: string) { return response; } -async function fetchContracts(marketid: string) { +async function fetchContracts(marketId: string, ctx: Context) { const response = await axios({ - url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, + url: `${apiEndpoint}/markets/${marketId}/contracts/?include_hidden=true`, method: "GET", }).then((res) => res.data); - VERBOSE && console.log(response); + ctx.verbose && console.log(response); if (!(response.contracts instanceof Array)) { throw new Error("Invalid response while fetching contracts"); @@ -43,154 +65,148 @@ async function fetchContracts(marketid: string) { return response.contracts as any[]; } -async function fetchPrices(marketid: string) { +async function fetchPrices(marketId: string, ctx: Context) { const response = await axios({ - url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, + url: `https://api.smarkets.com/v3/markets/${marketId}/last_executed_prices/`, method: "GET", }).then((res) => res.data); - VERBOSE && console.log(response); + ctx.verbose && console.log(response); if (!response.last_executed_prices) { throw new Error("Invalid response while fetching prices"); } return response.last_executed_prices; } -export const smarkets: Platform = { +async function processEventMarkets(event: any, ctx: Context) { + ctx.verbose && console.log(Date.now()); + ctx.verbose && console.log(event.name); + + let markets = await fetchMarkets(event.id); + markets = markets.map((market: any) => ({ + ...market, + // smarkets doesn't have separate urls for different markets in a single event + // we could use anchors (e.g. https://smarkets.com/event/886716/politics/uk/uk-party-leaders/next-conservative-leader#contract-collapse-9815728-control), but it's unclear if they aren't going to change + slug: event.full_slug, + })); + ctx.verbose && console.log(`Markets for ${event.id} fetched`); + ctx.verbose && console.log(markets); + + let results: FetchedQuestion[] = []; + for (const market of markets) { + ctx.verbose && console.log("================"); + ctx.verbose && console.log("Market:", market); + + const contracts = await fetchContracts(market.id, ctx); + ctx.verbose && console.log("Contracts:", contracts); + const prices = await fetchPrices(market.id, ctx); + ctx.verbose && console.log("Prices:", prices[market.id]); + + let optionsObj: { + [k: string]: QuestionOption; + } = {}; + + const contractsById = Object.fromEntries( + contracts.map((c) => [c.id as string, c]) + ); + + for (const price of prices[market.id]) { + const contract = contractsById[price.contract_id]; + if (!contract) { + console.warn( + `Couldn't find contract ${price.contract_id} in contracts data for ${market.id}, event ${market.event_id}, skipping` + ); + continue; + } + optionsObj[price.contract_id] = { + name: contract.name, + probability: contract.hidden ? 0 : Number(price.last_executed_price), + type: "PROBABILITY", + }; + } + let options: QuestionOption[] = Object.values(optionsObj); + ctx.verbose && console.log("Options before patching:", options); + + // monkey patch the case where there are only two options and only one has traded. + if ( + options.length === 2 && + options.map((option) => option.probability).includes(0) + ) { + const nonNullPrice = options[0].probability || options[1].probability; + + if (nonNullPrice) { + options = options.map((option) => { + return { + ...option, + probability: option.probability || 100 - nonNullPrice, + // yes, 100, because prices are not yet normalized. + }; + }); + } + } + ctx.verbose && console.log("Options after patching:", options); + + // Normalize normally + const totalValue = options + .map((element) => Number(element.probability)) + .reduce((a, b) => a + b, 0); + + options = options.map((element) => ({ + ...element, + probability: Number(element.probability) / totalValue, + })); + ctx.verbose && console.log("Normalized options:", options); + + const result: FetchedQuestion = { + id: `${platformName}-${market.id}`, + title: market.name, + url: "https://smarkets.com/event/" + market.event_id + market.slug, + description: market.description, + options, + timestamp: new Date(), + qualityindicators: {}, + }; + ctx.verbose && console.log(result); + results.push(result); + } + return results; +} + +export const smarkets: Platform<"eventId" | "verbose"> = { name: platformName, label: "Smarkets", color: "#6f5b41", - async fetcher() { - let htmlPath = - "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; + version: "v2", + fetcherArgs: ["eventId", "verbose"], + async fetcher(opts) { + const ctx = { + verbose: Boolean(opts.args?.verbose) || false, + }; - let events = []; - while (htmlPath) { - const data = await fetchEvents(htmlPath); - events.push(...data.events); - htmlPath = data.pagination.next_page; + let events: any[] = []; + let partial = true; + if (opts.args?.eventId) { + events = [await fetchSingleEvent(opts.args.eventId, ctx)]; + } else { + events = await fetchEvents(ctx); + partial = false; } - VERBOSE && console.log(events); - let markets = []; + let results: FetchedQuestion[] = []; for (const event of events) { - VERBOSE && console.log(Date.now()); - VERBOSE && console.log(event.name); - - let eventMarkets = await fetchMarkets(event.id); - eventMarkets = eventMarkets.map((market: any) => ({ - ...market, - // smarkets doesn't have separate urls for different markets in a single event - // we could use anchors (e.g. https://smarkets.com/event/886716/politics/uk/uk-party-leaders/next-conservative-leader#contract-collapse-9815728-control), but it's unclear if they aren't going to change - slug: event.full_slug, - })); - VERBOSE && console.log("Markets fetched"); - VERBOSE && console.log(event.id); - VERBOSE && console.log(eventMarkets); - markets.push(...eventMarkets); + const eventResults = await processEventMarkets(event, ctx); + results.push(...eventResults); } - VERBOSE && console.log(markets); - - let results = []; - for (let market of markets) { - VERBOSE && console.log("================"); - VERBOSE && console.log("Market: ", market); - - let contracts = await fetchContracts(market.id); - VERBOSE && console.log("Contracts: ", contracts); - let prices = await fetchPrices(market.id); - VERBOSE && console.log("Prices: ", prices[market.id]); - - let optionsObj: { - [k: string]: QuestionOption; - } = {}; - - const contractIdToName = Object.fromEntries( - contracts.map((c) => [c.id as string, c.name as string]) - ); - - for (const price of prices[market.id]) { - const contractName = contractIdToName[price.contract_id]; - if (!contractName) { - console.warn( - `Couldn't find contract ${price.contract_id} in contracts data, skipping` - ); - continue; - } - optionsObj[price.contract_id] = { - name: contractName, - probability: price.last_executed_price - ? Number(price.last_executed_price) - : undefined, - type: "PROBABILITY", - }; - } - let options: QuestionOption[] = Object.values(optionsObj); - // monkey patch the case where there are only two options and only one has traded. - if ( - options.length == 2 && - options.map((option) => option.probability).includes(undefined) - ) { - const nonNullPrice = - options[0].probability == null - ? options[1].probability - : options[0].probability; - - if (nonNullPrice != null) { - options = options.map((option) => { - let probability = option.probability; - return { - ...option, - probability: - probability == null ? 100 - nonNullPrice : probability, - // yes, 100, because prices are not yet normalized. - }; - }); - } - } - - // Normalize normally - const totalValue = options - .map((element) => Number(element.probability)) - .reduce((a, b) => a + b, 0); - - options = options.map((element) => ({ - ...element, - probability: Number(element.probability) / totalValue, - })); - VERBOSE && console.log(options); - - /* - if(contracts.length == 2){ - isBinary = true - percentage = ( Number(prices[market.id][0].last_executed_price) + (100 - Number(prices[market.id][1].last_executed_price)) ) / 2 - percentage = Math.round(percentage)+"%" - let contractName = contracts[0].name - name = name+ (contractName=="Yes"?'':` (${contracts[0].name})`) - } - */ - const id = `${platformName}-${market.id}`; - const title = market.name; - const result: FetchedQuestion = { - id, - title, - url: "https://smarkets.com/event/" + market.event_id + market.slug, - description: market.description, - options, - timestamp: new Date(), - qualityindicators: {}, - }; - VERBOSE && console.log(result); - results.push(result); - } - VERBOSE && console.log(results); - return results; + return { + questions: results, + partial, + }; }, calculateStars(data) { - let nuno = () => 2; - let eli = () => null; - let misha = () => null; - let starsDecimal = average([nuno()]); //, eli(), misha()]) - let starsInteger = Math.round(starsDecimal); + const nuno = () => 2; + const eli = () => null; + const misha = () => null; + const starsDecimal = average([nuno()]); //, eli(), misha()]) + const starsInteger = Math.round(starsDecimal); return starsInteger; }, }; diff --git a/src/backend/platforms/wildeford.ts b/src/backend/platforms/wildeford.ts index 532858d..30be453 100644 --- a/src/backend/platforms/wildeford.ts +++ b/src/backend/platforms/wildeford.ts @@ -121,6 +121,7 @@ export const wildeford: Platform = { name: platformName, label: "Peter Wildeford", color: "#984158", + version: "v1", async fetcher() { const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey return (await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner)) || null; diff --git a/src/backend/platforms/xrisk.ts b/src/backend/platforms/xrisk.ts index 0b2e02b..02115ff 100644 --- a/src/backend/platforms/xrisk.ts +++ b/src/backend/platforms/xrisk.ts @@ -9,6 +9,7 @@ export const xrisk: Platform = { name: "xrisk", label: "X-risk estimates", color: "#272600", + version: "v1", async fetcher() { // return; // not necessary to refill the DB every time let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {