feat: improve smarkets; cli args and partial fetchers

src/backend/flow/jobs.ts

@@ -4,18 +4,20 @@ import { platforms, processPlatform } from "../platforms";
 import { rebuildAlgoliaDatabase } from "../utils/algolia";
 import { sleep } from "../utils/sleep";
 
-interface Job {
+interface Job<ArgNames extends string = ""> {
   name: string;
   message: string;
-  run: () => Promise<void>;
+  args?: ArgNames[];
+  run: (args?: { [k in ArgNames]: string }) => Promise<void>;
   separate?: boolean;
 }
 
-export const jobs: Job[] = [
+export const jobs: Job<string>[] = [
   ...platforms.map((platform) => ({
     name: platform.name,
     message: `Download predictions from ${platform.name}`,
-    run: () => processPlatform(platform),
+    ...(platform.version === "v2" ? { args: platform.fetcherArgs } : {}),
+    run: (args: any) => processPlatform(platform, args),
   })),
   {
     name: "algolia",
@@ -35,27 +37,39 @@ export const jobs: Job[] = [
   },
 ];
 
-async function tryCatchTryAgain(fun: () => Promise<void>) {
+async function tryCatchTryAgain<T extends object = never>(
+  fun: (args: T) => Promise<void>,
+  args: T
+) {
   try {
     console.log("Initial try");
-    await fun();
+    await fun(args);
   } catch (error) {
     sleep(10000);
     console.log("Second try");
     console.log(error);
     try {
-      await fun();
+      await fun(args);
     } catch (error) {
       console.log(error);
     }
   }
 }
 
-export const executeJobByName = async (option: string) => {
-  const job = jobs.find((job) => job.name === option);
+export const executeJobByName = async (
+  jobName: string,
+  jobArgs: { [k: string]: string } = {}
+) => {
+  const job = jobs.find((job) => job.name === jobName);
   if (!job) {
-    console.log(`Error, job ${option} not found`);
-  } else {
-    await tryCatchTryAgain(job.run);
+    console.log(`Error, job ${jobName} not found`);
+    return;
   }
+  for (const key of Object.keys(jobArgs)) {
+    if (!job.args || job.args.indexOf(key) < 0) {
+      throw new Error(`Job ${jobName} doesn't accept ${key} argument`);
+    }
+  }
+
+  await tryCatchTryAgain(job.run, jobArgs);
 };

src/backend/index.ts

@@ -24,13 +24,7 @@ const generateWhatToDoMessage = () => {
 
 const whattodoMessage = generateWhatToDoMessage();
 
-/* BODY */
-const commandLineUtility = async () => {
-  const pickOption = async () => {
-    if (process.argv.length === 3) {
-      return process.argv[2]; // e.g., npm run cli polymarket
-    }
-
+const askForJobName = async () => {
   const rl = readline.createInterface({
     input: process.stdin,
     output: process.stdout,
@@ -48,7 +42,36 @@ const commandLineUtility = async () => {
   return answer;
 };
 
-  await executeJobByName(await pickOption());
+const pickJob = async (): Promise<[string, { [k: string]: string }]> => {
+  if (process.argv.length < 3) {
+    const jobName = await askForJobName();
+    return [jobName, {}]; // e.g., npm run cli polymarket
+  }
+
+  const jobName = process.argv[2];
+  if ((process.argv.length - 3) % 2) {
+    throw new Error("Number of extra arguments must be even");
+  }
+
+  const args: { [k: string]: string } = {};
+  for (let i = 3; i < process.argv.length; i += 2) {
+    let argName = process.argv[i];
+    const argValue = process.argv[i + 1];
+    if (argName.slice(0, 2) !== "--") {
+      throw new Error(`${argName} should start with --`);
+    }
+    argName = argName.slice(2);
+    args[argName] = argValue;
+  }
+
+  return [jobName, args];
+};
+
+/* BODY */
+const commandLineUtility = async () => {
+  const [jobName, jobArgs] = await pickJob();
+
+  await executeJobByName(jobName, jobArgs);
   process.exit();
 };
 

src/backend/platforms/_example.ts

@@ -59,6 +59,7 @@ export const example: Platform = {
   name: platformName,
   label: "Example platform",
   color: "#ff0000",
+  version: "v1",
   async fetcher() {
     let data = await fetchData();
     let results = await processPredictions(data); // somehow needed

src/backend/platforms/betfair.ts

@@ -141,6 +141,7 @@ export const betfair: Platform = {
   name: platformName,
   label: "Betfair",
   color: "#3d674a",
+  version: "v1",
   async fetcher() {
     const data = await fetchPredictions();
     const results = await processPredictions(data);

src/backend/platforms/fantasyscotus.ts

@@ -113,6 +113,7 @@ export const fantasyscotus: Platform = {
   name: platformName,
   label: "FantasySCOTUS",
   color: "#231149",
+  version: "v1",
   async fetcher() {
     let rawData = await fetchData();
     let results = await processData(rawData);

src/backend/platforms/foretold.ts

@@ -62,6 +62,7 @@ export const foretold: Platform = {
   name: platformName,
   label: "Foretold",
   color: "#62520b",
+  version: "v1",
   async fetcher() {
     let results: FetchedQuestion[] = [];
     for (let community of highQualityCommunities) {

src/backend/platforms/givewellopenphil.ts

@@ -68,6 +68,7 @@ export const givewellopenphil: Platform = {
   name: platformName,
   label: "GiveWell/OpenPhilanthropy",
   color: "#32407e",
+  version: "v1",
   async fetcher() {
     // main1()
     return; // not necessary to refill the DB every time

src/backend/platforms/goodjudgment.ts

@@ -15,6 +15,7 @@ export const goodjudgment: Platform = {
   name: platformName,
   label: "Good Judgment",
   color: "#7d4f1b",
+  version: "v1",
   async fetcher() {
     // Proxy fuckery
     // let proxy;

src/backend/platforms/goodjudgmentopen.ts

@@ -231,6 +231,7 @@ export const goodjudgmentopen: Platform = {
   name: platformName,
   label: "Good Judgment Open",
   color: "#002455",
+  version: "v1",
   async fetcher() {
     let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
     return (await applyIfSecretExists(cookie, goodjudgmentopen_inner)) || null;

src/backend/platforms/guesstimate.ts

@@ -92,6 +92,7 @@ export const guesstimate: Platform & {
   label: "Guesstimate",
   color: "#223900",
   search,
+  version: "v1",
   fetchQuestion,
   calculateStars: (q) => (q.description.length > 250 ? 2 : 1),
 };

src/backend/platforms/index.ts

@@ -51,25 +51,42 @@ export type FetchedQuestion = Omit<
 };
 
 // fetcher should return null if platform failed to fetch questions for some reason
-export type PlatformFetcher = () => Promise<FetchedQuestion[] | null>;
+type PlatformFetcherV1 = () => Promise<FetchedQuestion[] | null>;
 
-export interface Platform {
+type PlatformFetcherV2Result = {
+  questions: FetchedQuestion[];
+  // if partial is true then we won't cleanup old questions from the database; this is useful when manually invoking a fetcher with arguments for updating a single question
+  partial: boolean;
+} | null;
+
+type PlatformFetcherV2<ArgNames extends string> = (opts: {
+  args?: { [k in ArgNames]: string };
+}) => Promise<PlatformFetcherV2Result>;
+
+export type PlatformFetcher<ArgNames extends string> =
+  | PlatformFetcherV1
+  | PlatformFetcherV2<ArgNames>;
+
+// using "" as ArgNames default is technically incorrect, but shouldn't cause any real issues
+// (I couldn't find a better solution for signifying an empty value, though there probably is one)
+export type Platform<ArgNames extends string = ""> = {
   name: string; // short name for ids and `platform` db column, e.g. "xrisk"
   label: string; // longer name for displaying on frontend etc., e.g. "X-risk estimates"
   color: string; // used on frontend
-  fetcher?: PlatformFetcher;
   calculateStars: (question: FetchedQuestion) => number;
+} & (
+  | {
+      version: "v1";
+      fetcher?: PlatformFetcherV1;
     }
+  | {
+      version: "v2";
+      fetcherArgs?: ArgNames[];
+      fetcher?: PlatformFetcherV2<ArgNames>;
+    }
+);
 
-// draft for the future callback-based streaming/chunking API:
-// interface FetchOptions {
-//   since?: string; // some kind of cursor, Date object or opaque string?
-//   save: (questions: Question[]) => Promise<void>;
-// }
-
-// export type PlatformFetcher = (options: FetchOptions) => Promise<void>;
-
-export const platforms: Platform[] = [
+export const platforms: Platform<string>[] = [
   betfair,
   fantasyscotus,
   foretold,
@@ -104,7 +121,7 @@ type PreparedQuestion = Omit<
 
 export const prepareQuestion = (
   q: FetchedQuestion,
-  platform: Platform
+  platform: Platform<any>
 ): PreparedQuestion => {
   return {
     extra: {},
@@ -118,12 +135,26 @@ export const prepareQuestion = (
   };
 };
 
-export const processPlatform = async (platform: Platform) => {
+export const processPlatform = async <T extends string = "">(
+  platform: Platform<T>,
+  args?: { [k in T]: string }
+) => {
   if (!platform.fetcher) {
     console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`);
     return;
   }
-  const fetchedQuestions = await platform.fetcher();
+  const result =
+    platform.version === "v1"
+      ? { questions: await platform.fetcher(), partial: false } // this is not exactly PlatformFetcherV2Result, since `questions` can be null
+      : await platform.fetcher({ args });
+
+  if (!result) {
+    console.log(`Platform ${platform.name} didn't return any results`);
+    return;
+  }
+
+  const { questions: fetchedQuestions, partial } = result;
+
   if (!fetchedQuestions || !fetchedQuestions.length) {
     console.log(`Platform ${platform.name} didn't return any results`);
     return;
@@ -154,17 +185,23 @@ export const processPlatform = async (platform: Platform) => {
     }
   }
 
+  const stats: { created?: number; updated?: number; deleted?: number } = {};
+
   await prisma.question.createMany({
     data: createdQuestions,
   });
+  stats.created = createdQuestions.length;
 
   for (const q of updatedQuestions) {
     await prisma.question.update({
       where: { id: q.id },
       data: q,
     });
+    stats.updated ??= 0;
+    stats.updated++;
   }
 
+  if (!partial) {
     await prisma.question.deleteMany({
       where: {
         id: {
@@ -172,6 +209,8 @@ export const processPlatform = async (platform: Platform) => {
         },
       },
     });
+    stats.deleted = deletedIds.length;
+  }
 
   await prisma.history.createMany({
     data: [...createdQuestions, ...updatedQuestions].map((q) => ({
@@ -181,7 +220,10 @@ export const processPlatform = async (platform: Platform) => {
   });
 
   console.log(
-    `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created`
+    "Done, " +
+      Object.entries(stats)
+        .map(([k, v]) => `${v} ${k}`)
+        .join(", ")
   );
 };
 

src/backend/platforms/infer.ts

@@ -230,6 +230,7 @@ export const infer: Platform = {
   name: platformName,
   label: "Infer",
   color: "#223900",
+  version: "v1",
   async fetcher() {
     let cookie = process.env.INFER_COOKIE;
     return (await applyIfSecretExists(cookie, infer_inner)) || null;

src/backend/platforms/kalshi.ts

@@ -68,6 +68,7 @@ export const kalshi: Platform = {
   name: platformName,
   label: "Kalshi",
   color: "#615691",
+  version: "v1",
   fetcher: async function () {
     let markets = await fetchAllMarkets();
     return await processMarkets(markets);

src/backend/platforms/manifold.ts

@@ -88,6 +88,7 @@ export const manifold: Platform = {
   name: platformName,
   label: "Manifold Markets",
   color: "#793466",
+  version: "v1",
   async fetcher() {
     let data = await fetchData();
     let results = processPredictions(data); // somehow needed

src/backend/platforms/metaculus.ts

@@ -98,6 +98,7 @@ export const metaculus: Platform = {
   name: platformName,
   label: "Metaculus",
   color: "#006669",
+  version: "v1",
   async fetcher() {
     // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
     // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)

src/backend/platforms/polymarket.ts

@@ -67,6 +67,7 @@ export const polymarket: Platform = {
   name: platformName,
   label: "PolyMarket",
   color: "#00314e",
+  version: "v1",
   async fetcher() {
     let results: FetchedQuestion[] = [];
     let webpageEndpointData = await fetchAllContractInfo();

src/backend/platforms/predictit.ts

@@ -40,6 +40,7 @@ export const predictit: Platform = {
   name: platformName,
   label: "PredictIt",
   color: "#460c00",
+  version: "v1",
   async fetcher() {
     let markets = await fetchmarkets();
     let marketVolumes = await fetchmarketvolumes();

src/backend/platforms/rootclaim.ts

@@ -48,6 +48,7 @@ export const rootclaim: Platform = {
   name: platformName,
   label: "Rootclaim",
   color: "#0d1624",
+  version: "v1",
   async fetcher() {
     const claims = await fetchAllRootclaims();
     const results: FetchedQuestion[] = [];

src/backend/platforms/smarkets.ts

@@ -6,23 +6,45 @@ import { FetchedQuestion, Platform } from "./";
 
 /* Definitions */
 const platformName = "smarkets";
-let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
-let VERBOSE = false;
+const apiEndpoint = "https://api.smarkets.com/v3"; // documented at https://docs.smarkets.com/
+
+type Context = {
+  verbose: boolean;
+};
 
 /* Support functions */
 
-async function fetchEvents(url: string) {
-  const response = await axios({
-    url: htmlEndPointEntrance + url,
+async function fetchEvents(ctx: Context) {
+  let queryString =
+    "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
+
+  let events = [];
+  while (queryString) {
+    const data = await axios({
+      url: `${apiEndpoint}/events/${queryString}`,
       method: "GET",
     }).then((res) => res.data);
-  VERBOSE && console.log(response);
-  return response;
+
+    events.push(...data.events);
+    queryString = data.pagination.next_page;
+  }
+  ctx.verbose && console.log(events);
+
+  return events;
 }
 
-async function fetchMarkets(eventid: string) {
+async function fetchSingleEvent(id: string, ctx: Context) {
+  const events = await fetchEvents(ctx);
+  const event = events.find((event) => event.id === id);
+  if (!event) {
+    throw new Error(`Event ${id} not found`);
+  }
+  return event;
+}
+
+async function fetchMarkets(eventId: string) {
   const response = await axios({
-    url: `https://api.smarkets.com/v3/events/${eventid}/markets/`,
+    url: `${apiEndpoint}/events/${eventId}/markets/`,
     method: "GET",
   })
     .then((res) => res.data)
@@ -30,12 +52,12 @@ async function fetchMarkets(eventid: string) {
   return response;
 }
 
-async function fetchContracts(marketid: string) {
+async function fetchContracts(marketId: string, ctx: Context) {
   const response = await axios({
-    url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`,
+    url: `${apiEndpoint}/markets/${marketId}/contracts/?include_hidden=true`,
     method: "GET",
   }).then((res) => res.data);
-  VERBOSE && console.log(response);
+  ctx.verbose && console.log(response);
 
   if (!(response.contracts instanceof Array)) {
     throw new Error("Invalid response while fetching contracts");
@@ -43,110 +65,85 @@ async function fetchContracts(marketid: string) {
   return response.contracts as any[];
 }
 
-async function fetchPrices(marketid: string) {
+async function fetchPrices(marketId: string, ctx: Context) {
   const response = await axios({
-    url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`,
+    url: `https://api.smarkets.com/v3/markets/${marketId}/last_executed_prices/`,
     method: "GET",
   }).then((res) => res.data);
-  VERBOSE && console.log(response);
+  ctx.verbose && console.log(response);
   if (!response.last_executed_prices) {
     throw new Error("Invalid response while fetching prices");
   }
   return response.last_executed_prices;
 }
 
-export const smarkets: Platform = {
-  name: platformName,
-  label: "Smarkets",
-  color: "#6f5b41",
-  async fetcher() {
-    let htmlPath =
-      "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50";
+async function processEventMarkets(event: any, ctx: Context) {
+  ctx.verbose && console.log(Date.now());
+  ctx.verbose && console.log(event.name);
 
-    let events = [];
-    while (htmlPath) {
-      const data = await fetchEvents(htmlPath);
-      events.push(...data.events);
-      htmlPath = data.pagination.next_page;
-    }
-    VERBOSE && console.log(events);
-
-    let markets = [];
-    for (const event of events) {
-      VERBOSE && console.log(Date.now());
-      VERBOSE && console.log(event.name);
-
-      let eventMarkets = await fetchMarkets(event.id);
-      eventMarkets = eventMarkets.map((market: any) => ({
+  let markets = await fetchMarkets(event.id);
+  markets = markets.map((market: any) => ({
     ...market,
     // smarkets doesn't have separate urls for different markets in a single event
     // we could use anchors (e.g. https://smarkets.com/event/886716/politics/uk/uk-party-leaders/next-conservative-leader#contract-collapse-9815728-control), but it's unclear if they aren't going to change
     slug: event.full_slug,
   }));
-      VERBOSE && console.log("Markets fetched");
-      VERBOSE && console.log(event.id);
-      VERBOSE && console.log(eventMarkets);
-      markets.push(...eventMarkets);
-    }
-    VERBOSE && console.log(markets);
+  ctx.verbose && console.log(`Markets for ${event.id} fetched`);
+  ctx.verbose && console.log(markets);
 
-    let results = [];
-    for (let market of markets) {
-      VERBOSE && console.log("================");
-      VERBOSE && console.log("Market: ", market);
+  let results: FetchedQuestion[] = [];
+  for (const market of markets) {
+    ctx.verbose && console.log("================");
+    ctx.verbose && console.log("Market:", market);
 
-      let contracts = await fetchContracts(market.id);
-      VERBOSE && console.log("Contracts: ", contracts);
-      let prices = await fetchPrices(market.id);
-      VERBOSE && console.log("Prices: ", prices[market.id]);
+    const contracts = await fetchContracts(market.id, ctx);
+    ctx.verbose && console.log("Contracts:", contracts);
+    const prices = await fetchPrices(market.id, ctx);
+    ctx.verbose && console.log("Prices:", prices[market.id]);
 
     let optionsObj: {
       [k: string]: QuestionOption;
     } = {};
 
-      const contractIdToName = Object.fromEntries(
-        contracts.map((c) => [c.id as string, c.name as string])
+    const contractsById = Object.fromEntries(
+      contracts.map((c) => [c.id as string, c])
     );
 
     for (const price of prices[market.id]) {
-        const contractName = contractIdToName[price.contract_id];
-        if (!contractName) {
+      const contract = contractsById[price.contract_id];
+      if (!contract) {
         console.warn(
-            `Couldn't find contract ${price.contract_id} in contracts data, skipping`
+          `Couldn't find contract ${price.contract_id} in contracts data for ${market.id}, event ${market.event_id}, skipping`
         );
         continue;
       }
       optionsObj[price.contract_id] = {
-          name: contractName,
-          probability: price.last_executed_price
-            ? Number(price.last_executed_price)
-            : undefined,
+        name: contract.name,
+        probability: contract.hidden ? 0 : Number(price.last_executed_price),
         type: "PROBABILITY",
       };
     }
     let options: QuestionOption[] = Object.values(optionsObj);
+    ctx.verbose && console.log("Options before patching:", options);
+
     // monkey patch the case where there are only two options and only one has traded.
     if (
-        options.length == 2 &&
-        options.map((option) => option.probability).includes(undefined)
+      options.length === 2 &&
+      options.map((option) => option.probability).includes(0)
     ) {
-        const nonNullPrice =
-          options[0].probability == null
-            ? options[1].probability
-            : options[0].probability;
+      const nonNullPrice = options[0].probability || options[1].probability;
 
-        if (nonNullPrice != null) {
+      if (nonNullPrice) {
         options = options.map((option) => {
-            let probability = option.probability;
           return {
             ...option,
-              probability:
-                probability == null ? 100 - nonNullPrice : probability,
+            probability: option.probability || 100 - nonNullPrice,
             // yes, 100, because prices are not yet normalized.
           };
         });
       }
     }
+    ctx.verbose && console.log("Options after patching:", options);
 
     // Normalize normally
     const totalValue = options
@@ -157,40 +154,59 @@ export const smarkets: Platform = {
       ...element,
       probability: Number(element.probability) / totalValue,
     }));
-      VERBOSE && console.log(options);
+    ctx.verbose && console.log("Normalized options:", options);
 
-      /*
-    if(contracts.length == 2){
-      isBinary = true
-      percentage = ( Number(prices[market.id][0].last_executed_price) + (100 - Number(prices[market.id][1].last_executed_price)) ) / 2
-      percentage = Math.round(percentage)+"%"
-      let contractName = contracts[0].name
-      name = name+ (contractName=="Yes"?'':` (${contracts[0].name})`)
-    }
-    */
-      const id = `${platformName}-${market.id}`;
-      const title = market.name;
     const result: FetchedQuestion = {
-        id,
-        title,
+      id: `${platformName}-${market.id}`,
+      title: market.name,
       url: "https://smarkets.com/event/" + market.event_id + market.slug,
       description: market.description,
       options,
       timestamp: new Date(),
       qualityindicators: {},
     };
-      VERBOSE && console.log(result);
+    ctx.verbose && console.log(result);
     results.push(result);
   }
-    VERBOSE && console.log(results);
   return results;
+}
+
+export const smarkets: Platform<"eventId" | "verbose"> = {
+  name: platformName,
+  label: "Smarkets",
+  color: "#6f5b41",
+  version: "v2",
+  fetcherArgs: ["eventId", "verbose"],
+  async fetcher(opts) {
+    const ctx = {
+      verbose: Boolean(opts.args?.verbose) || false,
+    };
+
+    let events: any[] = [];
+    let partial = true;
+    if (opts.args?.eventId) {
+      events = [await fetchSingleEvent(opts.args.eventId, ctx)];
+    } else {
+      events = await fetchEvents(ctx);
+      partial = false;
+    }
+
+    let results: FetchedQuestion[] = [];
+    for (const event of events) {
+      const eventResults = await processEventMarkets(event, ctx);
+      results.push(...eventResults);
+    }
+    return {
+      questions: results,
+      partial,
+    };
   },
   calculateStars(data) {
-    let nuno = () => 2;
-    let eli = () => null;
-    let misha = () => null;
-    let starsDecimal = average([nuno()]); //, eli(), misha()])
-    let starsInteger = Math.round(starsDecimal);
+    const nuno = () => 2;
+    const eli = () => null;
+    const misha = () => null;
+    const starsDecimal = average([nuno()]); //, eli(), misha()])
+    const starsInteger = Math.round(starsDecimal);
     return starsInteger;
   },
 };

src/backend/platforms/wildeford.ts

@@ -121,6 +121,7 @@ export const wildeford: Platform = {
   name: platformName,
   label: "Peter Wildeford",
   color: "#984158",
+  version: "v1",
   async fetcher() {
     const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
     return (await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner)) || null;

src/backend/platforms/xrisk.ts

@@ -9,6 +9,7 @@ export const xrisk: Platform = {
   name: "xrisk",
   label: "X-risk estimates",
   color: "#272600",
+  version: "v1",
   async fetcher() {
     // return; // not necessary to refill the DB every time
     let fileRaw = fs.readFileSync("./input/xrisk-questions.json", {