feat: Added Kalshi

Also some fixes, and I think some functionality to get randomly selected forecasts

done.txt

@@ -0,0 +1 @@
+done

src/index.js

@@ -12,6 +12,7 @@ import {foretold} from "./platforms/foretold-fetch.js"
 import {goodjudgment} from "./platforms/goodjudgment-fetch.js"
 import {goodjudgmentopen} from "./platforms/goodjudmentopen-fetch.js"
 import {hypermind} from "./platforms/hypermind-fetch.js"
+import {kalshi} from "./platforms/kalshi-fetch.js"
 import {ladbrokes} from "./platforms/ladbrokes-fetch.js"
 import {metaculus} from "./platforms/metaculus-fetch.js"
 import {omen} from "./platforms/omen-fetch.js"
@@ -26,7 +27,7 @@ import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewD
 import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
 
 /* Support functions */
-let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything]
+let functions = [astralcodexten, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything]
 let functionNames =  functions.map(fun => fun.name)
 
 let whattodoMessage = functionNames

src/platforms/kalshi-fetch.js

@@ -0,0 +1,70 @@
+/* Imports */
+import fs from 'fs'
+import axios from "axios"
+import { calculateStars } from "../utils/stars.js"
+import {upsert} from "../utils/mongo-wrapper.js"
+
+/* Definitions */
+let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"//"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
+
+/* Support functions
+async function fetchAllContractInfo(){ // for info which the polymarket graphql API
+  let data = fs.readFileSync("./data/polymarket-contract-list.json")
+  let response = JSON.parse(data)
+  return response
+}
+ */
+async function fetchAllMarkets() { // for info which the polymarket graphql API
+  let response = await axios.get('https://trading-api.kalshi.com/v1/cached/markets/').then(response => response.data.markets)
+  // console.log(response)
+  return response
+}
+
+
+async function processMarkets(markets) {
+  let results = await markets.map(market => {
+    let probability = market.last_price/100
+    let options = [
+      {
+        "name": "Yes",
+        "probability": probability,
+        "type": "PROBABILITY"
+      },
+      {
+        "name": "No",
+        "probability": 1 - probability,
+        "type": "PROBABILITY"
+      }
+    ]
+    let result = ({
+      "title": market.title,
+      "url": `https://kalshi.com`,
+      "platform": "Kalshi",
+      "description": `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
+      "options": options,
+      "timestamp": new Date().toISOString(),
+      "qualityindicators": {
+          "stars": calculateStars("Kalshi", ({volume: market.volume, interest: market.open_interest})),
+          "yes_bid": market.yes_bid,
+          "yes_ask": market.yes_ask,
+          "spread": Math.abs(market.yes_bid-market.yes_ask),
+          "volume": market.volume,
+          "open_interest": market.open_interest
+      }
+    })
+    return result
+  })
+  return results //resultsProcessed
+}
+
+/* Body */
+export async function kalshi() {
+  let markets = await fetchAllMarkets()
+  let results = await processMarkets(markets) // somehow needed
+  // console.log(results)
+  // let string = JSON.stringify(results, null, 2)
+  // fs.writeFileSync('polymarket-questions.json', string);
+  await upsert(results, "kalshi-questions")
+  console.log("Done")
+}
+// kalshi()

src/utils/doEverything.js

@@ -8,6 +8,7 @@ import {foretold} from "../platforms/foretold-fetch.js"
 import {goodjudgment} from "../platforms/goodjudgment-fetch.js"
 import {goodjudgmentopen} from "../platforms/goodjudmentopen-fetch.js"
 import {hypermind} from "../platforms/hypermind-fetch.js"
+import {kalshi} from "../platforms/kalshi-fetch.js"
 import {ladbrokes} from "../platforms/ladbrokes-fetch.js"
 import {metaculus} from "../platforms/metaculus-fetch.js"
 import {omen} from "../platforms/omen-fetch.js"
@@ -37,7 +38,7 @@ export async function tryCatchTryAgain (fun) {
     }
 }
 export async function doEverything(){
-    let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData] 
+    let functions = [coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData] 
     // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
 
     console.log("")

src/utils/evals/notes.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+# shuf metaforecasts.tsv > metaforecastsShuffled2.tsv

src/utils/mergeEverything.js

@@ -1,7 +1,7 @@
 import { mongoRead, upsert } from "./mongo-wrapper.js"
 
 /* Merge everything */
-let sets = ["astralcodexten","coupcast", "csetforetell", "elicit", "estimize", "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "williamhill", "xrisk"]
+let sets = ["astralcodexten","coupcast", "csetforetell", "elicit", "estimize", "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "williamhill", "xrisk"]
 let suffix = "-questions"
 
 export async function mergeEverything(){

src/utils/mongo-wrapper.js

@@ -39,7 +39,6 @@ function roughSizeOfObject( object ) {
 export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
     const url = process.env.MONGODB_URL || getCookie("mongodb");
     const client = new MongoClient(url);
-
     try {
         await client.connect();
         console.log("Connected correctly to server");
@@ -63,7 +62,6 @@ export async function upsert (contents, documentName, collectionName="metaforeca
         
         // Find one document
         const myDocument = await collection.findOne(filter);
-    
         // Print to the console
         console.log(`Updating document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(contents)} MB`)
         console.log("Sample: ")
@@ -184,5 +182,5 @@ export async function mongoGetAllElements(databaseName = "metaforecastDatabase",
     }
 
 }
-//mongoGetAllElements()
+// mongoGetAllElements()
 //mongoGetAllElements("metaforecastDatabase", "metaforecastHistory")

src/utils/pullForecastsToCSVForRating.js

@@ -0,0 +1,29 @@
+/* Imports */
+import fs from "fs"
+import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
+
+/* Definitions */
+
+/* Utilities */
+
+/* Support functions */
+
+/* Body */
+
+let main = async () => {
+  let highQualityPlatforms = ['CSET-foretell', 'Foretold', 'Good Judgment Open', 'Metaculus', 'PredictIt', 'Rootclaim']
+  let json = await mongoReadWithReadCredentials("metaforecasts")
+  console.log(json.length)
+  //let uniquePlatforms = [...new Set(json.map(forecast => forecast.platform))]
+  //console.log(uniquePlatforms)
+  
+  let forecastsFromGoodPlatforms = json.filter(forecast => highQualityPlatforms.includes(forecast.platform))
+  let tsv = "index\ttitle\turl\tstars\n"+forecastsFromGoodPlatforms
+    .map((forecast, index) => `${index}\t${forecast.title}\t${forecast.url}\t0`)
+    .join("\n")
+  //console.log(tsv)
+
+  // let string = JSON.stringify(json, null, 2)
+  fs.writeFileSync('evals/metaforecasts.tsv', tsv);
+}
+main()

src/utils/pullSuperforecastsManually.js

@@ -0,0 +1,12 @@
+/* Imports */
+import {goodjudgment} from "../platforms/goodjudgment-fetch.js"
+
+/* Definitions */
+
+/* Utilities */
+
+/* Support functions */
+
+/* Body */
+goodjudgment()
+

src/utils/pullSuperforecastsManually.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cd /home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo
+/usr/bin/node ./src/utils/pullSuperforecastsManually.js
+echo "done" > done.txt

src/utils/stars.js

@@ -115,6 +115,22 @@ function calculateStarsHypermind(data) {
   return starsInteger
 }
 
+function calculateStarsKalshi(data) {
+  let nuno = data => data.interest > 1000 && data.volume > 1000 ? 4 : 3
+  let eli = (data) => data.interest > 10000 ? 5 : 4
+  let misha = (data) => 4
+  let starsDecimal = average([nuno(data), eli(data), misha(data)])
+  // Substract 1 star if probability is above 90% or below 10%
+  if(data.option &&
+    (data.option.probability < 0.1 || data.option.probability > 0.9)
+  ){
+    starsDecimal = starsDecimal - 1
+  }
+  
+  let starsInteger = Math.round(starsDecimal)
+  return starsInteger
+}
+
 function calculateStarsLadbrokes(data) {
   let nuno = data => 2
   let eli = (data) => null
@@ -227,7 +243,10 @@ export function calculateStars(platform, data) {
     case "Hypermind":
       stars = calculateStarsHypermind(data)
       break;
-    case "Hypermind":
+    case "Kalshi":
+      stars = calculateStarsKalshi(data)
+      break;
+    case "Ladbrokes":
       stars = calculateStarsLadbrokes(data)
       break;
     case "Metaculus":