feat: Added Manifold Markets

src/index.js

@@ -15,6 +15,7 @@ import {goodjudgmentopen} from "./platforms/goodjudmentopen-fetch.js"
 import { hypermind } from "./platforms/hypermind-fetch.js"
 import { kalshi } from "./platforms/kalshi-fetch.js"
 import { ladbrokes } from "./platforms/ladbrokes-fetch.js"
+import { manifoldmarkets } from "./platforms/manifoldmarkets-fetch.js"
 import { metaculus } from "./platforms/metaculus-fetch.js"
 import { omen } from "./platforms/omen-fetch.js"
 import { polymarket } from "./platforms/polymarket-fetch.js"
@@ -30,7 +31,7 @@ import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewD
 import { doEverything, tryCatchTryAgain } from "./utils/doEverything.js"
 
 /* Support functions */
-let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
+let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
 let functionNames = functions.map(fun => fun.name)
 
 let whattodoMessage = functionNames

src/platforms/csetforetell-fetch.js

@@ -10,6 +10,8 @@ import { upsert } from "../utils/mongo-wrapper.js"
 let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
 String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
 const DEBUG_MODE = "on"// "off"
+const SLEEP_TIME_RANDOM=100//5000 // miliseconds
+const SLEEP_TIME_EXTRA=0//1000
 /* Support functions */
 
 async function fetchPage(page, cookie) {
@@ -171,27 +173,28 @@ async function csetforetell_inner(cookie) {
   while (!isEnd(response) && isSignedIn(response)) {
 
     let htmlLines = response.split("\n")
-    let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
-    //let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
+    // let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
+    let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
     // console.log(questionHrefs)
     
 
     if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
       //console.log(response)
-      console.log(h4elements)
+      console.log("questionHrefs: ")
+      console.log(questionHrefs)
     }
 
     //console.log("")
     //console.log("")
     //console.log(h4elements)
 
-    for (let h4element of h4elements) {
+    for (let questionHref of questionHrefs) {
       //console.log(h4element)
 
-      let h4elementSplit = h4element.split('"><span>')
-      let url = h4elementSplit[0].split('<a href="')[1]
-      let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "")
-      await sleep(1000 + Math.random() * 1000) // don't be as noticeable
+      let elementSplit = questionHref.split('"><span>')
+      let url = elementSplit[0].split('<a href="')[1]
+      let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "")
+      await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
 
       try {
         let moreinfo = await fetchStats(url, cookie)
@@ -221,7 +224,7 @@ async function csetforetell_inner(cookie) {
     //i=Number(i)+1
 
     console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
-    await sleep(5000 + Math.random() * 1000) // don't be as noticeable
+    await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
 
     try {
       response = await fetchPage(i, cookie)

src/platforms/manifoldmarkets-fetch.js

@@ -0,0 +1,76 @@
+/* Imports */
+import fs from 'fs'
+import axios from "axios"
+import { calculateStars } from "../utils/stars.js"
+import { upsert } from "../utils/mongo-wrapper.js"
+
+/* Definitions */
+let endpoint = 'https://us-central1-mantic-markets.cloudfunctions.net/markets'
+
+/* Support functions */
+
+async function fetchData() {
+    let response = await axios({
+        url: endpoint,
+        method: 'GET',
+        headers: ({
+            'Content-Type': 'text/html',
+        }),
+    })
+        .then(response => response.data)
+    // console.log(response)
+    return response
+}
+
+async function processPredictions(predictions) {
+    let results = await predictions.map(prediction => {
+        let probability = prediction.probability
+        let options = [
+            {
+                "name": "Yes",
+                "probability": probability,
+                "type": "PROBABILITY"
+            },
+            {
+                "name": "No",
+                "probability": 1 - probability,
+                "type": "PROBABILITY"
+            }
+        ]
+        let result = ({
+            "title": prediction.question,
+            "url": prediction.url,
+            "platform": "Manifold Markets",
+            "description": prediction.description,
+            "options": options,
+            "timestamp": new Date().toISOString(),
+            "qualityindicators": {
+                "stars": calculateStars("Manifold Markets", ({
+                    volume7days: prediction.volume7days,
+                    volume24Hours: prediction.volume24Hours,
+                    liquidity: prediction.pool
+                })),
+                "createdTime": prediction.createdTime,
+                "volume7Days": prediction.volume7Days,
+                "volume24Hours": prediction.volume24Hours,
+                "liquidity": prediction.pool
+            }
+        })
+        return result
+    })
+    let unresolvedResults = results.filter(result => !result.isResolved)
+    return unresolvedResults //resultsProcessed
+}
+
+/* Body */
+
+export async function manifoldmarkets() {
+    let data = await fetchData()
+    let results = await processPredictions(data) // somehow needed
+    // console.log(results)
+    // let string = JSON.stringify(results, null, 2)
+    // fs.writeFileSync('polyprediction-questions.json', string);
+    await upsert(results, "manifoldmarkets-questions")
+    console.log("Done")
+}
+//manifoldmarkets()

src/utils/doEverything.js

@@ -11,6 +11,7 @@ import {goodjudgmentopen} from "../platforms/goodjudmentopen-fetch.js"
 import { hypermind } from "../platforms/hypermind-fetch.js"
 import { kalshi } from "../platforms/kalshi-fetch.js"
 import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
+import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
 import { metaculus } from "../platforms/metaculus-fetch.js"
 import { omen } from "../platforms/omen-fetch.js"
 import { polymarket } from "../platforms/polymarket-fetch.js"
@@ -46,7 +47,7 @@ export async function tryCatchTryAgain (fun) {
 }
 
 export async function doEverything() {
-    let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] 
+    let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
     // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
 
     console.log("")

src/utils/downloadFrontpage.js

@@ -2,7 +2,7 @@ import fs from "fs"
 
 import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
 
-let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json'
+let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-backend/data/frontpage.json'
 let shuffle = (array) => {
   // https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
   let currentIndex = array.length,  randomIndex;

src/utils/mergeEverything.js

@@ -1,14 +1,14 @@
 import { mongoRead, upsert } from "./mongo-wrapper.js"
 
 /* Merge everything */
-let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
+let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
 let suffix = "-questions"
 
 export async function mergeEverythingInner(){
   let merged = []
   for(let set of sets){
     let json = await mongoRead(set+suffix)
-    console.log(`${set} has ${json.length} questions`)
+    console.log(`${set} has ${json.length} questions\n`)
     merged = merged.concat(json)
   }
   let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")}))

src/utils/stars.js

@@ -156,6 +156,15 @@ function calculateStarsLadbrokes(data) {
 	return starsInteger
 }
 
+function calculateStarsManifoldMarkets(data) {
+	let nuno = data => 2
+	let eli = (data) => null
+	let misha = (data) => null
+	let starsDecimal = average([nuno(data)]) //, eli(data), misha(data)])
+	let starsInteger = Math.round(starsDecimal)
+	return starsInteger
+}
+
 function calculateStarsMetaculus(data) {
 	let nuno = data => data.numforecasts > 300 ? 4 : (data.numforecasts > 100 ? 3 : 2)
 	let eli = (data) => 3
@@ -278,6 +287,9 @@ export function calculateStars(platform, data) {
 		case "Ladbrokes":
 			stars = calculateStarsLadbrokes(data)
 			break;
+		case "Manifold Markets":
+			stars = calculateStarsManifoldMarkets(data)
+			break;
 		case "Metaculus":
 			stars = calculateStarsMetaculus(data)
 			break;