feat: Added Manifold Markets
This commit is contained in:
parent
62b0f8e210
commit
08715357fd
91
src/index.js
91
src/index.js
|
|
@ -2,52 +2,53 @@
|
|||
import fs from 'fs'
|
||||
import readline from "readline"
|
||||
|
||||
import {astralcodexten} from "./platforms/astralcodexten-fetch.js"
|
||||
import {betfair} from "./platforms/betfair-fetch.js"
|
||||
import {coupcast} from "./platforms/coupcast-fetch.js"
|
||||
import {csetforetell} from "./platforms/csetforetell-fetch.js"
|
||||
import {elicit} from "./platforms/elicit-fetch.js"
|
||||
import {estimize} from "./platforms/estimize-fetch.js"
|
||||
import {fantasyscotus} from "./platforms/fantasyscotus-fetch.js"
|
||||
import {foretold} from "./platforms/foretold-fetch.js"
|
||||
import {goodjudgment} from "./platforms/goodjudgment-fetch.js"
|
||||
import {goodjudgmentopen} from "./platforms/goodjudmentopen-fetch.js"
|
||||
import {hypermind} from "./platforms/hypermind-fetch.js"
|
||||
import {kalshi} from "./platforms/kalshi-fetch.js"
|
||||
import {ladbrokes} from "./platforms/ladbrokes-fetch.js"
|
||||
import {metaculus} from "./platforms/metaculus-fetch.js"
|
||||
import {omen} from "./platforms/omen-fetch.js"
|
||||
import {polymarket} from "./platforms/polymarket-fetch.js"
|
||||
import {predictit} from "./platforms/predictit-fetch.js"
|
||||
import {rootclaim} from "./platforms/rootclaim-fetch.js"
|
||||
import {smarkets} from "./platforms/smarkets-fetch.js"
|
||||
import {wildeford} from "./platforms/wildeford-fetch.js"
|
||||
import {williamhill} from "./platforms/williamhill-fetch.js"
|
||||
import {mergeEverything} from "./utils/mergeEverything.js"
|
||||
import {updateHistory} from "./utils/history/updateHistory.js"
|
||||
import {rebuildAlgoliaDatabase} from "./utils/algolia.js"
|
||||
import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
|
||||
import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
|
||||
import { astralcodexten } from "./platforms/astralcodexten-fetch.js"
|
||||
import { betfair } from "./platforms/betfair-fetch.js"
|
||||
import { coupcast } from "./platforms/coupcast-fetch.js"
|
||||
import { csetforetell } from "./platforms/csetforetell-fetch.js"
|
||||
import { elicit } from "./platforms/elicit-fetch.js"
|
||||
import { estimize } from "./platforms/estimize-fetch.js"
|
||||
import { fantasyscotus } from "./platforms/fantasyscotus-fetch.js"
|
||||
import { foretold } from "./platforms/foretold-fetch.js"
|
||||
import { goodjudgment } from "./platforms/goodjudgment-fetch.js"
|
||||
import { goodjudgmentopen } from "./platforms/goodjudmentopen-fetch.js"
|
||||
import { hypermind } from "./platforms/hypermind-fetch.js"
|
||||
import { kalshi } from "./platforms/kalshi-fetch.js"
|
||||
import { ladbrokes } from "./platforms/ladbrokes-fetch.js"
|
||||
import { manifoldmarkets } from "./platforms/manifoldmarkets-fetch.js"
|
||||
import { metaculus } from "./platforms/metaculus-fetch.js"
|
||||
import { omen } from "./platforms/omen-fetch.js"
|
||||
import { polymarket } from "./platforms/polymarket-fetch.js"
|
||||
import { predictit } from "./platforms/predictit-fetch.js"
|
||||
import { rootclaim } from "./platforms/rootclaim-fetch.js"
|
||||
import { smarkets } from "./platforms/smarkets-fetch.js"
|
||||
import { wildeford } from "./platforms/wildeford-fetch.js"
|
||||
import { williamhill } from "./platforms/williamhill-fetch.js"
|
||||
import { mergeEverything } from "./utils/mergeEverything.js"
|
||||
import { updateHistory } from "./utils/history/updateHistory.js"
|
||||
import { rebuildAlgoliaDatabase } from "./utils/algolia.js"
|
||||
import { rebuildNetlifySiteWithNewData } from "./utils/rebuildNetliftySiteWithNewData.js"
|
||||
import { doEverything, tryCatchTryAgain } from "./utils/doEverything.js"
|
||||
|
||||
/* Support functions */
|
||||
let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
|
||||
let functionNames = functions.map(fun => fun.name)
|
||||
let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
|
||||
let functionNames = functions.map(fun => fun.name)
|
||||
|
||||
let whattodoMessage = functionNames
|
||||
.slice(0,functionNames.length-5)
|
||||
.map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
|
||||
.slice(0, functionNames.length - 5)
|
||||
.map((functionName, i) => `[${i}]: Download predictions from ${functionName}`)
|
||||
.join('\n') +
|
||||
`\n[${functionNames.length-5}]: Merge jsons them into one big json (and push it to mongodb database)` +
|
||||
`\n[${functionNames.length-4}]: Update history` +
|
||||
`\n[${functionNames.length-3}]: Rebuild algolia database ("index")` +
|
||||
`\n[${functionNames.length-2}]: Rebuild netlify site with new data` +
|
||||
`\n[${functionNames.length - 5}]: Merge jsons them into one big json (and push it to mongodb database)` +
|
||||
`\n[${functionNames.length - 4}]: Update history` +
|
||||
`\n[${functionNames.length - 3}]: Rebuild algolia database ("index")` +
|
||||
`\n[${functionNames.length - 2}]: Rebuild netlify site with new data` +
|
||||
// `\n[${functionNames.length-1}]: Add to history` +
|
||||
`\n[${functionNames.length-1}]: All of the above` +
|
||||
`\n[${functionNames.length - 1}]: All of the above` +
|
||||
`\nChoose one option, wisely: #`
|
||||
|
||||
/* BODY */
|
||||
let commandLineUtility = async () => {
|
||||
let whattodo = async (message,callback) => {
|
||||
let commandLineUtility = async () => {
|
||||
let whattodo = async (message, callback) => {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
|
|
@ -61,24 +62,24 @@ let commandLineUtility = async () => {
|
|||
let executeoption = async (option) => {
|
||||
option = Number(option)
|
||||
//console.log(functionNames[option])
|
||||
if(option < 0){
|
||||
if (option < 0) {
|
||||
console.log(`Error, ${option} < 0 or ${option} < 0`)
|
||||
}else if(option < functions.length){
|
||||
} else if (option < functions.length) {
|
||||
await tryCatchTryAgain(functions[option])
|
||||
}
|
||||
}
|
||||
|
||||
if(process.argv.length==3){
|
||||
if (process.argv.length == 3) {
|
||||
const option = process.argv[2] // e.g., npm start 15 <-
|
||||
const optionNum = Number(option)
|
||||
if(!isNaN(optionNum)){
|
||||
if (!isNaN(optionNum)) {
|
||||
await executeoption(optionNum)
|
||||
}else if(option == "all"){
|
||||
await executeoption(functions.length-1) // 15 = execute all fetchers
|
||||
}else{
|
||||
} else if (option == "all") {
|
||||
await executeoption(functions.length - 1) // 15 = execute all fetchers
|
||||
} else {
|
||||
await whattodo(whattodoMessage, executeoption)
|
||||
}
|
||||
}else(
|
||||
} else (
|
||||
await whattodo(whattodoMessage, executeoption)
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import { upsert } from "../utils/mongo-wrapper.js"
|
|||
let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
|
||||
const DEBUG_MODE = "on"// "off"
|
||||
const SLEEP_TIME_RANDOM=100//5000 // miliseconds
|
||||
const SLEEP_TIME_EXTRA=0//1000
|
||||
/* Support functions */
|
||||
|
||||
async function fetchPage(page, cookie) {
|
||||
|
|
@ -171,27 +173,28 @@ async function csetforetell_inner(cookie) {
|
|||
while (!isEnd(response) && isSignedIn(response)) {
|
||||
|
||||
let htmlLines = response.split("\n")
|
||||
let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
|
||||
//let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
|
||||
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
|
||||
let questionHrefs = htmlLines.filter(str => str.includes("https://www.cset-foretell.com/questions/"))
|
||||
// console.log(questionHrefs)
|
||||
|
||||
|
||||
|
||||
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
|
||||
//console.log(response)
|
||||
console.log(h4elements)
|
||||
console.log("questionHrefs: ")
|
||||
console.log(questionHrefs)
|
||||
}
|
||||
|
||||
//console.log("")
|
||||
//console.log("")
|
||||
//console.log(h4elements)
|
||||
|
||||
for (let h4element of h4elements) {
|
||||
for (let questionHref of questionHrefs) {
|
||||
//console.log(h4element)
|
||||
|
||||
let h4elementSplit = h4element.split('"><span>')
|
||||
let url = h4elementSplit[0].split('<a href="')[1]
|
||||
let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "")
|
||||
await sleep(1000 + Math.random() * 1000) // don't be as noticeable
|
||||
let elementSplit = questionHref.split('"><span>')
|
||||
let url = elementSplit[0].split('<a href="')[1]
|
||||
let title = elementSplit[1].replace('</h4>', "").replace('</h5>', "").replace("</span></a>", "")
|
||||
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
|
||||
|
||||
try {
|
||||
let moreinfo = await fetchStats(url, cookie)
|
||||
|
|
@ -221,7 +224,7 @@ async function csetforetell_inner(cookie) {
|
|||
//i=Number(i)+1
|
||||
|
||||
console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
|
||||
await sleep(5000 + Math.random() * 1000) // don't be as noticeable
|
||||
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA) // don't be as noticeable
|
||||
|
||||
try {
|
||||
response = await fetchPage(i, cookie)
|
||||
|
|
|
|||
76
src/platforms/manifoldmarkets-fetch.js
Normal file
76
src/platforms/manifoldmarkets-fetch.js
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
/* Imports */
|
||||
import fs from 'fs'
|
||||
import axios from "axios"
|
||||
import { calculateStars } from "../utils/stars.js"
|
||||
import { upsert } from "../utils/mongo-wrapper.js"
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = 'https://us-central1-mantic-markets.cloudfunctions.net/markets'
|
||||
|
||||
/* Support functions */
|
||||
|
||||
async function fetchData() {
|
||||
let response = await axios({
|
||||
url: endpoint,
|
||||
method: 'GET',
|
||||
headers: ({
|
||||
'Content-Type': 'text/html',
|
||||
}),
|
||||
})
|
||||
.then(response => response.data)
|
||||
// console.log(response)
|
||||
return response
|
||||
}
|
||||
|
||||
async function processPredictions(predictions) {
|
||||
let results = await predictions.map(prediction => {
|
||||
let probability = prediction.probability
|
||||
let options = [
|
||||
{
|
||||
"name": "Yes",
|
||||
"probability": probability,
|
||||
"type": "PROBABILITY"
|
||||
},
|
||||
{
|
||||
"name": "No",
|
||||
"probability": 1 - probability,
|
||||
"type": "PROBABILITY"
|
||||
}
|
||||
]
|
||||
let result = ({
|
||||
"title": prediction.question,
|
||||
"url": prediction.url,
|
||||
"platform": "Manifold Markets",
|
||||
"description": prediction.description,
|
||||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"stars": calculateStars("Manifold Markets", ({
|
||||
volume7days: prediction.volume7days,
|
||||
volume24Hours: prediction.volume24Hours,
|
||||
liquidity: prediction.pool
|
||||
})),
|
||||
"createdTime": prediction.createdTime,
|
||||
"volume7Days": prediction.volume7Days,
|
||||
"volume24Hours": prediction.volume24Hours,
|
||||
"liquidity": prediction.pool
|
||||
}
|
||||
})
|
||||
return result
|
||||
})
|
||||
let unresolvedResults = results.filter(result => !result.isResolved)
|
||||
return unresolvedResults //resultsProcessed
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
export async function manifoldmarkets() {
|
||||
let data = await fetchData()
|
||||
let results = await processPredictions(data) // somehow needed
|
||||
// console.log(results)
|
||||
// let string = JSON.stringify(results, null, 2)
|
||||
// fs.writeFileSync('polyprediction-questions.json', string);
|
||||
await upsert(results, "manifoldmarkets-questions")
|
||||
console.log("Done")
|
||||
}
|
||||
//manifoldmarkets()
|
||||
|
|
@ -1,52 +1,53 @@
|
|||
import {astralcodexten} from "../platforms/astralcodexten-fetch.js"
|
||||
import {betfair} from "../platforms/betfair-fetch.js"
|
||||
import {coupcast} from "../platforms/coupcast-fetch.js"
|
||||
import {csetforetell} from "../platforms/csetforetell-fetch.js"
|
||||
import {elicit} from "../platforms/elicit-fetch.js"
|
||||
import {estimize} from "../platforms/estimize-fetch.js"
|
||||
import {fantasyscotus} from "../platforms/fantasyscotus-fetch.js"
|
||||
import {foretold} from "../platforms/foretold-fetch.js"
|
||||
import {goodjudgment} from "../platforms/goodjudgment-fetch.js"
|
||||
import {goodjudgmentopen} from "../platforms/goodjudmentopen-fetch.js"
|
||||
import {hypermind} from "../platforms/hypermind-fetch.js"
|
||||
import {kalshi} from "../platforms/kalshi-fetch.js"
|
||||
import {ladbrokes} from "../platforms/ladbrokes-fetch.js"
|
||||
import {metaculus} from "../platforms/metaculus-fetch.js"
|
||||
import {omen} from "../platforms/omen-fetch.js"
|
||||
import {polymarket} from "../platforms/polymarket-fetch.js"
|
||||
import {predictit} from "../platforms/predictit-fetch.js"
|
||||
import {rootclaim} from "../platforms/rootclaim-fetch.js"
|
||||
import {smarkets} from "../platforms/smarkets-fetch.js"
|
||||
import {wildeford} from "../platforms/wildeford-fetch.js"
|
||||
import {williamhill} from "../platforms/williamhill-fetch.js"
|
||||
import {mergeEverything} from "./mergeEverything.js"
|
||||
import {updateHistory} from "./history/updateHistory.js"
|
||||
import {rebuildAlgoliaDatabase} from "./algolia.js"
|
||||
import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js"
|
||||
import { astralcodexten } from "../platforms/astralcodexten-fetch.js"
|
||||
import { betfair } from "../platforms/betfair-fetch.js"
|
||||
import { coupcast } from "../platforms/coupcast-fetch.js"
|
||||
import { csetforetell } from "../platforms/csetforetell-fetch.js"
|
||||
import { elicit } from "../platforms/elicit-fetch.js"
|
||||
import { estimize } from "../platforms/estimize-fetch.js"
|
||||
import { fantasyscotus } from "../platforms/fantasyscotus-fetch.js"
|
||||
import { foretold } from "../platforms/foretold-fetch.js"
|
||||
import { goodjudgment } from "../platforms/goodjudgment-fetch.js"
|
||||
import { goodjudgmentopen } from "../platforms/goodjudmentopen-fetch.js"
|
||||
import { hypermind } from "../platforms/hypermind-fetch.js"
|
||||
import { kalshi } from "../platforms/kalshi-fetch.js"
|
||||
import { ladbrokes } from "../platforms/ladbrokes-fetch.js"
|
||||
import { manifoldmarkets } from "../platforms/manifoldmarkets-fetch.js"
|
||||
import { metaculus } from "../platforms/metaculus-fetch.js"
|
||||
import { omen } from "../platforms/omen-fetch.js"
|
||||
import { polymarket } from "../platforms/polymarket-fetch.js"
|
||||
import { predictit } from "../platforms/predictit-fetch.js"
|
||||
import { rootclaim } from "../platforms/rootclaim-fetch.js"
|
||||
import { smarkets } from "../platforms/smarkets-fetch.js"
|
||||
import { wildeford } from "../platforms/wildeford-fetch.js"
|
||||
import { williamhill } from "../platforms/williamhill-fetch.js"
|
||||
import { mergeEverything } from "./mergeEverything.js"
|
||||
import { updateHistory } from "./history/updateHistory.js"
|
||||
import { rebuildAlgoliaDatabase } from "./algolia.js"
|
||||
import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData.js"
|
||||
|
||||
/* Do everything */
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
|
||||
export async function tryCatchTryAgain (fun) {
|
||||
try{
|
||||
export async function tryCatchTryAgain(fun) {
|
||||
try {
|
||||
console.log("Initial try")
|
||||
await fun()
|
||||
}catch (error) {
|
||||
} catch (error) {
|
||||
sleep(10000)
|
||||
console.log("Second try")
|
||||
console.log(error)
|
||||
try{
|
||||
try {
|
||||
await fun()
|
||||
}catch (error){
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function doEverything(){
|
||||
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
|
||||
export async function doEverything() {
|
||||
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, manifoldmarkets, metaculus, omen, polymarket, predictit, rootclaim, smarkets, wildeford, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
|
||||
// Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
|
||||
|
||||
console.log("")
|
||||
|
|
@ -61,7 +62,7 @@ export async function doEverything(){
|
|||
console.log("")
|
||||
console.log("")
|
||||
|
||||
for(let fun of functions){
|
||||
for (let fun of functions) {
|
||||
console.log("")
|
||||
console.log("")
|
||||
console.log("****************************")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import fs from "fs"
|
|||
|
||||
import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
|
||||
|
||||
let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/data/frontpage.json'
|
||||
let filename = '/home/loki/Documents/core/software/fresh/js/metaforecasts/metaforecasts-backend/data/frontpage.json'
|
||||
let shuffle = (array) => {
|
||||
// https://stackoverflow.com/questions/2450954/how-to-randomi ze-shuffle-a-javascript-array
|
||||
let currentIndex = array.length, randomIndex;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import { mongoRead, upsert } from "./mongo-wrapper.js"
|
||||
|
||||
/* Merge everything */
|
||||
let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
|
||||
let sets = ["astralcodexten", "betfair", "coupcast", "csetforetell", "elicit", /* "estimize" ,*/ "fantasyscotus", "foretold", "givewellopenphil", "goodjudgment","goodjudmentopen", "hypermind", "kalshi", "ladbrokes", "manifoldmarkets", "metaculus", "omen", "polymarket", "predictit", "rootclaim", "smarkets", "wildeford", "williamhill", "xrisk"]
|
||||
let suffix = "-questions"
|
||||
|
||||
export async function mergeEverythingInner(){
|
||||
let merged = []
|
||||
for(let set of sets){
|
||||
let json = await mongoRead(set+suffix)
|
||||
console.log(`${set} has ${json.length} questions`)
|
||||
console.log(`${set} has ${json.length} questions\n`)
|
||||
merged = merged.concat(json)
|
||||
}
|
||||
let mergedprocessed = merged.map(element => ({...element, optionsstringforsearch: element.options.map(option => option.name).join(", ")}))
|
||||
|
|
|
|||
|
|
@ -156,6 +156,15 @@ function calculateStarsLadbrokes(data) {
|
|||
return starsInteger
|
||||
}
|
||||
|
||||
function calculateStarsManifoldMarkets(data) {
|
||||
let nuno = data => 2
|
||||
let eli = (data) => null
|
||||
let misha = (data) => null
|
||||
let starsDecimal = average([nuno(data)]) //, eli(data), misha(data)])
|
||||
let starsInteger = Math.round(starsDecimal)
|
||||
return starsInteger
|
||||
}
|
||||
|
||||
function calculateStarsMetaculus(data) {
|
||||
let nuno = data => data.numforecasts > 300 ? 4 : (data.numforecasts > 100 ? 3 : 2)
|
||||
let eli = (data) => 3
|
||||
|
|
@ -278,6 +287,9 @@ export function calculateStars(platform, data) {
|
|||
case "Ladbrokes":
|
||||
stars = calculateStarsLadbrokes(data)
|
||||
break;
|
||||
case "Manifold Markets":
|
||||
stars = calculateStarsManifoldMarkets(data)
|
||||
break;
|
||||
case "Metaculus":
|
||||
stars = calculateStarsMetaculus(data)
|
||||
break;
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user