refactor: reorganize platforms code further
This commit is contained in:
		
							parent
							
								
									4a8377704d
								
							
						
					
					
						commit
						89fc5ec8b6
					
				|  | @ -1,11 +1,4 @@ | ||||||
| /* Imports */ | import { processPlatform } from "../platforms"; | ||||||
| import { goodjudgment } from "../platforms/goodjudgment-fetch"; | import { goodjudgment } from "../platforms/goodjudgment"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | processPlatform(goodjudgment); | ||||||
| 
 |  | ||||||
| /* Utilities */ |  | ||||||
| 
 |  | ||||||
| /* Support functions */ |  | ||||||
| 
 |  | ||||||
| /* Body */ |  | ||||||
| goodjudgment(); |  | ||||||
|  |  | ||||||
|  | @ -3,7 +3,7 @@ import axios from "axios"; | ||||||
| import https from "https"; | import https from "https"; | ||||||
| 
 | 
 | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
| import { Forecast, PlatformFetcher } from "./"; | import { Forecast, Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; | let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; | ||||||
|  | @ -135,11 +135,11 @@ async function processPredictions(data) { | ||||||
|   return results; //resultsProcessed
 |   return results; //resultsProcessed
 | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| /* Body */ | export const betfair: Platform = { | ||||||
| 
 |   name: "betfair", | ||||||
| export const betfair: PlatformFetcher = async function () { |   async fetcher() { | ||||||
|   const data = await fetchPredictions(); |     const data = await fetchPredictions(); | ||||||
|   const results = await processPredictions(data); // somehow needed
 |     const results = await processPredictions(data); // somehow needed
 | ||||||
|   return results; |     return results; | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
| // betfair()
 |  | ||||||
|  | @ -2,7 +2,7 @@ | ||||||
| import axios from "axios"; | import axios from "axios"; | ||||||
| 
 | 
 | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
| import { PlatformFetcher } from "./"; | import { Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| let unixtime = new Date().getTime(); | let unixtime = new Date().getTime(); | ||||||
|  | @ -111,9 +111,11 @@ async function processData(data) { | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| /* Body */ | /* Body */ | ||||||
| export const fantasyscotus: PlatformFetcher = async function () { | export const fantasyscotus: Platform = { | ||||||
|   let rawData = await fetchData(); |   name: "fantasyscotus", | ||||||
|   let results = await processData(rawData); |   async fetcher() { | ||||||
|   return results; |     let rawData = await fetchData(); | ||||||
|  |     let results = await processData(rawData); | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
| //fantasyscotus()
 |  | ||||||
|  | @ -1,103 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| 
 |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| 
 |  | ||||||
| /* Definitions */ |  | ||||||
| let graphQLendpoint = "https://api.foretold.io/graphql"; |  | ||||||
| let highQualityCommunities = [ |  | ||||||
|   "0104d8e8-07e4-464b-8b32-74ef22b49f21", |  | ||||||
|   "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", |  | ||||||
|   "cf663021-f87f-4632-ad82-962d889a2d39", |  | ||||||
|   "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", |  | ||||||
|   "b2412a1d-0aa4-4e37-a12a-0aca9e440a96", |  | ||||||
| ]; |  | ||||||
| 
 |  | ||||||
| /* Support functions */ |  | ||||||
| async function fetchAllCommunityQuestions(communityId) { |  | ||||||
|   let response = await axios({ |  | ||||||
|     url: graphQLendpoint, |  | ||||||
|     method: "POST", |  | ||||||
|     headers: { "Content-Type": "application/json" }, |  | ||||||
|     data: JSON.stringify({ |  | ||||||
|       query: ` |  | ||||||
|       query { |  | ||||||
|         measurables( |  | ||||||
|           channelId: "${communityId}", |  | ||||||
|           states: OPEN, |  | ||||||
|           first: 500 |  | ||||||
|         ){ |  | ||||||
|           total |  | ||||||
|           edges{ |  | ||||||
|             node{ |  | ||||||
|               id |  | ||||||
|               name |  | ||||||
|               valueType |  | ||||||
|               measurementCount |  | ||||||
|               previousAggregate{ |  | ||||||
|                 value{ |  | ||||||
|                   percentage |  | ||||||
|                 } |  | ||||||
|               } |  | ||||||
|             } |  | ||||||
|           } |  | ||||||
|         } |  | ||||||
|       } |  | ||||||
|       `,
 |  | ||||||
|     }), |  | ||||||
|   }) |  | ||||||
|     .then((res) => res.data) |  | ||||||
|     .then((res) => res.data.measurables.edges); |  | ||||||
|   //console.log(response)
 |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| /* Body */ |  | ||||||
| 
 |  | ||||||
| export async function foretold() { |  | ||||||
|   let results = []; |  | ||||||
|   for (let community of highQualityCommunities) { |  | ||||||
|     let questions = await fetchAllCommunityQuestions(community); |  | ||||||
|     questions = questions.map((question) => question.node); |  | ||||||
|     questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
 |  | ||||||
|     questions.forEach((question) => { |  | ||||||
|       let id = `foretold-${question.id}`; |  | ||||||
|       let options = []; |  | ||||||
|       if (question.valueType == "PERCENTAGE") { |  | ||||||
|         let probability = question.previousAggregate.value.percentage; |  | ||||||
|         options = [ |  | ||||||
|           { |  | ||||||
|             name: "Yes", |  | ||||||
|             probability: probability / 100, |  | ||||||
|             type: "PROBABILITY", |  | ||||||
|           }, |  | ||||||
|           { |  | ||||||
|             name: "No", |  | ||||||
|             probability: 1 - probability / 100, |  | ||||||
|             type: "PROBABILITY", |  | ||||||
|           }, |  | ||||||
|         ]; |  | ||||||
|       } |  | ||||||
|       let result = { |  | ||||||
|         id: id, |  | ||||||
|         title: question.name, |  | ||||||
|         url: `https://www.foretold.io/c/${community}/m/${question.id}`, |  | ||||||
|         platform: "Foretold", |  | ||||||
|         description: "", |  | ||||||
|         options: options, |  | ||||||
|         timestamp: new Date().toISOString(), |  | ||||||
|         qualityindicators: { |  | ||||||
|           numforecasts: Math.floor(Number(question.measurementCount) / 2), |  | ||||||
|           stars: calculateStars("Foretold", {}), |  | ||||||
|         }, |  | ||||||
|         /*liquidity: liquidity.toFixed(2), |  | ||||||
|           tradevolume: tradevolume.toFixed(2), |  | ||||||
|           address: obj.address*/ |  | ||||||
|       }; |  | ||||||
|       // console.log(result)
 |  | ||||||
|       results.push(result); |  | ||||||
|     }); |  | ||||||
|   } |  | ||||||
|   return results; |  | ||||||
| } |  | ||||||
| // foretold()
 |  | ||||||
							
								
								
									
										104
									
								
								src/backend/platforms/foretold.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										104
									
								
								src/backend/platforms/foretold.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,104 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | 
 | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import { Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Definitions */ | ||||||
|  | let graphQLendpoint = "https://api.foretold.io/graphql"; | ||||||
|  | let highQualityCommunities = [ | ||||||
|  |   "0104d8e8-07e4-464b-8b32-74ef22b49f21", | ||||||
|  |   "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", | ||||||
|  |   "cf663021-f87f-4632-ad82-962d889a2d39", | ||||||
|  |   "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", | ||||||
|  |   "b2412a1d-0aa4-4e37-a12a-0aca9e440a96", | ||||||
|  | ]; | ||||||
|  | 
 | ||||||
|  | /* Support functions */ | ||||||
|  | async function fetchAllCommunityQuestions(communityId) { | ||||||
|  |   let response = await axios({ | ||||||
|  |     url: graphQLendpoint, | ||||||
|  |     method: "POST", | ||||||
|  |     headers: { "Content-Type": "application/json" }, | ||||||
|  |     data: JSON.stringify({ | ||||||
|  |       query: ` | ||||||
|  |       query { | ||||||
|  |         measurables( | ||||||
|  |           channelId: "${communityId}", | ||||||
|  |           states: OPEN, | ||||||
|  |           first: 500 | ||||||
|  |         ){ | ||||||
|  |           total | ||||||
|  |           edges{ | ||||||
|  |             node{ | ||||||
|  |               id | ||||||
|  |               name | ||||||
|  |               valueType | ||||||
|  |               measurementCount | ||||||
|  |               previousAggregate{ | ||||||
|  |                 value{ | ||||||
|  |                   percentage | ||||||
|  |                 } | ||||||
|  |               } | ||||||
|  |             } | ||||||
|  |           } | ||||||
|  |         } | ||||||
|  |       } | ||||||
|  |       `,
 | ||||||
|  |     }), | ||||||
|  |   }) | ||||||
|  |     .then((res) => res.data) | ||||||
|  |     .then((res) => res.data.measurables.edges); | ||||||
|  |   //console.log(response)
 | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export const foretold: Platform = { | ||||||
|  |   name: "foretold", | ||||||
|  |   async fetcher() { | ||||||
|  |     let results = []; | ||||||
|  |     for (let community of highQualityCommunities) { | ||||||
|  |       let questions = await fetchAllCommunityQuestions(community); | ||||||
|  |       questions = questions.map((question) => question.node); | ||||||
|  |       questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
 | ||||||
|  |       questions.forEach((question) => { | ||||||
|  |         let id = `foretold-${question.id}`; | ||||||
|  |         let options = []; | ||||||
|  |         if (question.valueType == "PERCENTAGE") { | ||||||
|  |           let probability = question.previousAggregate.value.percentage; | ||||||
|  |           options = [ | ||||||
|  |             { | ||||||
|  |               name: "Yes", | ||||||
|  |               probability: probability / 100, | ||||||
|  |               type: "PROBABILITY", | ||||||
|  |             }, | ||||||
|  |             { | ||||||
|  |               name: "No", | ||||||
|  |               probability: 1 - probability / 100, | ||||||
|  |               type: "PROBABILITY", | ||||||
|  |             }, | ||||||
|  |           ]; | ||||||
|  |         } | ||||||
|  |         let result = { | ||||||
|  |           id: id, | ||||||
|  |           title: question.name, | ||||||
|  |           url: `https://www.foretold.io/c/${community}/m/${question.id}`, | ||||||
|  |           platform: "Foretold", | ||||||
|  |           description: "", | ||||||
|  |           options: options, | ||||||
|  |           timestamp: new Date().toISOString(), | ||||||
|  |           qualityindicators: { | ||||||
|  |             numforecasts: Math.floor(Number(question.measurementCount) / 2), | ||||||
|  |             stars: calculateStars("Foretold", {}), | ||||||
|  |           }, | ||||||
|  |           /*liquidity: liquidity.toFixed(2), | ||||||
|  |           tradevolume: tradevolume.toFixed(2), | ||||||
|  |           address: obj.address*/ | ||||||
|  |         }; | ||||||
|  |         // console.log(result)
 | ||||||
|  |         results.push(result); | ||||||
|  |       }); | ||||||
|  |     } | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -1,126 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| import { Tabletojson } from "tabletojson"; |  | ||||||
| import tunnel from "tunnel"; |  | ||||||
| 
 |  | ||||||
| import { hash } from "../utils/hash"; |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| import { PlatformFetcher } from "./"; |  | ||||||
| 
 |  | ||||||
| /* Definitions */ |  | ||||||
| let endpoint = "https://goodjudgment.io/superforecasts/"; |  | ||||||
| String.prototype.replaceAll = function replaceAll(search, replace) { |  | ||||||
|   return this.split(search).join(replace); |  | ||||||
| }; |  | ||||||
| 
 |  | ||||||
| // Tunelling
 |  | ||||||
| /* Support functions */ |  | ||||||
| 
 |  | ||||||
| /* Body */ |  | ||||||
| export const goodjudgment: PlatformFetcher = async function () { |  | ||||||
|   // Proxy fuckery
 |  | ||||||
|   let proxy; |  | ||||||
|   /* |  | ||||||
| 	 * try { |  | ||||||
|     proxy = await axios |  | ||||||
|       .get("http://pubproxy.com/api/proxy") |  | ||||||
|       .then((query) => query.data); |  | ||||||
|     console.log(proxy); |  | ||||||
|   } catch (error) { |  | ||||||
|     console.log("Proxy generation failed; using backup proxy instead"); |  | ||||||
|     // hard-coded backup proxy
 |  | ||||||
| 		*/ |  | ||||||
|   proxy = { |  | ||||||
|     ip: process.env.BACKUP_PROXY_IP, |  | ||||||
|     port: process.env.BACKUP_PROXY_PORT, |  | ||||||
|   }; |  | ||||||
|   // }
 |  | ||||||
|   let agent = tunnel.httpsOverHttp({ |  | ||||||
|     proxy: { |  | ||||||
|       host: proxy.ip, |  | ||||||
|       port: proxy.port, |  | ||||||
|     }, |  | ||||||
|   }); |  | ||||||
| 
 |  | ||||||
|   let content = await axios |  | ||||||
|     .request({ |  | ||||||
|       url: "https://goodjudgment.io/superforecasts/", |  | ||||||
|       method: "get", |  | ||||||
|       headers: { |  | ||||||
|         "User-Agent": "Chrome", |  | ||||||
|       }, |  | ||||||
|       // agent,
 |  | ||||||
|       // port: 80,
 |  | ||||||
|     }) |  | ||||||
|     .then((query) => query.data); |  | ||||||
| 
 |  | ||||||
|   // Processing
 |  | ||||||
|   let results = []; |  | ||||||
|   let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); |  | ||||||
|   jsonTable.shift(); // deletes first element
 |  | ||||||
|   jsonTable.pop(); // deletes last element
 |  | ||||||
|   // console.log(jsonTable)
 |  | ||||||
|   for (let table of jsonTable) { |  | ||||||
|     // console.log(table)
 |  | ||||||
|     let title = table[0]["0"].split("\t\t\t").splice(3)[0]; |  | ||||||
|     if (title != undefined) { |  | ||||||
|       title = title.replaceAll("</a>", ""); |  | ||||||
|       let id = `goodjudgment-${hash(title)}`; |  | ||||||
|       let description = table |  | ||||||
|         .filter((row) => row["0"].includes("BACKGROUND:")) |  | ||||||
|         .map((row) => row["0"]) |  | ||||||
|         .map((text) => |  | ||||||
|           text |  | ||||||
|             .split("BACKGROUND:")[1] |  | ||||||
|             .split("Examples of Superforecaster")[0] |  | ||||||
|             .split("AT A GLANCE")[0] |  | ||||||
|             .replaceAll("\n\n", "\n") |  | ||||||
|             .split("\n") |  | ||||||
|             .slice(3) |  | ||||||
|             .join(" ") |  | ||||||
|             .replaceAll("      ", "") |  | ||||||
|             .replaceAll("<br> ", "") |  | ||||||
|         )[0]; |  | ||||||
|       let options = table |  | ||||||
|         .filter((row) => "4" in row) |  | ||||||
|         .map((row) => ({ |  | ||||||
|           name: row["2"] |  | ||||||
|             .split('<span class="qTitle">')[1] |  | ||||||
|             .replace("</span>", ""), |  | ||||||
|           probability: Number(row["3"].split("%")[0]) / 100, |  | ||||||
|           type: "PROBABILITY", |  | ||||||
|         })); |  | ||||||
|       let analysis = table.filter((row) => |  | ||||||
|         row[0] ? row[0].toLowerCase().includes("commentary") : false |  | ||||||
|       ); |  | ||||||
|       // "Examples of Superforecaster Commentary" / Analysis
 |  | ||||||
|       // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list.
 |  | ||||||
|       analysis = analysis ? analysis[0] : ""; |  | ||||||
|       analysis = analysis ? analysis[0] : ""; // not a duplicate
 |  | ||||||
|       // console.log(analysis)
 |  | ||||||
|       let standardObj = { |  | ||||||
|         id: id, |  | ||||||
|         title: title, |  | ||||||
|         url: endpoint, |  | ||||||
|         platform: "Good Judgment", |  | ||||||
|         description: description, |  | ||||||
|         options: options, |  | ||||||
|         timestamp: new Date().toISOString(), |  | ||||||
|         qualityindicators: { |  | ||||||
|           stars: calculateStars("Good Judgment", {}), |  | ||||||
|         }, |  | ||||||
|         extra: { |  | ||||||
|           superforecastercommentary: analysis || "", |  | ||||||
|         }, |  | ||||||
|       }; |  | ||||||
|       results.push(standardObj); |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
| 
 |  | ||||||
|   console.log( |  | ||||||
|     "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js" |  | ||||||
|   ); |  | ||||||
| 
 |  | ||||||
|   return results; |  | ||||||
| }; |  | ||||||
| // goodjudgment()
 |  | ||||||
							
								
								
									
										125
									
								
								src/backend/platforms/goodjudgment.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										125
									
								
								src/backend/platforms/goodjudgment.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,125 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | import { Tabletojson } from "tabletojson"; | ||||||
|  | import tunnel from "tunnel"; | ||||||
|  | 
 | ||||||
|  | import { hash } from "../utils/hash"; | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import { Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Definitions */ | ||||||
|  | let endpoint = "https://goodjudgment.io/superforecasts/"; | ||||||
|  | String.prototype.replaceAll = function replaceAll(search, replace) { | ||||||
|  |   return this.split(search).join(replace); | ||||||
|  | }; | ||||||
|  | 
 | ||||||
|  | /* Body */ | ||||||
|  | export const goodjudgment: Platform = { | ||||||
|  |   name: "goodjudgment", | ||||||
|  |   async fetcher() { | ||||||
|  |     // Proxy fuckery
 | ||||||
|  |     let proxy; | ||||||
|  |     /* | ||||||
|  | 	 * try { | ||||||
|  |     proxy = await axios | ||||||
|  |       .get("http://pubproxy.com/api/proxy") | ||||||
|  |       .then((query) => query.data); | ||||||
|  |     console.log(proxy); | ||||||
|  |   } catch (error) { | ||||||
|  |     console.log("Proxy generation failed; using backup proxy instead"); | ||||||
|  |     // hard-coded backup proxy
 | ||||||
|  | 		*/ | ||||||
|  |     proxy = { | ||||||
|  |       ip: process.env.BACKUP_PROXY_IP, | ||||||
|  |       port: process.env.BACKUP_PROXY_PORT, | ||||||
|  |     }; | ||||||
|  |     // }
 | ||||||
|  |     let agent = tunnel.httpsOverHttp({ | ||||||
|  |       proxy: { | ||||||
|  |         host: proxy.ip, | ||||||
|  |         port: proxy.port, | ||||||
|  |       }, | ||||||
|  |     }); | ||||||
|  | 
 | ||||||
|  |     let content = await axios | ||||||
|  |       .request({ | ||||||
|  |         url: "https://goodjudgment.io/superforecasts/", | ||||||
|  |         method: "get", | ||||||
|  |         headers: { | ||||||
|  |           "User-Agent": "Chrome", | ||||||
|  |         }, | ||||||
|  |         // agent,
 | ||||||
|  |         // port: 80,
 | ||||||
|  |       }) | ||||||
|  |       .then((query) => query.data); | ||||||
|  | 
 | ||||||
|  |     // Processing
 | ||||||
|  |     let results = []; | ||||||
|  |     let jsonTable = Tabletojson.convert(content, { stripHtmlFromCells: false }); | ||||||
|  |     jsonTable.shift(); // deletes first element
 | ||||||
|  |     jsonTable.pop(); // deletes last element
 | ||||||
|  |     // console.log(jsonTable)
 | ||||||
|  |     for (let table of jsonTable) { | ||||||
|  |       // console.log(table)
 | ||||||
|  |       let title = table[0]["0"].split("\t\t\t").splice(3)[0]; | ||||||
|  |       if (title != undefined) { | ||||||
|  |         title = title.replaceAll("</a>", ""); | ||||||
|  |         let id = `goodjudgment-${hash(title)}`; | ||||||
|  |         let description = table | ||||||
|  |           .filter((row) => row["0"].includes("BACKGROUND:")) | ||||||
|  |           .map((row) => row["0"]) | ||||||
|  |           .map((text) => | ||||||
|  |             text | ||||||
|  |               .split("BACKGROUND:")[1] | ||||||
|  |               .split("Examples of Superforecaster")[0] | ||||||
|  |               .split("AT A GLANCE")[0] | ||||||
|  |               .replaceAll("\n\n", "\n") | ||||||
|  |               .split("\n") | ||||||
|  |               .slice(3) | ||||||
|  |               .join(" ") | ||||||
|  |               .replaceAll("      ", "") | ||||||
|  |               .replaceAll("<br> ", "") | ||||||
|  |           )[0]; | ||||||
|  |         let options = table | ||||||
|  |           .filter((row) => "4" in row) | ||||||
|  |           .map((row) => ({ | ||||||
|  |             name: row["2"] | ||||||
|  |               .split('<span class="qTitle">')[1] | ||||||
|  |               .replace("</span>", ""), | ||||||
|  |             probability: Number(row["3"].split("%")[0]) / 100, | ||||||
|  |             type: "PROBABILITY", | ||||||
|  |           })); | ||||||
|  |         let analysis = table.filter((row) => | ||||||
|  |           row[0] ? row[0].toLowerCase().includes("commentary") : false | ||||||
|  |         ); | ||||||
|  |         // "Examples of Superforecaster Commentary" / Analysis
 | ||||||
|  |         // The following is necessary twice, because we want to check if there is an empty list, and then get the first element of the first element of the list.
 | ||||||
|  |         analysis = analysis ? analysis[0] : ""; | ||||||
|  |         analysis = analysis ? analysis[0] : ""; // not a duplicate
 | ||||||
|  |         // console.log(analysis)
 | ||||||
|  |         let standardObj = { | ||||||
|  |           id: id, | ||||||
|  |           title: title, | ||||||
|  |           url: endpoint, | ||||||
|  |           platform: "Good Judgment", | ||||||
|  |           description: description, | ||||||
|  |           options: options, | ||||||
|  |           timestamp: new Date().toISOString(), | ||||||
|  |           qualityindicators: { | ||||||
|  |             stars: calculateStars("Good Judgment", {}), | ||||||
|  |           }, | ||||||
|  |           extra: { | ||||||
|  |             superforecastercommentary: analysis || "", | ||||||
|  |           }, | ||||||
|  |         }; | ||||||
|  |         results.push(standardObj); | ||||||
|  |       } | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     console.log( | ||||||
|  |       "Failing is not unexpected; see utils/pullSuperforecastsManually.sh/js" | ||||||
|  |     ); | ||||||
|  | 
 | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -5,7 +5,7 @@ import { Tabletojson } from "tabletojson"; | ||||||
| import { applyIfSecretExists } from "../utils/getSecrets"; | import { applyIfSecretExists } from "../utils/getSecrets"; | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
| import toMarkdown from "../utils/toMarkdown"; | import toMarkdown from "../utils/toMarkdown"; | ||||||
| import { PlatformFetcher } from "./"; | import { Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| let htmlEndPoint = "https://www.gjopen.com/questions?page="; | let htmlEndPoint = "https://www.gjopen.com/questions?page="; | ||||||
|  | @ -236,7 +236,10 @@ async function goodjudgmentopen_inner(cookie) { | ||||||
|   return results; |   return results; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| export const goodjudmentopen: PlatformFetcher = async function () { | export const goodjudmentopen: Platform = { | ||||||
|   let cookie = process.env.GOODJUDGMENTOPENCOOKIE; |   name: "goodjudmentopen", // note the typo! current table name is without `g`, `goodjudmentopen`
 | ||||||
|   return await applyIfSecretExists(cookie, goodjudgmentopen_inner); |   async fetcher() { | ||||||
|  |     let cookie = process.env.GOODJUDGMENTOPENCOOKIE; | ||||||
|  |     return await applyIfSecretExists(cookie, goodjudgmentopen_inner); | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
|  | @ -1,18 +1,18 @@ | ||||||
| import { databaseUpsert } from "../database/database-wrapper"; | import { databaseUpsert } from "../database/database-wrapper"; | ||||||
| import { betfair } from "./betfair-fetch"; | import { betfair } from "./betfair"; | ||||||
| import { fantasyscotus } from "./fantasyscotus-fetch"; | import { fantasyscotus } from "./fantasyscotus"; | ||||||
| import { foretold } from "./foretold-fetch"; | import { foretold } from "./foretold"; | ||||||
| import { goodjudgment } from "./goodjudgment-fetch"; | import { goodjudgment } from "./goodjudgment"; | ||||||
| import { goodjudmentopen } from "./goodjudmentopen-fetch"; | import { goodjudmentopen } from "./goodjudmentopen"; | ||||||
| import { infer } from "./infer-fetch"; | import { infer } from "./infer"; | ||||||
| import { kalshi } from "./kalshi-fetch"; | import { kalshi } from "./kalshi"; | ||||||
| import { manifoldmarkets } from "./manifoldmarkets-fetch"; | import { manifoldmarkets } from "./manifoldmarkets"; | ||||||
| import { metaculus } from "./metaculus-fetch"; | import { metaculus } from "./metaculus"; | ||||||
| import { polymarket } from "./polymarket-fetch"; | import { polymarket } from "./polymarket"; | ||||||
| import { predictit } from "./predictit-fetch"; | import { predictit } from "./predictit"; | ||||||
| import { rootclaim } from "./rootclaim-fetch"; | import { rootclaim } from "./rootclaim"; | ||||||
| import { smarkets } from "./smarkets-fetch"; | import { smarkets } from "./smarkets"; | ||||||
| import { wildeford } from "./wildeford-fetch"; | import { wildeford } from "./wildeford"; | ||||||
| 
 | 
 | ||||||
| export interface Forecast { | export interface Forecast { | ||||||
|   id: string; |   id: string; | ||||||
|  | @ -26,9 +26,10 @@ export interface Forecast { | ||||||
|   extra?: any; |   extra?: any; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | // fetcher should return null if platform failed to fetch forecasts for some reason
 | ||||||
| export type PlatformFetcher = () => Promise<Forecast[] | null>; | export type PlatformFetcher = () => Promise<Forecast[] | null>; | ||||||
| 
 | 
 | ||||||
| interface Platform { | export interface Platform { | ||||||
|   name: string; |   name: string; | ||||||
|   fetcher: PlatformFetcher; |   fetcher: PlatformFetcher; | ||||||
| } | } | ||||||
|  | @ -53,7 +54,7 @@ export const platforms: Platform[] = [ | ||||||
|   fantasyscotus, |   fantasyscotus, | ||||||
|   foretold, |   foretold, | ||||||
|   goodjudgment, |   goodjudgment, | ||||||
|   goodjudmentopen, // note the typo! current table name is without `g`, `goodjudmentopen`
 |   goodjudmentopen, | ||||||
|   infer, |   infer, | ||||||
|   kalshi, |   kalshi, | ||||||
|   manifoldmarkets, |   manifoldmarkets, | ||||||
|  | @ -63,7 +64,7 @@ export const platforms: Platform[] = [ | ||||||
|   rootclaim, |   rootclaim, | ||||||
|   smarkets, |   smarkets, | ||||||
|   wildeford, |   wildeford, | ||||||
| ].map((fun) => ({ name: fun.name, fetcher: fun })); | ]; | ||||||
| 
 | 
 | ||||||
| export const processPlatform = async (platform: Platform) => { | export const processPlatform = async (platform: Platform) => { | ||||||
|   let results = await platform.fetcher(); |   let results = await platform.fetcher(); | ||||||
|  |  | ||||||
|  | @ -5,7 +5,7 @@ import { Tabletojson } from "tabletojson"; | ||||||
| import { applyIfSecretExists } from "../utils/getSecrets"; | import { applyIfSecretExists } from "../utils/getSecrets"; | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
| import toMarkdown from "../utils/toMarkdown"; | import toMarkdown from "../utils/toMarkdown"; | ||||||
| import { Forecast, PlatformFetcher } from "./"; | import { Forecast, Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| let htmlEndPoint = "https://www.infer-pub.com/questions"; | let htmlEndPoint = "https://www.infer-pub.com/questions"; | ||||||
|  | @ -277,7 +277,10 @@ async function infer_inner(cookie) { | ||||||
|   return results; |   return results; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| export const infer: PlatformFetcher = async function () { | export const infer: Platform = { | ||||||
|   let cookie = process.env.INFER_COOKIE; |   name: "infer", | ||||||
|   return await applyIfSecretExists(cookie, infer_inner); |   async fetcher() { | ||||||
|  |     let cookie = process.env.INFER_COOKIE; | ||||||
|  |     return await applyIfSecretExists(cookie, infer_inner); | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
|  | @ -2,7 +2,7 @@ | ||||||
| import axios from "axios"; | import axios from "axios"; | ||||||
| 
 | 
 | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
| import { PlatformFetcher } from "./"; | import { Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 | let jsonEndpoint = "https://trading-api.kalshi.com/v1/cached/markets/"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 | ||||||
|  | @ -69,9 +69,10 @@ async function processMarkets(markets) { | ||||||
|   return results; //resultsProcessed
 |   return results; //resultsProcessed
 | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| /* Body */ | export const kalshi: Platform = { | ||||||
| export const kalshi: PlatformFetcher = async function () { |   name: "kalshi", | ||||||
|   let markets = await fetchAllMarkets(); |   fetcher: async function () { | ||||||
|   return await processMarkets(markets); |     let markets = await fetchAllMarkets(); | ||||||
|  |     return await processMarkets(markets); | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
| // kalshi()
 |  | ||||||
|  | @ -2,6 +2,7 @@ | ||||||
| import axios from "axios"; | import axios from "axios"; | ||||||
| 
 | 
 | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
|  | import { Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| let endpoint = "https://manifold.markets/api/v0/markets"; | let endpoint = "https://manifold.markets/api/v0/markets"; | ||||||
|  | @ -87,12 +88,12 @@ async function processPredictions(predictions) { | ||||||
|   return unresolvedResults; //resultsProcessed
 |   return unresolvedResults; //resultsProcessed
 | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| /* Body */ | export const manifoldmarkets: Platform = { | ||||||
| 
 |   name: "manifoldmarkets", | ||||||
| export const manifoldmarkets = async function () { |   async fetcher() { | ||||||
|   let data = await fetchData(); |     let data = await fetchData(); | ||||||
|   let results = await processPredictions(data); // somehow needed
 |     let results = await processPredictions(data); // somehow needed
 | ||||||
|   showStatistics(results); |     showStatistics(results); | ||||||
|   return results; |     return results; | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
| // manifoldmarkets()
 |  | ||||||
|  | @ -1,195 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| 
 |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| import toMarkdown from "../utils/toMarkdown"; |  | ||||||
| import { PlatformFetcher } from "./"; |  | ||||||
| 
 |  | ||||||
| /* Definitions */ |  | ||||||
| let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; |  | ||||||
| let now = new Date().toISOString(); |  | ||||||
| let DEBUG_MODE = "off"; |  | ||||||
| let SLEEP_TIME = 5000; |  | ||||||
| /* Support functions */ |  | ||||||
| async function fetchMetaculusQuestions(next) { |  | ||||||
|   // Numbers about a given address: how many, how much, at what price, etc.
 |  | ||||||
|   let response; |  | ||||||
|   let data; |  | ||||||
|   try { |  | ||||||
|     response = await axios({ |  | ||||||
|       url: next, |  | ||||||
|       method: "GET", |  | ||||||
|       headers: { "Content-Type": "application/json" }, |  | ||||||
|     }); |  | ||||||
|     data = response.data; |  | ||||||
|   } catch (error) { |  | ||||||
|     console.log(`Error in  async function fetchMetaculusQuestions(next)`); |  | ||||||
|     if (!!error.response.headers["retry-after"]) { |  | ||||||
|       let timeout = error.response.headers["retry-after"]; |  | ||||||
|       console.log(`Timeout: ${timeout}`); |  | ||||||
|       await sleep(Number(timeout) * 1000 + SLEEP_TIME); |  | ||||||
|     } else { |  | ||||||
|       await sleep(SLEEP_TIME); |  | ||||||
|     } |  | ||||||
|     console.log(error); |  | ||||||
|   } finally { |  | ||||||
|     try { |  | ||||||
|       response = await axios({ |  | ||||||
|         url: next, |  | ||||||
|         method: "GET", |  | ||||||
|         headers: { "Content-Type": "application/json" }, |  | ||||||
|       }); |  | ||||||
|       data = response.data; |  | ||||||
|     } catch (error) { |  | ||||||
|       console.log(error); |  | ||||||
|       return { results: [] }; |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
|   // console.log(response)
 |  | ||||||
|   return data; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| function sleep(ms) { |  | ||||||
|   return new Promise((resolve) => setTimeout(resolve, ms)); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchMetaculusQuestionDescription(slug) { |  | ||||||
|   try { |  | ||||||
|     let response = await axios({ |  | ||||||
|       method: "get", |  | ||||||
|       url: "https://www.metaculus.com" + slug, |  | ||||||
|     }).then((response) => response.data); |  | ||||||
|     return response; |  | ||||||
|   } catch (error) { |  | ||||||
|     console.log(`Error in: fetchMetaculusQuestionDescription`); |  | ||||||
|     console.log( |  | ||||||
|       `We encountered some error when attempting to fetch a metaculus page. Trying again` |  | ||||||
|     ); |  | ||||||
|     if ( |  | ||||||
|       typeof error.response != "undefined" && |  | ||||||
|       typeof error.response.headers != "undefined" && |  | ||||||
|       typeof error.response.headers["retry-after"] != "undefined" |  | ||||||
|     ) { |  | ||||||
|       let timeout = error.response.headers["retry-after"]; |  | ||||||
|       console.log(`Timeout: ${timeout}`); |  | ||||||
|       await sleep(Number(timeout) * 1000 + SLEEP_TIME); |  | ||||||
|     } else { |  | ||||||
|       await sleep(SLEEP_TIME); |  | ||||||
|     } |  | ||||||
|     try { |  | ||||||
|       let response = await axios({ |  | ||||||
|         method: "get", |  | ||||||
|         url: "https://www.metaculus.com" + slug, |  | ||||||
|       }).then((response) => response.data); |  | ||||||
|       // console.log(response)
 |  | ||||||
|       return response; |  | ||||||
|     } catch (error) { |  | ||||||
|       console.log( |  | ||||||
|         `We encountered some error when attempting to fetch a metaculus page.` |  | ||||||
|       ); |  | ||||||
|       console.log("Error", error); |  | ||||||
|       throw "Giving up"; |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| /* Body */ |  | ||||||
| 
 |  | ||||||
| export const metaculus: PlatformFetcher = async function () { |  | ||||||
|   // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
 |  | ||||||
|   // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
 |  | ||||||
|   // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
 |  | ||||||
|   // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test
 |  | ||||||
|   let all_questions = []; |  | ||||||
|   let next = "https://www.metaculus.com/api2/questions/"; |  | ||||||
|   let i = 1; |  | ||||||
|   while (next) { |  | ||||||
|     if (i % 20 == 0) { |  | ||||||
|       console.log("Sleeping for 500ms"); |  | ||||||
|       await sleep(SLEEP_TIME); |  | ||||||
|     } |  | ||||||
|     console.log(`\nQuery #${i}`); |  | ||||||
|     let metaculusQuestions = await fetchMetaculusQuestions(next); |  | ||||||
|     let results = metaculusQuestions.results; |  | ||||||
|     let j = false; |  | ||||||
|     for (let result of results) { |  | ||||||
|       if (result.publish_time < now && now < result.resolve_time) { |  | ||||||
|         await sleep(SLEEP_TIME / 2); |  | ||||||
|         let questionPage = await fetchMetaculusQuestionDescription( |  | ||||||
|           result.page_url |  | ||||||
|         ); |  | ||||||
|         if (!questionPage.includes("A public prediction by")) { |  | ||||||
|           // console.log(questionPage)
 |  | ||||||
|           let descriptionraw = questionPage.split( |  | ||||||
|             `<div class="content" ng-bind-html-compile="qctrl.question.description_html">` |  | ||||||
|           )[1]; //.split(`<div class="question__content">`)[1]
 |  | ||||||
|           let descriptionprocessed1 = descriptionraw.split("</div>")[0]; |  | ||||||
|           let descriptionprocessed2 = toMarkdown(descriptionprocessed1); |  | ||||||
|           let description = descriptionprocessed2; |  | ||||||
| 
 |  | ||||||
|           let isbinary = result.possibilities.type == "binary"; |  | ||||||
|           let options = []; |  | ||||||
|           if (isbinary) { |  | ||||||
|             let probability = Number(result.community_prediction.full.q2); |  | ||||||
|             options = [ |  | ||||||
|               { |  | ||||||
|                 name: "Yes", |  | ||||||
|                 probability: probability, |  | ||||||
|                 type: "PROBABILITY", |  | ||||||
|               }, |  | ||||||
|               { |  | ||||||
|                 name: "No", |  | ||||||
|                 probability: 1 - probability, |  | ||||||
|                 type: "PROBABILITY", |  | ||||||
|               }, |  | ||||||
|             ]; |  | ||||||
|           } |  | ||||||
|           let id = `metaculus-${result.id}`; |  | ||||||
|           let interestingInfo = { |  | ||||||
|             id: id, |  | ||||||
|             title: result.title, |  | ||||||
|             url: "https://www.metaculus.com" + result.page_url, |  | ||||||
|             platform: "Metaculus", |  | ||||||
|             description: description, |  | ||||||
|             options: options, |  | ||||||
|             timestamp: new Date().toISOString(), |  | ||||||
|             qualityindicators: { |  | ||||||
|               numforecasts: Number(result.number_of_predictions), |  | ||||||
|               stars: calculateStars("Metaculus", { |  | ||||||
|                 numforecasts: result.number_of_predictions, |  | ||||||
|               }), |  | ||||||
|             }, |  | ||||||
|             extra: { |  | ||||||
|               resolution_data: { |  | ||||||
|                 publish_time: result.publish_time, |  | ||||||
|                 resolution: result.resolution, |  | ||||||
|                 close_time: result.close_time, |  | ||||||
|                 resolve_time: result.resolve_time, |  | ||||||
|               }, |  | ||||||
|             }, |  | ||||||
|             //"status": result.status,
 |  | ||||||
|             //"publish_time": result.publish_time,
 |  | ||||||
|             //"close_time": result.close_time,
 |  | ||||||
|             //"type": result.possibilities.type, // We want binary ones here.
 |  | ||||||
|             //"last_activity_time": result.last_activity_time,
 |  | ||||||
|           }; |  | ||||||
|           if (Number(result.number_of_predictions) >= 10) { |  | ||||||
|             console.log(`- ${interestingInfo.title}`); |  | ||||||
|             all_questions.push(interestingInfo); |  | ||||||
|             if ((!j && i % 20 == 0) || DEBUG_MODE == "on") { |  | ||||||
|               console.log(interestingInfo); |  | ||||||
|               j = true; |  | ||||||
|             } |  | ||||||
|           } |  | ||||||
|         } else { |  | ||||||
|           console.log("- [Skipping public prediction]"); |  | ||||||
|         } |  | ||||||
|       } |  | ||||||
|     } |  | ||||||
|     next = metaculusQuestions.next; |  | ||||||
|     i = i + 1; |  | ||||||
|   } |  | ||||||
| 
 |  | ||||||
|   return all_questions; |  | ||||||
| }; |  | ||||||
| //metaculus()
 |  | ||||||
							
								
								
									
										195
									
								
								src/backend/platforms/metaculus.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										195
									
								
								src/backend/platforms/metaculus.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,195 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | 
 | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import toMarkdown from "../utils/toMarkdown"; | ||||||
|  | import { Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Definitions */ | ||||||
|  | let jsonEndPoint = "https://www.metaculus.com/api2/questions/?page="; | ||||||
|  | let now = new Date().toISOString(); | ||||||
|  | let DEBUG_MODE = "off"; | ||||||
|  | let SLEEP_TIME = 5000; | ||||||
|  | /* Support functions */ | ||||||
|  | async function fetchMetaculusQuestions(next) { | ||||||
|  |   // Numbers about a given address: how many, how much, at what price, etc.
 | ||||||
|  |   let response; | ||||||
|  |   let data; | ||||||
|  |   try { | ||||||
|  |     response = await axios({ | ||||||
|  |       url: next, | ||||||
|  |       method: "GET", | ||||||
|  |       headers: { "Content-Type": "application/json" }, | ||||||
|  |     }); | ||||||
|  |     data = response.data; | ||||||
|  |   } catch (error) { | ||||||
|  |     console.log(`Error in  async function fetchMetaculusQuestions(next)`); | ||||||
|  |     if (!!error.response.headers["retry-after"]) { | ||||||
|  |       let timeout = error.response.headers["retry-after"]; | ||||||
|  |       console.log(`Timeout: ${timeout}`); | ||||||
|  |       await sleep(Number(timeout) * 1000 + SLEEP_TIME); | ||||||
|  |     } else { | ||||||
|  |       await sleep(SLEEP_TIME); | ||||||
|  |     } | ||||||
|  |     console.log(error); | ||||||
|  |   } finally { | ||||||
|  |     try { | ||||||
|  |       response = await axios({ | ||||||
|  |         url: next, | ||||||
|  |         method: "GET", | ||||||
|  |         headers: { "Content-Type": "application/json" }, | ||||||
|  |       }); | ||||||
|  |       data = response.data; | ||||||
|  |     } catch (error) { | ||||||
|  |       console.log(error); | ||||||
|  |       return { results: [] }; | ||||||
|  |     } | ||||||
|  |   } | ||||||
|  |   // console.log(response)
 | ||||||
|  |   return data; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | function sleep(ms) { | ||||||
|  |   return new Promise((resolve) => setTimeout(resolve, ms)); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchMetaculusQuestionDescription(slug) { | ||||||
|  |   try { | ||||||
|  |     let response = await axios({ | ||||||
|  |       method: "get", | ||||||
|  |       url: "https://www.metaculus.com" + slug, | ||||||
|  |     }).then((response) => response.data); | ||||||
|  |     return response; | ||||||
|  |   } catch (error) { | ||||||
|  |     console.log(`Error in: fetchMetaculusQuestionDescription`); | ||||||
|  |     console.log( | ||||||
|  |       `We encountered some error when attempting to fetch a metaculus page. Trying again` | ||||||
|  |     ); | ||||||
|  |     if ( | ||||||
|  |       typeof error.response != "undefined" && | ||||||
|  |       typeof error.response.headers != "undefined" && | ||||||
|  |       typeof error.response.headers["retry-after"] != "undefined" | ||||||
|  |     ) { | ||||||
|  |       let timeout = error.response.headers["retry-after"]; | ||||||
|  |       console.log(`Timeout: ${timeout}`); | ||||||
|  |       await sleep(Number(timeout) * 1000 + SLEEP_TIME); | ||||||
|  |     } else { | ||||||
|  |       await sleep(SLEEP_TIME); | ||||||
|  |     } | ||||||
|  |     try { | ||||||
|  |       let response = await axios({ | ||||||
|  |         method: "get", | ||||||
|  |         url: "https://www.metaculus.com" + slug, | ||||||
|  |       }).then((response) => response.data); | ||||||
|  |       // console.log(response)
 | ||||||
|  |       return response; | ||||||
|  |     } catch (error) { | ||||||
|  |       console.log( | ||||||
|  |         `We encountered some error when attempting to fetch a metaculus page.` | ||||||
|  |       ); | ||||||
|  |       console.log("Error", error); | ||||||
|  |       throw "Giving up"; | ||||||
|  |     } | ||||||
|  |   } | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export const metaculus: Platform = { | ||||||
|  |   name: "metaculus", | ||||||
|  |   async fetcher() { | ||||||
|  |     // let metaculusQuestionsInit = await fetchMetaculusQuestions(1)
 | ||||||
|  |     // let numQueries = Math.round(Number(metaculusQuestionsInit.count) / 20)
 | ||||||
|  |     // console.log(`Downloading... This might take a while. Total number of queries: ${numQueries}`)
 | ||||||
|  |     // for (let i = 4; i <= numQueries; i++) { // change numQueries to 10 if one want to just test }
 | ||||||
|  |     let all_questions = []; | ||||||
|  |     let next = "https://www.metaculus.com/api2/questions/"; | ||||||
|  |     let i = 1; | ||||||
|  |     while (next) { | ||||||
|  |       if (i % 20 == 0) { | ||||||
|  |         console.log("Sleeping for 500ms"); | ||||||
|  |         await sleep(SLEEP_TIME); | ||||||
|  |       } | ||||||
|  |       console.log(`\nQuery #${i}`); | ||||||
|  |       let metaculusQuestions = await fetchMetaculusQuestions(next); | ||||||
|  |       let results = metaculusQuestions.results; | ||||||
|  |       let j = false; | ||||||
|  |       for (let result of results) { | ||||||
|  |         if (result.publish_time < now && now < result.resolve_time) { | ||||||
|  |           await sleep(SLEEP_TIME / 2); | ||||||
|  |           let questionPage = await fetchMetaculusQuestionDescription( | ||||||
|  |             result.page_url | ||||||
|  |           ); | ||||||
|  |           if (!questionPage.includes("A public prediction by")) { | ||||||
|  |             // console.log(questionPage)
 | ||||||
|  |             let descriptionraw = questionPage.split( | ||||||
|  |               `<div class="content" ng-bind-html-compile="qctrl.question.description_html">` | ||||||
|  |             )[1]; //.split(`<div class="question__content">`)[1]
 | ||||||
|  |             let descriptionprocessed1 = descriptionraw.split("</div>")[0]; | ||||||
|  |             let descriptionprocessed2 = toMarkdown(descriptionprocessed1); | ||||||
|  |             let description = descriptionprocessed2; | ||||||
|  | 
 | ||||||
|  |             let isbinary = result.possibilities.type == "binary"; | ||||||
|  |             let options = []; | ||||||
|  |             if (isbinary) { | ||||||
|  |               let probability = Number(result.community_prediction.full.q2); | ||||||
|  |               options = [ | ||||||
|  |                 { | ||||||
|  |                   name: "Yes", | ||||||
|  |                   probability: probability, | ||||||
|  |                   type: "PROBABILITY", | ||||||
|  |                 }, | ||||||
|  |                 { | ||||||
|  |                   name: "No", | ||||||
|  |                   probability: 1 - probability, | ||||||
|  |                   type: "PROBABILITY", | ||||||
|  |                 }, | ||||||
|  |               ]; | ||||||
|  |             } | ||||||
|  |             let id = `metaculus-${result.id}`; | ||||||
|  |             let interestingInfo = { | ||||||
|  |               id: id, | ||||||
|  |               title: result.title, | ||||||
|  |               url: "https://www.metaculus.com" + result.page_url, | ||||||
|  |               platform: "Metaculus", | ||||||
|  |               description: description, | ||||||
|  |               options: options, | ||||||
|  |               timestamp: new Date().toISOString(), | ||||||
|  |               qualityindicators: { | ||||||
|  |                 numforecasts: Number(result.number_of_predictions), | ||||||
|  |                 stars: calculateStars("Metaculus", { | ||||||
|  |                   numforecasts: result.number_of_predictions, | ||||||
|  |                 }), | ||||||
|  |               }, | ||||||
|  |               extra: { | ||||||
|  |                 resolution_data: { | ||||||
|  |                   publish_time: result.publish_time, | ||||||
|  |                   resolution: result.resolution, | ||||||
|  |                   close_time: result.close_time, | ||||||
|  |                   resolve_time: result.resolve_time, | ||||||
|  |                 }, | ||||||
|  |               }, | ||||||
|  |               //"status": result.status,
 | ||||||
|  |               //"publish_time": result.publish_time,
 | ||||||
|  |               //"close_time": result.close_time,
 | ||||||
|  |               //"type": result.possibilities.type, // We want binary ones here.
 | ||||||
|  |               //"last_activity_time": result.last_activity_time,
 | ||||||
|  |             }; | ||||||
|  |             if (Number(result.number_of_predictions) >= 10) { | ||||||
|  |               console.log(`- ${interestingInfo.title}`); | ||||||
|  |               all_questions.push(interestingInfo); | ||||||
|  |               if ((!j && i % 20 == 0) || DEBUG_MODE == "on") { | ||||||
|  |                 console.log(interestingInfo); | ||||||
|  |                 j = true; | ||||||
|  |               } | ||||||
|  |             } | ||||||
|  |           } else { | ||||||
|  |             console.log("- [Skipping public prediction]"); | ||||||
|  |           } | ||||||
|  |         } | ||||||
|  |       } | ||||||
|  |       next = metaculusQuestions.next; | ||||||
|  |       i = i + 1; | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     return all_questions; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -1,142 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| 
 |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| import { Forecast, PlatformFetcher } from "./"; |  | ||||||
| 
 |  | ||||||
| /* Definitions */ |  | ||||||
| let graphQLendpoint = |  | ||||||
|   "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; |  | ||||||
| let units = 10 ** 6; |  | ||||||
| 
 |  | ||||||
| async function fetchAllContractInfo() { |  | ||||||
|   // for info which the polymarket graphql API
 |  | ||||||
|   let response = await axios |  | ||||||
|     .get( |  | ||||||
|       "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1" |  | ||||||
|     ) |  | ||||||
|     .then((query) => query.data); |  | ||||||
|   response = response.filter((res) => res.closed != true); |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchAllContractData() { |  | ||||||
|   let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week
 |  | ||||||
|   let response = await axios({ |  | ||||||
|     url: graphQLendpoint, |  | ||||||
|     method: "POST", |  | ||||||
|     headers: { "Content-Type": "application/json" }, |  | ||||||
|     data: JSON.stringify({ |  | ||||||
|       query: ` |  | ||||||
|       { |  | ||||||
|           fixedProductMarketMakers(first: 1000 |  | ||||||
|           where: { |  | ||||||
|             lastActiveDay_gt: ${daysSinceEra} |  | ||||||
|           }){ |  | ||||||
|             id |  | ||||||
|             creator |  | ||||||
|             creationTimestamp |  | ||||||
|             fee |  | ||||||
|             tradesQuantity |  | ||||||
|             buysQuantity |  | ||||||
|             sellsQuantity |  | ||||||
|             lastActiveDay |  | ||||||
|             outcomeTokenPrices |  | ||||||
|             outcomeTokenAmounts |  | ||||||
|             liquidityParameter |  | ||||||
|             collateralBuyVolume |  | ||||||
|             collateralSellVolume |  | ||||||
|             conditions { |  | ||||||
|               outcomeSlotCount |  | ||||||
|             } |  | ||||||
|         } |  | ||||||
|       } |  | ||||||
|       `,
 |  | ||||||
|     }), |  | ||||||
|   }) |  | ||||||
|     .then((res) => res.data) |  | ||||||
|     .then((res) => res.data.fixedProductMarketMakers); |  | ||||||
| 
 |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| export const polymarket: PlatformFetcher = async function () { |  | ||||||
|   let allData = await fetchAllContractData(); |  | ||||||
|   let allInfo = await fetchAllContractInfo(); |  | ||||||
| 
 |  | ||||||
|   let used = process.memoryUsage().heapUsed / 1024 / 1024; |  | ||||||
|   console.log( |  | ||||||
|     `The script uses approximately ${Math.round(used * 100) / 100} MB` |  | ||||||
|   ); |  | ||||||
| 
 |  | ||||||
|   let infos = {}; |  | ||||||
|   for (let info of allInfo) { |  | ||||||
|     let address = info.marketMakerAddress; |  | ||||||
|     let addressLowerCase = address.toLowerCase(); |  | ||||||
| 
 |  | ||||||
|     if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") |  | ||||||
|       infos[addressLowerCase] = { |  | ||||||
|         title: info.question, |  | ||||||
|         url: "https://polymarket.com/market/" + info.slug, |  | ||||||
|         address: address, |  | ||||||
|         description: info.description, |  | ||||||
|         outcomes: info.outcomes, |  | ||||||
|         options: [], |  | ||||||
|         category: info.category, |  | ||||||
|       }; |  | ||||||
|   } |  | ||||||
| 
 |  | ||||||
|   let results = []; |  | ||||||
|   for (let data of allData) { |  | ||||||
|     let addressLowerCase = data.id; |  | ||||||
| 
 |  | ||||||
|     if (infos[addressLowerCase] != undefined) { |  | ||||||
|       let id = `polymarket-${addressLowerCase.slice(0, 10)}`; |  | ||||||
|       let info = infos[addressLowerCase]; |  | ||||||
|       let numforecasts = Number(data.tradesQuantity); |  | ||||||
|       let tradevolume = |  | ||||||
|         (Number(data.collateralBuyVolume) + Number(data.collateralSellVolume)) / |  | ||||||
|         units; |  | ||||||
|       let liquidity = Number(data.liquidityParameter) / units; |  | ||||||
|       // let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2
 |  | ||||||
|       // let percentage = Number(data.outcomeTokenPrices[0]) * 100
 |  | ||||||
|       // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none"
 |  | ||||||
|       let options = []; |  | ||||||
|       for (let outcome in data.outcomeTokenPrices) { |  | ||||||
|         options.push({ |  | ||||||
|           name: info.outcomes[outcome], |  | ||||||
|           probability: data.outcomeTokenPrices[outcome], |  | ||||||
|           type: "PROBABILITY", |  | ||||||
|         }); |  | ||||||
|       } |  | ||||||
| 
 |  | ||||||
|       let result: Forecast = { |  | ||||||
|         id: id, |  | ||||||
|         title: info.title, |  | ||||||
|         url: info.url, |  | ||||||
|         platform: "PolyMarket", |  | ||||||
|         description: info.description, |  | ||||||
|         options: options, |  | ||||||
|         timestamp: new Date().toISOString(), |  | ||||||
|         qualityindicators: { |  | ||||||
|           numforecasts: numforecasts.toFixed(0), |  | ||||||
|           liquidity: liquidity.toFixed(2), |  | ||||||
|           tradevolume: tradevolume.toFixed(2), |  | ||||||
|           stars: calculateStars("Polymarket", { |  | ||||||
|             liquidity, |  | ||||||
|             option: options[0], |  | ||||||
|             volume: tradevolume, |  | ||||||
|           }), |  | ||||||
|         }, |  | ||||||
|         extra: { |  | ||||||
|           address: info.address, |  | ||||||
|         }, |  | ||||||
|       }; |  | ||||||
|       if (info.category != "Sports") { |  | ||||||
|         results.push(result); |  | ||||||
|       } |  | ||||||
|     } |  | ||||||
|   } |  | ||||||
| 
 |  | ||||||
|   return results; |  | ||||||
| }; |  | ||||||
							
								
								
									
										146
									
								
								src/backend/platforms/polymarket.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										146
									
								
								src/backend/platforms/polymarket.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,146 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | 
 | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import { Forecast, Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Definitions */ | ||||||
|  | let graphQLendpoint = | ||||||
|  |   "https://api.thegraph.com/subgraphs/name/polymarket/matic-markets-5"; | ||||||
|  | let units = 10 ** 6; | ||||||
|  | 
 | ||||||
|  | async function fetchAllContractInfo() { | ||||||
|  |   // for info which the polymarket graphql API
 | ||||||
|  |   let response = await axios | ||||||
|  |     .get( | ||||||
|  |       "https://strapi-matic.poly.market/markets?active=true&_sort=volume:desc&_limit=-1" | ||||||
|  |     ) | ||||||
|  |     .then((query) => query.data); | ||||||
|  |   response = response.filter((res) => res.closed != true); | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchAllContractData() { | ||||||
|  |   let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 7; // last week
 | ||||||
|  |   let response = await axios({ | ||||||
|  |     url: graphQLendpoint, | ||||||
|  |     method: "POST", | ||||||
|  |     headers: { "Content-Type": "application/json" }, | ||||||
|  |     data: JSON.stringify({ | ||||||
|  |       query: ` | ||||||
|  |       { | ||||||
|  |           fixedProductMarketMakers(first: 1000 | ||||||
|  |           where: { | ||||||
|  |             lastActiveDay_gt: ${daysSinceEra} | ||||||
|  |           }){ | ||||||
|  |             id | ||||||
|  |             creator | ||||||
|  |             creationTimestamp | ||||||
|  |             fee | ||||||
|  |             tradesQuantity | ||||||
|  |             buysQuantity | ||||||
|  |             sellsQuantity | ||||||
|  |             lastActiveDay | ||||||
|  |             outcomeTokenPrices | ||||||
|  |             outcomeTokenAmounts | ||||||
|  |             liquidityParameter | ||||||
|  |             collateralBuyVolume | ||||||
|  |             collateralSellVolume | ||||||
|  |             conditions { | ||||||
|  |               outcomeSlotCount | ||||||
|  |             } | ||||||
|  |         } | ||||||
|  |       } | ||||||
|  |       `,
 | ||||||
|  |     }), | ||||||
|  |   }) | ||||||
|  |     .then((res) => res.data) | ||||||
|  |     .then((res) => res.data.fixedProductMarketMakers); | ||||||
|  | 
 | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export const polymarket: Platform = { | ||||||
|  |   name: "polymarket", | ||||||
|  |   async fetcher() { | ||||||
|  |     let allData = await fetchAllContractData(); | ||||||
|  |     let allInfo = await fetchAllContractInfo(); | ||||||
|  | 
 | ||||||
|  |     let used = process.memoryUsage().heapUsed / 1024 / 1024; | ||||||
|  |     console.log( | ||||||
|  |       `The script uses approximately ${Math.round(used * 100) / 100} MB` | ||||||
|  |     ); | ||||||
|  | 
 | ||||||
|  |     let infos = {}; | ||||||
|  |     for (let info of allInfo) { | ||||||
|  |       let address = info.marketMakerAddress; | ||||||
|  |       let addressLowerCase = address.toLowerCase(); | ||||||
|  | 
 | ||||||
|  |       if (info.outcomes[0] != "Long" || info.outcomes[1] != "Long") | ||||||
|  |         infos[addressLowerCase] = { | ||||||
|  |           title: info.question, | ||||||
|  |           url: "https://polymarket.com/market/" + info.slug, | ||||||
|  |           address: address, | ||||||
|  |           description: info.description, | ||||||
|  |           outcomes: info.outcomes, | ||||||
|  |           options: [], | ||||||
|  |           category: info.category, | ||||||
|  |         }; | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     let results = []; | ||||||
|  |     for (let data of allData) { | ||||||
|  |       let addressLowerCase = data.id; | ||||||
|  | 
 | ||||||
|  |       if (infos[addressLowerCase] != undefined) { | ||||||
|  |         let id = `polymarket-${addressLowerCase.slice(0, 10)}`; | ||||||
|  |         let info = infos[addressLowerCase]; | ||||||
|  |         let numforecasts = Number(data.tradesQuantity); | ||||||
|  |         let tradevolume = | ||||||
|  |           (Number(data.collateralBuyVolume) + | ||||||
|  |             Number(data.collateralSellVolume)) / | ||||||
|  |           units; | ||||||
|  |         let liquidity = Number(data.liquidityParameter) / units; | ||||||
|  |         // let isbinary = Number(data.conditions[0].outcomeSlotCount) == 2
 | ||||||
|  |         // let percentage = Number(data.outcomeTokenPrices[0]) * 100
 | ||||||
|  |         // let percentageFormatted = isbinary ? (percentage.toFixed(0) + "%") : "none"
 | ||||||
|  |         let options = []; | ||||||
|  |         for (let outcome in data.outcomeTokenPrices) { | ||||||
|  |           options.push({ | ||||||
|  |             name: info.outcomes[outcome], | ||||||
|  |             probability: data.outcomeTokenPrices[outcome], | ||||||
|  |             type: "PROBABILITY", | ||||||
|  |           }); | ||||||
|  |         } | ||||||
|  | 
 | ||||||
|  |         let result: Forecast = { | ||||||
|  |           id: id, | ||||||
|  |           title: info.title, | ||||||
|  |           url: info.url, | ||||||
|  |           platform: "PolyMarket", | ||||||
|  |           description: info.description, | ||||||
|  |           options: options, | ||||||
|  |           timestamp: new Date().toISOString(), | ||||||
|  |           qualityindicators: { | ||||||
|  |             numforecasts: numforecasts.toFixed(0), | ||||||
|  |             liquidity: liquidity.toFixed(2), | ||||||
|  |             tradevolume: tradevolume.toFixed(2), | ||||||
|  |             stars: calculateStars("Polymarket", { | ||||||
|  |               liquidity, | ||||||
|  |               option: options[0], | ||||||
|  |               volume: tradevolume, | ||||||
|  |             }), | ||||||
|  |           }, | ||||||
|  |           extra: { | ||||||
|  |             address: info.address, | ||||||
|  |           }, | ||||||
|  |         }; | ||||||
|  |         if (info.category != "Sports") { | ||||||
|  |           results.push(result); | ||||||
|  |         } | ||||||
|  |       } | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -1,112 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| 
 |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| import toMarkdown from "../utils/toMarkdown"; |  | ||||||
| import { PlatformFetcher } from "./"; |  | ||||||
| 
 |  | ||||||
| /* Support functions */ |  | ||||||
| async function fetchmarkets() { |  | ||||||
|   let response = await axios({ |  | ||||||
|     method: "get", |  | ||||||
|     url: "https://www.predictit.org/api/marketdata/all/", |  | ||||||
|   }); |  | ||||||
|   let openMarkets = response.data.markets.filter( |  | ||||||
|     (market) => market.status == "Open" |  | ||||||
|   ); |  | ||||||
|   return openMarkets; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchmarketrules(market_id) { |  | ||||||
|   let response = await axios({ |  | ||||||
|     method: "get", |  | ||||||
|     url: "https://www.predictit.org/api/Market/" + market_id, |  | ||||||
|   }); |  | ||||||
|   return response.data.rule; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchmarketvolumes() { |  | ||||||
|   let response = await axios({ |  | ||||||
|     method: "get", |  | ||||||
|     url: "https://predictit-f497e.firebaseio.com/marketStats.json", |  | ||||||
|   }); |  | ||||||
|   return response.data; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| function sleep(ms) { |  | ||||||
|   return new Promise((resolve) => setTimeout(resolve, ms)); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| /* Body */ |  | ||||||
| export const predictit: PlatformFetcher = async function () { |  | ||||||
|   let markets = await fetchmarkets(); |  | ||||||
|   let marketVolumes = await fetchmarketvolumes(); |  | ||||||
| 
 |  | ||||||
|   markets = markets.map((market) => ({ |  | ||||||
|     ...market, |  | ||||||
|     TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"], |  | ||||||
|   })); |  | ||||||
|   // console.log(markets)
 |  | ||||||
| 
 |  | ||||||
|   let results = []; |  | ||||||
|   for (let market of markets) { |  | ||||||
|     // console.log(market.name)
 |  | ||||||
|     let id = `predictit-${market.id}`; |  | ||||||
|     let isbinary = market.contracts.length == 1; |  | ||||||
|     await sleep(3000 * (1 + Math.random())); |  | ||||||
|     let descriptionraw = await fetchmarketrules(market.id); |  | ||||||
|     let descriptionprocessed1 = toMarkdown(descriptionraw); |  | ||||||
|     let description = descriptionprocessed1; |  | ||||||
|     let shares_volume = market["TotalSharesTraded"]; |  | ||||||
|     // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none"
 |  | ||||||
| 
 |  | ||||||
|     let options = market.contracts.map((contract) => ({ |  | ||||||
|       name: contract.name, |  | ||||||
|       probability: contract.lastTradePrice, |  | ||||||
|       type: "PROBABILITY", |  | ||||||
|     })); |  | ||||||
|     let totalValue = options |  | ||||||
|       .map((element) => Number(element.probability)) |  | ||||||
|       .reduce((a, b) => a + b, 0); |  | ||||||
| 
 |  | ||||||
|     if (options.length != 1 && totalValue > 1) { |  | ||||||
|       options = options.map((element) => ({ |  | ||||||
|         ...element, |  | ||||||
|         probability: Number(element.probability) / totalValue, |  | ||||||
|       })); |  | ||||||
|     } else if (options.length == 1) { |  | ||||||
|       let option = options[0]; |  | ||||||
|       let probability = option["probability"]; |  | ||||||
|       options = [ |  | ||||||
|         { |  | ||||||
|           name: "Yes", |  | ||||||
|           probability: probability, |  | ||||||
|           type: "PROBABILITY", |  | ||||||
|         }, |  | ||||||
|         { |  | ||||||
|           name: "No", |  | ||||||
|           probability: 1 - probability, |  | ||||||
|           type: "PROBABILITY", |  | ||||||
|         }, |  | ||||||
|       ]; |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     let obj = { |  | ||||||
|       id: id, |  | ||||||
|       title: market["name"], |  | ||||||
|       url: market.url, |  | ||||||
|       platform: "PredictIt", |  | ||||||
|       description: description, |  | ||||||
|       options: options, |  | ||||||
|       timestamp: new Date().toISOString(), |  | ||||||
|       qualityindicators: { |  | ||||||
|         stars: calculateStars("PredictIt", {}), |  | ||||||
|         shares_volume: shares_volume, |  | ||||||
|       }, |  | ||||||
|     }; |  | ||||||
|     // console.log(obj)
 |  | ||||||
|     results.push(obj); |  | ||||||
|   } |  | ||||||
| 
 |  | ||||||
|   return results; |  | ||||||
| }; |  | ||||||
							
								
								
									
										115
									
								
								src/backend/platforms/predictit.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										115
									
								
								src/backend/platforms/predictit.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,115 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | 
 | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import toMarkdown from "../utils/toMarkdown"; | ||||||
|  | import { Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Support functions */ | ||||||
|  | async function fetchmarkets() { | ||||||
|  |   let response = await axios({ | ||||||
|  |     method: "get", | ||||||
|  |     url: "https://www.predictit.org/api/marketdata/all/", | ||||||
|  |   }); | ||||||
|  |   let openMarkets = response.data.markets.filter( | ||||||
|  |     (market) => market.status == "Open" | ||||||
|  |   ); | ||||||
|  |   return openMarkets; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchmarketrules(market_id) { | ||||||
|  |   let response = await axios({ | ||||||
|  |     method: "get", | ||||||
|  |     url: "https://www.predictit.org/api/Market/" + market_id, | ||||||
|  |   }); | ||||||
|  |   return response.data.rule; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchmarketvolumes() { | ||||||
|  |   let response = await axios({ | ||||||
|  |     method: "get", | ||||||
|  |     url: "https://predictit-f497e.firebaseio.com/marketStats.json", | ||||||
|  |   }); | ||||||
|  |   return response.data; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | function sleep(ms: number) { | ||||||
|  |   return new Promise((resolve) => setTimeout(resolve, ms)); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | /* Body */ | ||||||
|  | export const predictit: Platform = { | ||||||
|  |   name: "predictit", | ||||||
|  |   async fetcher() { | ||||||
|  |     let markets = await fetchmarkets(); | ||||||
|  |     let marketVolumes = await fetchmarketvolumes(); | ||||||
|  | 
 | ||||||
|  |     markets = markets.map((market) => ({ | ||||||
|  |       ...market, | ||||||
|  |       TotalSharesTraded: marketVolumes[market.id]["TotalSharesTraded"], | ||||||
|  |     })); | ||||||
|  |     // console.log(markets)
 | ||||||
|  | 
 | ||||||
|  |     let results = []; | ||||||
|  |     for (let market of markets) { | ||||||
|  |       // console.log(market.name)
 | ||||||
|  |       let id = `predictit-${market.id}`; | ||||||
|  |       let isbinary = market.contracts.length == 1; | ||||||
|  |       await sleep(3000 * (1 + Math.random())); | ||||||
|  |       let descriptionraw = await fetchmarketrules(market.id); | ||||||
|  |       let descriptionprocessed1 = toMarkdown(descriptionraw); | ||||||
|  |       let description = descriptionprocessed1; | ||||||
|  |       let shares_volume = market["TotalSharesTraded"]; | ||||||
|  |       // let percentageFormatted = isbinary ? Number(Number(market.contracts[0].lastTradePrice) * 100).toFixed(0) + "%" : "none"
 | ||||||
|  | 
 | ||||||
|  |       let options = market.contracts.map((contract) => ({ | ||||||
|  |         name: contract.name, | ||||||
|  |         probability: contract.lastTradePrice, | ||||||
|  |         type: "PROBABILITY", | ||||||
|  |       })); | ||||||
|  |       let totalValue = options | ||||||
|  |         .map((element) => Number(element.probability)) | ||||||
|  |         .reduce((a, b) => a + b, 0); | ||||||
|  | 
 | ||||||
|  |       if (options.length != 1 && totalValue > 1) { | ||||||
|  |         options = options.map((element) => ({ | ||||||
|  |           ...element, | ||||||
|  |           probability: Number(element.probability) / totalValue, | ||||||
|  |         })); | ||||||
|  |       } else if (options.length == 1) { | ||||||
|  |         let option = options[0]; | ||||||
|  |         let probability = option["probability"]; | ||||||
|  |         options = [ | ||||||
|  |           { | ||||||
|  |             name: "Yes", | ||||||
|  |             probability: probability, | ||||||
|  |             type: "PROBABILITY", | ||||||
|  |           }, | ||||||
|  |           { | ||||||
|  |             name: "No", | ||||||
|  |             probability: 1 - probability, | ||||||
|  |             type: "PROBABILITY", | ||||||
|  |           }, | ||||||
|  |         ]; | ||||||
|  |       } | ||||||
|  | 
 | ||||||
|  |       let obj = { | ||||||
|  |         id: id, | ||||||
|  |         title: market["name"], | ||||||
|  |         url: market.url, | ||||||
|  |         platform: "PredictIt", | ||||||
|  |         description: description, | ||||||
|  |         options: options, | ||||||
|  |         timestamp: new Date().toISOString(), | ||||||
|  |         qualityindicators: { | ||||||
|  |           stars: calculateStars("PredictIt", {}), | ||||||
|  |           shares_volume: shares_volume, | ||||||
|  |         }, | ||||||
|  |       }; | ||||||
|  |       // console.log(obj)
 | ||||||
|  |       results.push(obj); | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -1,57 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| 
 |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| import toMarkdown from "../utils/toMarkdown"; |  | ||||||
| import { PlatformFetcher } from "./"; |  | ||||||
| 
 |  | ||||||
| /* Definitions */ |  | ||||||
| let jsonEndpoint = |  | ||||||
|   "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 |  | ||||||
| 
 |  | ||||||
| async function fetchAllRootclaims() { |  | ||||||
|   // for info which the polymarket graphql API
 |  | ||||||
|   let response = await axios |  | ||||||
|     .get(jsonEndpoint) |  | ||||||
|     .then((response) => response.data); |  | ||||||
|   if (response.length != response[0] + 1) { |  | ||||||
|     console.log(response.length); |  | ||||||
|     console.log(response[0]); |  | ||||||
|     //throw Error("Rootclaim's backend has changed.")
 |  | ||||||
|   } |  | ||||||
|   response.shift(); |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| export const rootclaim: PlatformFetcher = async function () { |  | ||||||
|   let claims = await fetchAllRootclaims(); |  | ||||||
|   let results = []; |  | ||||||
|   for (let claim of claims) { |  | ||||||
|     let id = `rootclaim-${claim.slug.toLowerCase()}`; |  | ||||||
|     let options = []; |  | ||||||
|     for (let scenario of claim.scenarios) { |  | ||||||
|       //console.log(scenario)
 |  | ||||||
|       options.push({ |  | ||||||
|         name: toMarkdown(scenario.text).replace("\n", "").replace("'", "'"), |  | ||||||
|         probability: scenario.net_prob / 100, |  | ||||||
|         type: "PROBABILITY", |  | ||||||
|       }); |  | ||||||
|     } |  | ||||||
|     let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"; |  | ||||||
|     let obj = { |  | ||||||
|       id: id, |  | ||||||
|       title: toMarkdown(claim.question).replace("\n", ""), |  | ||||||
|       url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, |  | ||||||
|       platform: "Rootclaim", |  | ||||||
|       description: toMarkdown(claim.background).replace("'", "'"), |  | ||||||
|       options: options, |  | ||||||
|       timestamp: new Date().toISOString(), |  | ||||||
|       qualityindicators: { |  | ||||||
|         numforecasts: 1, |  | ||||||
|         stars: calculateStars("Rootclaim", {}), |  | ||||||
|       }, |  | ||||||
|     }; |  | ||||||
|     results.push(obj); |  | ||||||
|   } |  | ||||||
|   return results; |  | ||||||
| }; |  | ||||||
							
								
								
									
										62
									
								
								src/backend/platforms/rootclaim.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										62
									
								
								src/backend/platforms/rootclaim.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,62 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | 
 | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import toMarkdown from "../utils/toMarkdown"; | ||||||
|  | import { Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Definitions */ | ||||||
|  | let jsonEndpoint = | ||||||
|  |   "https://www.rootclaim.com/main_page_stories?number=100&offset=0"; //"https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket"//"https://subgraph-backup.poly.market/subgraphs/name/TokenUnion/polymarket"//'https://subgraph-matic.poly.market/subgraphs/name/TokenUnion/polymarket3'
 | ||||||
|  | 
 | ||||||
|  | async function fetchAllRootclaims() { | ||||||
|  |   // for info which the polymarket graphql API
 | ||||||
|  |   let response = await axios | ||||||
|  |     .get(jsonEndpoint) | ||||||
|  |     .then((response) => response.data); | ||||||
|  |   if (response.length != response[0] + 1) { | ||||||
|  |     console.log(response.length); | ||||||
|  |     console.log(response[0]); | ||||||
|  |     //throw Error("Rootclaim's backend has changed.")
 | ||||||
|  |   } | ||||||
|  |   response.shift(); | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export const rootclaim: Platform = { | ||||||
|  |   name: "rootclaim", | ||||||
|  |   async fetcher() { | ||||||
|  |     let claims = await fetchAllRootclaims(); | ||||||
|  |     let results = []; | ||||||
|  |     for (let claim of claims) { | ||||||
|  |       let id = `rootclaim-${claim.slug.toLowerCase()}`; | ||||||
|  |       let options = []; | ||||||
|  |       for (let scenario of claim.scenarios) { | ||||||
|  |         //console.log(scenario)
 | ||||||
|  |         options.push({ | ||||||
|  |           name: toMarkdown(scenario.text) | ||||||
|  |             .replace("\n", "") | ||||||
|  |             .replace("'", "'"), | ||||||
|  |           probability: scenario.net_prob / 100, | ||||||
|  |           type: "PROBABILITY", | ||||||
|  |         }); | ||||||
|  |       } | ||||||
|  |       let claimUrlPath = claim.created_at < "2020" ? "claims" : "analysis"; | ||||||
|  |       let obj = { | ||||||
|  |         id: id, | ||||||
|  |         title: toMarkdown(claim.question).replace("\n", ""), | ||||||
|  |         url: `https://www.rootclaim.com/${claimUrlPath}/${claim.slug}`, | ||||||
|  |         platform: "Rootclaim", | ||||||
|  |         description: toMarkdown(claim.background).replace("'", "'"), | ||||||
|  |         options: options, | ||||||
|  |         timestamp: new Date().toISOString(), | ||||||
|  |         qualityindicators: { | ||||||
|  |           numforecasts: 1, | ||||||
|  |           stars: calculateStars("Rootclaim", {}), | ||||||
|  |         }, | ||||||
|  |       }; | ||||||
|  |       results.push(obj); | ||||||
|  |     } | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -1,177 +0,0 @@ | ||||||
| /* Imports */ |  | ||||||
| import axios from "axios"; |  | ||||||
| 
 |  | ||||||
| import { calculateStars } from "../utils/stars"; |  | ||||||
| import { PlatformFetcher } from "./"; |  | ||||||
| 
 |  | ||||||
| /* Definitions */ |  | ||||||
| let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; |  | ||||||
| let VERBOSE = false; |  | ||||||
| let empty = () => 0; |  | ||||||
| /* Support functions */ |  | ||||||
| 
 |  | ||||||
| async function fetchEvents(url) { |  | ||||||
|   let response = await axios({ |  | ||||||
|     url: htmlEndPointEntrance + url, |  | ||||||
|     method: "GET", |  | ||||||
|     headers: { |  | ||||||
|       "Content-Type": "text/html", |  | ||||||
|     }, |  | ||||||
|   }).then((res) => res.data); |  | ||||||
|   VERBOSE ? console.log(response) : empty(); |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchMarkets(eventid) { |  | ||||||
|   let response = await axios({ |  | ||||||
|     url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, |  | ||||||
|     method: "GET", |  | ||||||
|     headers: { |  | ||||||
|       "Content-Type": "text/json", |  | ||||||
|     }, |  | ||||||
|   }) |  | ||||||
|     .then((res) => res.data) |  | ||||||
|     .then((res) => res.markets); |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchContracts(marketid) { |  | ||||||
|   let response = await axios({ |  | ||||||
|     url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, |  | ||||||
|     method: "GET", |  | ||||||
|     headers: { |  | ||||||
|       "Content-Type": "text/html", |  | ||||||
|     }, |  | ||||||
|   }).then((res) => res.data); |  | ||||||
|   VERBOSE ? console.log(response) : empty(); |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| async function fetchPrices(marketid) { |  | ||||||
|   let response = await axios({ |  | ||||||
|     url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, |  | ||||||
|     method: "GET", |  | ||||||
|     headers: { |  | ||||||
|       "Content-Type": "text/html", |  | ||||||
|     }, |  | ||||||
|   }).then((res) => res.data); |  | ||||||
|   VERBOSE ? console.log(response) : empty(); |  | ||||||
|   return response; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| /* Body */ |  | ||||||
| 
 |  | ||||||
| export const smarkets: PlatformFetcher = async function () { |  | ||||||
|   let htmlPath = |  | ||||||
|     "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; |  | ||||||
| 
 |  | ||||||
|   let events = []; |  | ||||||
|   while (htmlPath) { |  | ||||||
|     let data = await fetchEvents(htmlPath); |  | ||||||
|     events.push(...data.events); |  | ||||||
|     htmlPath = data.pagination.next_page; |  | ||||||
|   } |  | ||||||
|   VERBOSE ? console.log(events) : empty(); |  | ||||||
|   let markets = []; |  | ||||||
|   for (let event of events) { |  | ||||||
|     VERBOSE ? console.log(Date.now()) : empty(); |  | ||||||
|     VERBOSE ? console.log(event.name) : empty(); |  | ||||||
|     let eventMarkets = await fetchMarkets(event.id); |  | ||||||
|     eventMarkets = eventMarkets.map((market) => ({ |  | ||||||
|       ...market, |  | ||||||
|       slug: event.full_slug, |  | ||||||
|     })); |  | ||||||
|     VERBOSE ? console.log("Markets fetched") : empty(); |  | ||||||
|     VERBOSE ? console.log(event.id) : empty(); |  | ||||||
|     VERBOSE ? console.log(eventMarkets) : empty(); |  | ||||||
|     markets.push(...eventMarkets); |  | ||||||
|     //let lastPrices = await fetchPrices(market.id)
 |  | ||||||
|   } |  | ||||||
|   VERBOSE ? console.log(markets) : empty(); |  | ||||||
| 
 |  | ||||||
|   let results = []; |  | ||||||
|   for (let market of markets) { |  | ||||||
|     VERBOSE ? console.log("================") : empty(); |  | ||||||
|     VERBOSE ? console.log("Market: ", market) : empty(); |  | ||||||
|     let id = `smarkets-${market.id}`; |  | ||||||
|     let name = market.name; |  | ||||||
| 
 |  | ||||||
|     let contracts = await fetchContracts(market.id); |  | ||||||
|     VERBOSE ? console.log("Contracts: ", contracts) : empty(); |  | ||||||
|     let prices = await fetchPrices(market.id); |  | ||||||
|     VERBOSE |  | ||||||
|       ? console.log("Prices: ", prices["last_executed_prices"][market.id]) |  | ||||||
|       : empty(); |  | ||||||
| 
 |  | ||||||
|     let optionsObj = {}; |  | ||||||
|     for (let contract of contracts["contracts"]) { |  | ||||||
|       optionsObj[contract.id] = { name: contract.name }; |  | ||||||
|     } |  | ||||||
|     for (let price of prices["last_executed_prices"][market.id]) { |  | ||||||
|       optionsObj[price.contract_id] = { |  | ||||||
|         ...optionsObj[price.contract_id], |  | ||||||
|         probability: price.last_executed_price |  | ||||||
|           ? Number(price.last_executed_price) |  | ||||||
|           : null, |  | ||||||
|         type: "PROBABILITY", |  | ||||||
|       }; |  | ||||||
|     } |  | ||||||
|     let options: any[] = Object.values(optionsObj); |  | ||||||
|     // monkey patch the case where there are only two options and only one has traded.
 |  | ||||||
|     if ( |  | ||||||
|       options.length == 2 && |  | ||||||
|       options.map((option) => option.probability).includes(null) |  | ||||||
|     ) { |  | ||||||
|       let nonNullPrice = |  | ||||||
|         options[0].probability == null |  | ||||||
|           ? options[1].probability |  | ||||||
|           : options[0].probability; |  | ||||||
|       options = options.map((option) => { |  | ||||||
|         let probability = option.probability; |  | ||||||
|         return { |  | ||||||
|           ...option, |  | ||||||
|           probability: probability == null ? 100 - nonNullPrice : probability, |  | ||||||
|           // yes, 100, because prices are not yet normalized.
 |  | ||||||
|         }; |  | ||||||
|       }); |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     // Normalize normally
 |  | ||||||
|     let totalValue = options |  | ||||||
|       .map((element) => Number(element.probability)) |  | ||||||
|       .reduce((a, b) => a + b, 0); |  | ||||||
| 
 |  | ||||||
|     options = options.map((element) => ({ |  | ||||||
|       ...element, |  | ||||||
|       probability: Number(element.probability) / totalValue, |  | ||||||
|     })); |  | ||||||
|     VERBOSE ? console.log(options) : empty(); |  | ||||||
| 
 |  | ||||||
|     /* |  | ||||||
|     if(contracts["contracts"].length == 2){ |  | ||||||
|       isBinary = true |  | ||||||
|       percentage = ( Number(prices["last_executed_prices"][market.id][0].last_executed_price) + (100 - Number(prices["last_executed_prices"][market.id][1].last_executed_price)) ) / 2 |  | ||||||
|       percentage = Math.round(percentage)+"%" |  | ||||||
|       let contractName = contracts["contracts"][0].name |  | ||||||
|       name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) |  | ||||||
|     } |  | ||||||
|     */ |  | ||||||
|     let result = { |  | ||||||
|       id: id, |  | ||||||
|       title: name, |  | ||||||
|       url: "https://smarkets.com/event/" + market.event_id + market.slug, |  | ||||||
|       platform: "Smarkets", |  | ||||||
|       description: market.description, |  | ||||||
|       options: options, |  | ||||||
|       timestamp: new Date().toISOString(), |  | ||||||
|       qualityindicators: { |  | ||||||
|         stars: calculateStars("Smarkets", {}), |  | ||||||
|       }, |  | ||||||
|     }; |  | ||||||
|     VERBOSE ? console.log(result) : empty(); |  | ||||||
|     results.push(result); |  | ||||||
|   } |  | ||||||
|   VERBOSE ? console.log(results) : empty(); |  | ||||||
|   return results; |  | ||||||
| }; |  | ||||||
| //smarkets()
 |  | ||||||
							
								
								
									
										177
									
								
								src/backend/platforms/smarkets.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										177
									
								
								src/backend/platforms/smarkets.ts
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,177 @@ | ||||||
|  | /* Imports */ | ||||||
|  | import axios from "axios"; | ||||||
|  | 
 | ||||||
|  | import { calculateStars } from "../utils/stars"; | ||||||
|  | import { Platform } from "./"; | ||||||
|  | 
 | ||||||
|  | /* Definitions */ | ||||||
|  | let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; | ||||||
|  | let VERBOSE = false; | ||||||
|  | let empty = () => 0; | ||||||
|  | /* Support functions */ | ||||||
|  | 
 | ||||||
|  | async function fetchEvents(url) { | ||||||
|  |   let response = await axios({ | ||||||
|  |     url: htmlEndPointEntrance + url, | ||||||
|  |     method: "GET", | ||||||
|  |     headers: { | ||||||
|  |       "Content-Type": "text/html", | ||||||
|  |     }, | ||||||
|  |   }).then((res) => res.data); | ||||||
|  |   VERBOSE ? console.log(response) : empty(); | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchMarkets(eventid) { | ||||||
|  |   let response = await axios({ | ||||||
|  |     url: `https://api.smarkets.com/v3/events/${eventid}/markets/`, | ||||||
|  |     method: "GET", | ||||||
|  |     headers: { | ||||||
|  |       "Content-Type": "text/json", | ||||||
|  |     }, | ||||||
|  |   }) | ||||||
|  |     .then((res) => res.data) | ||||||
|  |     .then((res) => res.markets); | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchContracts(marketid) { | ||||||
|  |   let response = await axios({ | ||||||
|  |     url: `https://api.smarkets.com/v3/markets/${marketid}/contracts/`, | ||||||
|  |     method: "GET", | ||||||
|  |     headers: { | ||||||
|  |       "Content-Type": "text/html", | ||||||
|  |     }, | ||||||
|  |   }).then((res) => res.data); | ||||||
|  |   VERBOSE ? console.log(response) : empty(); | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function fetchPrices(marketid) { | ||||||
|  |   let response = await axios({ | ||||||
|  |     url: `https://api.smarkets.com/v3/markets/${marketid}/last_executed_prices/`, | ||||||
|  |     method: "GET", | ||||||
|  |     headers: { | ||||||
|  |       "Content-Type": "text/html", | ||||||
|  |     }, | ||||||
|  |   }).then((res) => res.data); | ||||||
|  |   VERBOSE ? console.log(response) : empty(); | ||||||
|  |   return response; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export const smarkets: Platform = { | ||||||
|  |   name: "smarkets", | ||||||
|  |   async fetcher() { | ||||||
|  |     let htmlPath = | ||||||
|  |       "?state=new&state=upcoming&state=live&type_domain=politics&type_scope=single_event&with_new_type=true&sort=id&limit=50"; | ||||||
|  | 
 | ||||||
|  |     let events = []; | ||||||
|  |     while (htmlPath) { | ||||||
|  |       let data = await fetchEvents(htmlPath); | ||||||
|  |       events.push(...data.events); | ||||||
|  |       htmlPath = data.pagination.next_page; | ||||||
|  |     } | ||||||
|  |     VERBOSE ? console.log(events) : empty(); | ||||||
|  |     let markets = []; | ||||||
|  |     for (let event of events) { | ||||||
|  |       VERBOSE ? console.log(Date.now()) : empty(); | ||||||
|  |       VERBOSE ? console.log(event.name) : empty(); | ||||||
|  |       let eventMarkets = await fetchMarkets(event.id); | ||||||
|  |       eventMarkets = eventMarkets.map((market) => ({ | ||||||
|  |         ...market, | ||||||
|  |         slug: event.full_slug, | ||||||
|  |       })); | ||||||
|  |       VERBOSE ? console.log("Markets fetched") : empty(); | ||||||
|  |       VERBOSE ? console.log(event.id) : empty(); | ||||||
|  |       VERBOSE ? console.log(eventMarkets) : empty(); | ||||||
|  |       markets.push(...eventMarkets); | ||||||
|  |       //let lastPrices = await fetchPrices(market.id)
 | ||||||
|  |     } | ||||||
|  |     VERBOSE ? console.log(markets) : empty(); | ||||||
|  | 
 | ||||||
|  |     let results = []; | ||||||
|  |     for (let market of markets) { | ||||||
|  |       VERBOSE ? console.log("================") : empty(); | ||||||
|  |       VERBOSE ? console.log("Market: ", market) : empty(); | ||||||
|  |       let id = `smarkets-${market.id}`; | ||||||
|  |       let name = market.name; | ||||||
|  | 
 | ||||||
|  |       let contracts = await fetchContracts(market.id); | ||||||
|  |       VERBOSE ? console.log("Contracts: ", contracts) : empty(); | ||||||
|  |       let prices = await fetchPrices(market.id); | ||||||
|  |       VERBOSE | ||||||
|  |         ? console.log("Prices: ", prices["last_executed_prices"][market.id]) | ||||||
|  |         : empty(); | ||||||
|  | 
 | ||||||
|  |       let optionsObj = {}; | ||||||
|  |       for (let contract of contracts["contracts"]) { | ||||||
|  |         optionsObj[contract.id] = { name: contract.name }; | ||||||
|  |       } | ||||||
|  |       for (let price of prices["last_executed_prices"][market.id]) { | ||||||
|  |         optionsObj[price.contract_id] = { | ||||||
|  |           ...optionsObj[price.contract_id], | ||||||
|  |           probability: price.last_executed_price | ||||||
|  |             ? Number(price.last_executed_price) | ||||||
|  |             : null, | ||||||
|  |           type: "PROBABILITY", | ||||||
|  |         }; | ||||||
|  |       } | ||||||
|  |       let options: any[] = Object.values(optionsObj); | ||||||
|  |       // monkey patch the case where there are only two options and only one has traded.
 | ||||||
|  |       if ( | ||||||
|  |         options.length == 2 && | ||||||
|  |         options.map((option) => option.probability).includes(null) | ||||||
|  |       ) { | ||||||
|  |         let nonNullPrice = | ||||||
|  |           options[0].probability == null | ||||||
|  |             ? options[1].probability | ||||||
|  |             : options[0].probability; | ||||||
|  |         options = options.map((option) => { | ||||||
|  |           let probability = option.probability; | ||||||
|  |           return { | ||||||
|  |             ...option, | ||||||
|  |             probability: probability == null ? 100 - nonNullPrice : probability, | ||||||
|  |             // yes, 100, because prices are not yet normalized.
 | ||||||
|  |           }; | ||||||
|  |         }); | ||||||
|  |       } | ||||||
|  | 
 | ||||||
|  |       // Normalize normally
 | ||||||
|  |       let totalValue = options | ||||||
|  |         .map((element) => Number(element.probability)) | ||||||
|  |         .reduce((a, b) => a + b, 0); | ||||||
|  | 
 | ||||||
|  |       options = options.map((element) => ({ | ||||||
|  |         ...element, | ||||||
|  |         probability: Number(element.probability) / totalValue, | ||||||
|  |       })); | ||||||
|  |       VERBOSE ? console.log(options) : empty(); | ||||||
|  | 
 | ||||||
|  |       /* | ||||||
|  |     if(contracts["contracts"].length == 2){ | ||||||
|  |       isBinary = true | ||||||
|  |       percentage = ( Number(prices["last_executed_prices"][market.id][0].last_executed_price) + (100 - Number(prices["last_executed_prices"][market.id][1].last_executed_price)) ) / 2 | ||||||
|  |       percentage = Math.round(percentage)+"%" | ||||||
|  |       let contractName = contracts["contracts"][0].name | ||||||
|  |       name = name+ (contractName=="Yes"?'':` (${contracts["contracts"][0].name})`) | ||||||
|  |     } | ||||||
|  |     */ | ||||||
|  |       let result = { | ||||||
|  |         id: id, | ||||||
|  |         title: name, | ||||||
|  |         url: "https://smarkets.com/event/" + market.event_id + market.slug, | ||||||
|  |         platform: "Smarkets", | ||||||
|  |         description: market.description, | ||||||
|  |         options: options, | ||||||
|  |         timestamp: new Date().toISOString(), | ||||||
|  |         qualityindicators: { | ||||||
|  |           stars: calculateStars("Smarkets", {}), | ||||||
|  |         }, | ||||||
|  |       }; | ||||||
|  |       VERBOSE ? console.log(result) : empty(); | ||||||
|  |       results.push(result); | ||||||
|  |     } | ||||||
|  |     VERBOSE ? console.log(results) : empty(); | ||||||
|  |     return results; | ||||||
|  |   }, | ||||||
|  | }; | ||||||
|  | @ -1,11 +1,10 @@ | ||||||
| /* Imports */ | /* Imports */ | ||||||
| // import axios from "axios"
 |  | ||||||
| import { GoogleSpreadsheet } from "google-spreadsheet"; | import { GoogleSpreadsheet } from "google-spreadsheet"; | ||||||
| 
 | 
 | ||||||
| import { applyIfSecretExists } from "../utils/getSecrets"; | import { applyIfSecretExists } from "../utils/getSecrets"; | ||||||
| import { hash } from "../utils/hash"; | import { hash } from "../utils/hash"; | ||||||
| import { calculateStars } from "../utils/stars"; | import { calculateStars } from "../utils/stars"; | ||||||
| import { PlatformFetcher } from "./"; | import { Platform } from "./"; | ||||||
| 
 | 
 | ||||||
| /* Definitions */ | /* Definitions */ | ||||||
| const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
 | const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
 | ||||||
|  | @ -113,18 +112,17 @@ async function processPredictions(predictions) { | ||||||
|     uniqueTitles.push(result.title); |     uniqueTitles.push(result.title); | ||||||
|   }); |   }); | ||||||
|   return uniqueResults; |   return uniqueResults; | ||||||
|   // console.log(results)
 |  | ||||||
|   // console.log(results.map(result => result.options))
 |  | ||||||
|   // processPredictions()
 |  | ||||||
| } | } | ||||||
| /* Body */ | 
 | ||||||
| export async function wildeford_inner(google_api_key) { | export async function wildeford_inner(google_api_key) { | ||||||
|   let predictions = await fetchGoogleDoc(google_api_key); |   let predictions = await fetchGoogleDoc(google_api_key); | ||||||
|   return await processPredictions(predictions); |   return await processPredictions(predictions); | ||||||
| } | } | ||||||
| //example()
 |  | ||||||
| 
 | 
 | ||||||
| export const wildeford: PlatformFetcher = async function () { | export const wildeford: Platform = { | ||||||
|   const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
 |   name: "wildeford", | ||||||
|   return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); |   async fetcher() { | ||||||
|  |     const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
 | ||||||
|  |     return await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); | ||||||
|  |   }, | ||||||
| }; | }; | ||||||
		Loading…
	
		Reference in New Issue
	
	Block a user