diff --git a/src/backend/database/database-wrapper.js b/src/backend/database/database-wrapper.ts similarity index 95% rename from src/backend/database/database-wrapper.js rename to src/backend/database/database-wrapper.ts index 2a5714c..c3a7b4c 100644 --- a/src/backend/database/database-wrapper.js +++ b/src/backend/database/database-wrapper.ts @@ -1,9 +1,5 @@ -import { - mongoRead, - mongoReadWithReadCredentials, - mongoUpsert, -} from "./mongo-wrapper"; -import { pgRead, pgReadWithReadCredentials, pgUpsert } from "./pg-wrapper"; +import { mongoRead, mongoReadWithReadCredentials, mongoUpsert } from './mongo-wrapper'; +import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper'; export async function databaseUpsert({ contents, group }) { // No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear. @@ -146,7 +142,7 @@ export async function databaseReadWithReadCredentials({ group }) { break; default: mongoDocName = `${group}-questions`; - responseMongo = mongoRemongoReadWithReadCredentialsad( + responseMongo = mongoReadWithReadCredentials( mongoDocName, "metaforecastCollection", "metaforecastDatabase" diff --git a/src/backend/database/mongo-wrapper.js b/src/backend/database/mongo-wrapper.ts similarity index 98% rename from src/backend/database/mongo-wrapper.js rename to src/backend/database/mongo-wrapper.ts index 02c38d8..f922407 100644 --- a/src/backend/database/mongo-wrapper.js +++ b/src/backend/database/mongo-wrapper.ts @@ -1,5 +1,7 @@ -import pkg from "mongodb"; -import { roughSizeOfObject } from "../utils/roughSize"; +import pkg from 'mongodb'; + +import { roughSizeOfObject } from '../utils/roughSize'; + const { MongoClient } = pkg; export async function mongoUpsert( diff --git a/src/backend/database/pg-wrapper.js b/src/backend/database/pg-wrapper.ts similarity index 98% rename from src/backend/database/pg-wrapper.js rename to src/backend/database/pg-wrapper.ts index 76c59ac..185da83 100644 --- a/src/backend/database/pg-wrapper.js +++ b/src/backend/database/pg-wrapper.ts @@ -1,7 +1,9 @@ -import pkg from "pg"; -import { platformNames } from "../platforms/all/platformNames"; -import { hash } from "../utils/hash"; -import { roughSizeOfObject } from "../utils/roughSize"; +import pkg from 'pg'; + +import { platformNames } from '../platforms/all/platformNames'; +import { hash } from '../utils/hash'; +import { roughSizeOfObject } from '../utils/roughSize'; + const { Pool } = pkg; // Definitions diff --git a/src/backend/flow/doEverything.js b/src/backend/flow/doEverything.ts similarity index 80% rename from src/backend/flow/doEverything.js rename to src/backend/flow/doEverything.ts index 81498c8..eabb4f1 100644 --- a/src/backend/flow/doEverything.js +++ b/src/backend/flow/doEverything.ts @@ -1,8 +1,9 @@ -import { platformFetchers } from "../platforms/all-platforms"; -import { rebuildAlgoliaDatabase } from "../utils/algolia"; -import { updateHistory } from "./history/updateHistory"; -import { mergeEverything } from "./mergeEverything"; -import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData"; +import { platformFetchers } from '../platforms/all-platforms'; +import { rebuildAlgoliaDatabase } from '../utils/algolia'; +import { updateHistory } from './history/updateHistory'; +import { mergeEverything } from './mergeEverything'; +import { rebuildNetlifySiteWithNewData } from './rebuildNetliftySiteWithNewData'; + /* Do everything */ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/src/backend/flow/doEverythingForScheduler.js b/src/backend/flow/doEverythingForScheduler.js deleted file mode 100644 index 4b8e69f..0000000 --- a/src/backend/flow/doEverythingForScheduler.js +++ /dev/null @@ -1,3 +0,0 @@ -import { doEverything } from "./doEverything"; - -doEverything(); diff --git a/src/backend/flow/doEverythingForScheduler.ts b/src/backend/flow/doEverythingForScheduler.ts new file mode 100644 index 0000000..d2753d9 --- /dev/null +++ b/src/backend/flow/doEverythingForScheduler.ts @@ -0,0 +1,3 @@ +import { doEverything } from './doEverything'; + +doEverything(); diff --git a/src/backend/flow/history/old/addToHistory.js b/src/backend/flow/history/old/addToHistory.js deleted file mode 100644 index 3f4f880..0000000 --- a/src/backend/flow/history/old/addToHistory.js +++ /dev/null @@ -1,110 +0,0 @@ -import { - databaseReadWithReadCredentials, - databaseUpsert, -} from "../../database/database-wrapper"; -let isEmptyArray = (arr) => arr.length == 0; - -export async function addToHistory() { - let currentDate = new Date(); - let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); - - let currentJSONwithMetaculus = await databaseReadWithReadCredentials({ - group: "combined", - }); - let currentJSON = currentJSONwithMetaculus.filter( - (element) => - element.platform != "Metaculus" && element.platform != "Estimize" - ); // without Metaculus - // console.log(currentJSON.slice(0,20)) - // console.log(currentJSON) - let historyJSON = await databaseReadWithReadCredentials({ group: "history" }); - // console.log(historyJSON) - - let currentForecastsWithAHistory = currentJSON.filter( - (element) => - !isEmptyArray( - historyJSON.filter( - (historyElement) => - historyElement.title == element.title && - historyElement.url == element.url - ) - ) - ); - // console.log(currentForecastsWithAHistory) - - let currentForecastsWithoutAHistory = currentJSON.filter((element) => - isEmptyArray( - historyJSON.filter( - (historyElement) => - historyElement.title == element.title && - historyElement.url == element.url - ) - ) - ); - // console.log(currentForecastsWithoutAHistory) - - // Add both types of forecast - let newHistoryJSON = []; - for (let historyElement of historyJSON) { - let correspondingNewElementArray = currentForecastsWithAHistory.filter( - (element) => - historyElement.title == element.title && - historyElement.url == element.url - ); - // console.log(correspondingNewElement) - if (!isEmptyArray(correspondingNewElementArray)) { - let correspondingNewElement = correspondingNewElementArray[0]; - let timeStampOfNewElement = correspondingNewElement.timestamp; - let doesHistoryAlreadyContainElement = historyElement.history - .map((element) => element.timestamp) - .includes(timeStampOfNewElement); - if (!doesHistoryAlreadyContainElement) { - let historyWithNewElement = historyElement["history"].concat({ - timestamp: correspondingNewElement.timestamp, - options: correspondingNewElement.options, - qualityindicators: correspondingNewElement.qualityindicators, - }); - let newHistoryElement = { - ...correspondingNewElement, - history: historyWithNewElement, - }; - // If some element (like the description) changes, we keep the new one. - newHistoryJSON.push(newHistoryElement); - } else { - newHistoryJSON.push(historyElement); - } - } else { - // console.log(historyElement) - newHistoryJSON.push(historyElement); - } - } - - for (let currentForecast of currentForecastsWithoutAHistory) { - let newHistoryElement = { - ...currentForecast, - history: [ - { - timestamp: currentForecast.timestamp, - options: currentForecast.options, - qualityindicators: currentForecast.qualityindicators, - }, - ], - }; - delete newHistoryElement.timestamp; - delete newHistoryElement.options; - delete newHistoryElement.qualityindicators; - newHistoryJSON.push(newHistoryElement); - } - - await databaseUpsert({ contents: newHistoryJSON, group: "history" }); - - // console.log(newHistoryJSON.slice(0,5)) - // writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2)) - // writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json") - //console.log(newHistoryJSON) - /* - let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url ))) - */ - //console.log(new Date().toISOString()) -} -// updateHistory() diff --git a/src/backend/flow/history/old/createHistoryForMonth.js b/src/backend/flow/history/old/createHistoryForMonth.js deleted file mode 100644 index ba55000..0000000 --- a/src/backend/flow/history/old/createHistoryForMonth.js +++ /dev/null @@ -1,36 +0,0 @@ -import { databaseRead, databaseUpsert } from "../../database/database-wrapper"; - -export async function createHistoryForMonth() { - let currentDate = new Date(); - let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); - let metaforecasts = await databaseRead({ group: "combined" }); - let metaforecastsHistorySeed = metaforecasts - .map((element) => { - // let moreoriginsdata = element.author ? ({author: element.author}) : ({}) - return { - title: element.title, - url: element.url, - platform: element.platform, - moreoriginsdata: element.moreoriginsdata || {}, - description: element.description, - history: [ - { - timestamp: element.timestamp, - options: element.options, - qualityindicators: element.qualityindicators, - }, - ], - extra: element.extra || {}, - }; - }) - .filter( - (element) => - element.platform != "Metaculus" && element.platform != "Estimize" - ); - //console.log(metaforecastsHistorySeed) - await databaseUpsert({ - contents: metaforecastsHistorySeed, - group: "history", - }); -} -////createInitialHistory() diff --git a/src/backend/flow/history/old/createInitialHistoryWithMetaculus.js b/src/backend/flow/history/old/createInitialHistoryWithMetaculus.js deleted file mode 100644 index 9ed86aa..0000000 --- a/src/backend/flow/history/old/createInitialHistoryWithMetaculus.js +++ /dev/null @@ -1,29 +0,0 @@ -import { databaseRead, databaseUpsert } from "../database-wrapper"; - -let createInitialHistory = async () => { - let metaforecasts = await databaseRead({ group: "combined" }); - let metaforecastsHistorySeed = metaforecasts.map((element) => { - // let moreoriginsdata = element.author ? ({author: element.author}) : ({}) - return { - title: element.title, - url: element.url, - platform: element.platform, - moreoriginsdata: element.moreoriginsdata || {}, - description: element.description, - history: [ - { - timestamp: element.timestamp, - options: element.options, - qualityindicators: element.qualityindicators, - }, - ], - extra: element.extra || {}, - }; - }); - console.log(metaforecastsHistorySeed); - await databaseUpsert({ - contents: metaforecastsHistorySeed, - group: "history", - }); -}; -createInitialHistory(); diff --git a/src/backend/flow/history/old/updateHistory.js b/src/backend/flow/history/old/updateHistory.js deleted file mode 100644 index 6f67ce0..0000000 --- a/src/backend/flow/history/old/updateHistory.js +++ /dev/null @@ -1,21 +0,0 @@ -import { addToHistory } from "./addToHistory"; -import { createHistoryForMonth } from "./createHistoryForMonth"; - -export async function updateHistoryOld() { - let currentDate = new Date(); - let dayOfMonth = currentDate.getDate(); - if (dayOfMonth == 1) { - console.log( - `Creating history for the month ${currentDate.toISOString().slice(0, 7)}` - ); - await createHistoryForMonth(); - } else { - console.log(`Updating history for ${currentDate.toISOString()}`); - await addToHistory(); - } -} - -export async function updateHistory() { - let currentDate = new Date(); - let year = currentDate.toISOString().slice(0, 4); -} diff --git a/src/backend/flow/history/updateHistory.js b/src/backend/flow/history/updateHistory.ts similarity index 65% rename from src/backend/flow/history/updateHistory.js rename to src/backend/flow/history/updateHistory.ts index b4733dc..9caf1e0 100644 --- a/src/backend/flow/history/updateHistory.js +++ b/src/backend/flow/history/updateHistory.ts @@ -1,7 +1,4 @@ -import { - databaseReadWithReadCredentials, - databaseUpsert, -} from "../../database/database-wrapper"; +import { databaseReadWithReadCredentials, databaseUpsert } from '../../database/database-wrapper'; export async function updateHistory() { let latest = await databaseReadWithReadCredentials({ group: "combined" }); diff --git a/src/backend/flow/mergeEverything.js b/src/backend/flow/mergeEverything.ts similarity index 84% rename from src/backend/flow/mergeEverything.js rename to src/backend/flow/mergeEverything.ts index 300b5ac..4377bb3 100644 --- a/src/backend/flow/mergeEverything.js +++ b/src/backend/flow/mergeEverything.ts @@ -1,5 +1,6 @@ -import { databaseRead, databaseUpsert } from "../database/database-wrapper"; -import { platformNames } from "../platforms/all-platforms"; +import { databaseRead, databaseUpsert } from '../database/database-wrapper'; +import { platformNames } from '../platforms/all-platforms'; + /* Merge everything */ export async function mergeEverythingInner() { diff --git a/src/backend/flow/rebuildNetliftySiteWithNewData.js b/src/backend/flow/rebuildNetliftySiteWithNewData.ts similarity index 80% rename from src/backend/flow/rebuildNetliftySiteWithNewData.js rename to src/backend/flow/rebuildNetliftySiteWithNewData.ts index 5204ead..7d4fe71 100644 --- a/src/backend/flow/rebuildNetliftySiteWithNewData.js +++ b/src/backend/flow/rebuildNetliftySiteWithNewData.ts @@ -1,5 +1,6 @@ -import axios from "axios"; -import { applyIfSecretExists } from "../utils/getSecrets"; +import axios from 'axios'; + +import { applyIfSecretExists } from '../utils/getSecrets'; async function rebuildNetlifySiteWithNewData_inner(cookie) { let payload = {}; diff --git a/src/backend/manual/manualDownload.js b/src/backend/manual/manualDownload.ts similarity index 69% rename from src/backend/manual/manualDownload.js rename to src/backend/manual/manualDownload.ts index c5eb4ea..f229616 100644 --- a/src/backend/manual/manualDownload.js +++ b/src/backend/manual/manualDownload.ts @@ -1,6 +1,8 @@ -import "dotenv/config"; -import fs from "fs"; -import { databaseReadWithReadCredentials } from "../database/database-wrapper"; +import 'dotenv/config'; + +import fs from 'fs'; + +import { databaseReadWithReadCredentials } from '../database/database-wrapper'; let main = async () => { let json = await databaseReadWithReadCredentials({ group: "combined" }); diff --git a/src/backend/manual/manualInitialize.js b/src/backend/manual/manualInitialize.js deleted file mode 100644 index 1427013..0000000 --- a/src/backend/manual/manualInitialize.js +++ /dev/null @@ -1,3 +0,0 @@ -import { pgInitialize } from "../database/pg-wrapper"; - -pgInitialize(); diff --git a/src/backend/manual/manualInitialize.ts b/src/backend/manual/manualInitialize.ts new file mode 100644 index 0000000..cdb1d09 --- /dev/null +++ b/src/backend/manual/manualInitialize.ts @@ -0,0 +1,3 @@ +import { pgInitialize } from '../database/pg-wrapper'; + +pgInitialize(); diff --git a/src/backend/manual/manualSendToMongo.js b/src/backend/manual/manualSendToMongo.ts similarity index 83% rename from src/backend/manual/manualSendToMongo.js rename to src/backend/manual/manualSendToMongo.ts index 2aea360..19ddcfd 100644 --- a/src/backend/manual/manualSendToMongo.js +++ b/src/backend/manual/manualSendToMongo.ts @@ -1,5 +1,6 @@ -import fs from "fs"; -import { databaseUpsert } from "../database/database-wrapper"; +import fs from 'fs'; + +import { databaseUpsert } from '../database/database-wrapper'; /* This is necessary for estimize, the database of x-risk estimates, and for the OpenPhil/GiveWell predictions. Unlike the others, I'm not fetching them constantly, but only once. */ @@ -9,7 +10,9 @@ let suffixMongo = "-questions"; let main = async () => { for (let file of pushManualFiles) { - let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`); + let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`, { + encoding: "utf-8", + }); let fileContents = JSON.parse(fileRaw); console.log(fileContents); await databaseUpsert({ contents: fileContents, group: file }); diff --git a/src/backend/manual/pullSuperforecastsManually.js b/src/backend/manual/pullSuperforecastsManually.ts similarity index 61% rename from src/backend/manual/pullSuperforecastsManually.js rename to src/backend/manual/pullSuperforecastsManually.ts index 1a4829a..8f64ca3 100644 --- a/src/backend/manual/pullSuperforecastsManually.js +++ b/src/backend/manual/pullSuperforecastsManually.ts @@ -1,5 +1,5 @@ /* Imports */ -import { goodjudgment } from "../platforms/goodjudgment-fetch"; +import { goodjudgment } from '../platforms/goodjudgment-fetch'; /* Definitions */ diff --git a/src/backend/platforms/all-platforms.js b/src/backend/platforms/all-platforms.ts similarity index 100% rename from src/backend/platforms/all-platforms.js rename to src/backend/platforms/all-platforms.ts diff --git a/src/backend/platforms/all/platformFetchers.js b/src/backend/platforms/all/platformFetchers.ts similarity index 50% rename from src/backend/platforms/all/platformFetchers.js rename to src/backend/platforms/all/platformFetchers.ts index 04c1ce5..5bf174f 100644 --- a/src/backend/platforms/all/platformFetchers.js +++ b/src/backend/platforms/all/platformFetchers.ts @@ -1,17 +1,17 @@ -import { betfair } from "../betfair-fetch"; -import { fantasyscotus } from "../fantasyscotus-fetch"; -import { foretold } from "../foretold-fetch"; -import { goodjudgment } from "../goodjudgment-fetch"; -import { goodjudgmentopen } from "../goodjudmentopen-fetch"; -import { infer } from "../infer-fetch"; -import { kalshi } from "../kalshi-fetch"; -import { manifoldmarkets } from "../manifoldmarkets-fetch"; -import { metaculus } from "../metaculus-fetch"; -import { polymarket } from "../polymarket-fetch"; -import { predictit } from "../predictit-fetch"; -import { rootclaim } from "../rootclaim-fetch"; -import { smarkets } from "../smarkets-fetch"; -import { wildeford } from "../wildeford-fetch"; +import { betfair } from '../betfair-fetch'; +import { fantasyscotus } from '../fantasyscotus-fetch'; +import { foretold } from '../foretold-fetch'; +import { goodjudgment } from '../goodjudgment-fetch'; +import { goodjudgmentopen } from '../goodjudmentopen-fetch'; +import { infer } from '../infer-fetch'; +import { kalshi } from '../kalshi-fetch'; +import { manifoldmarkets } from '../manifoldmarkets-fetch'; +import { metaculus } from '../metaculus-fetch'; +import { polymarket } from '../polymarket-fetch'; +import { predictit } from '../predictit-fetch'; +import { rootclaim } from '../rootclaim-fetch'; +import { smarkets } from '../smarkets-fetch'; +import { wildeford } from '../wildeford-fetch'; /* Deprecated import { astralcodexten } from "../platforms/astralcodexten-fetch" diff --git a/src/backend/platforms/all/platformNames.js b/src/backend/platforms/all/platformNames.ts similarity index 100% rename from src/backend/platforms/all/platformNames.js rename to src/backend/platforms/all/platformNames.ts diff --git a/src/backend/platforms/betfair-fetch.js b/src/backend/platforms/betfair-fetch.ts similarity index 100% rename from src/backend/platforms/betfair-fetch.js rename to src/backend/platforms/betfair-fetch.ts diff --git a/src/backend/platforms/deprecated/astralcodexten-fetch.js b/src/backend/platforms/deprecated/astralcodexten-fetch.js deleted file mode 100644 index 5615238..0000000 --- a/src/backend/platforms/deprecated/astralcodexten-fetch.js +++ /dev/null @@ -1,97 +0,0 @@ -/* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { calculateStars } from "../utils/stars"; - -/* Definitions */ -let graphQLendpoint = "https://api.foretold.io/graphql"; -let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"]; - -/* Support functions */ -async function fetchAllCommunityQuestions(communityId) { - let response = await axios({ - url: graphQLendpoint, - method: "POST", - headers: { "Content-Type": "application/json" }, - data: JSON.stringify({ - query: ` - query { - measurables( - channelId: "${communityId}", - states: OPEN, - first: 500 - ){ - total - edges{ - node{ - id - name - valueType - measurementCount - previousAggregate{ - value{ - percentage - } - } - } - } - } - } - `, - }), - }) - .then((res) => res.data) - .then((res) => res.data.measurables.edges); - //console.log(response) - return response; -} - -/* Body */ - -export async function astralcodexten() { - let results = []; - for (let community of ScottAlexanderPredictions) { - let questions = await fetchAllCommunityQuestions(community); - questions = questions.map((question) => question.node); - questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions - questions.forEach((question) => { - let options = []; - if (question.valueType == "PERCENTAGE") { - let probability = question.previousAggregate.value.percentage; - options = [ - { - name: "Yes", - probability: probability / 100, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability / 100, - type: "PROBABILITY", - }, - ]; - } - let result = { - title: question.name.split(". ")[1], - url: `https://www.foretold.io/c/${community}/m/${question.id}`, - platform: "AstralCodexTen", - description: "...by the end of 2021", - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: Number((question.measurementCount + 1) / 2), - stars: calculateStars("AstralCodexTen", {}), - }, - /*liquidity: liquidity.toFixed(2), - tradevolume: tradevolume.toFixed(2), - address: obj.address*/ - }; - // console.log(result) - results.push(result); - }); - } - await databaseUpsert(results, "astralcodexten-questions"); - // console.log(results) - console.log("Done"); -} -// astralcodexten() diff --git a/src/backend/platforms/deprecated/coupcast-fetch.js b/src/backend/platforms/deprecated/coupcast-fetch.js deleted file mode 100644 index d3627df..0000000 --- a/src/backend/platforms/deprecated/coupcast-fetch.js +++ /dev/null @@ -1,184 +0,0 @@ -/* Imports */ -import axios from "axios"; -import Papa from "papaparse"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { calculateStars } from "../utils/stars"; - -/* Definitions */ -let coupCastEndpoint = - "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv"; -var datenow = new Date(); -var currentmonth = datenow.getMonth() + 1; -dd; -/* Support functions */ -let unique = (arr) => [...new Set(arr)]; -let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -let sanitizeCountryName = (country_name) => { - let sanitized_name; - switch (country_name) { - case "Cen African Rep": - sanitized_name = "Central African Republic"; - break; - case "Congo-Brz": - sanitized_name = "Republic of the Congo"; - break; - case "Congo/Zaire": - sanitized_name = "Democratic Republic of the Congo"; - break; - case "Czech Rep": - sanitized_name = "Czech Republic"; - break; - case "Dominican Rep": - sanitized_name = "Dominican Republic"; - break; - case "Korea North": - sanitized_name = "North Korea"; - break; - case "Korea South": - sanitized_name = "South Korea"; - break; - case "UKG": - sanitized_name = "UK"; - break; - default: - sanitized_name = country_name; - } - return sanitized_name; -}; - -async function processArray(countryArray) { - let results = []; - for (let country of countryArray) { - let url = `https://www.oneearthfuture.org/activities/coup-cast`; - - // We don't really want the prediction for all months; one is enough - // console.log(country.month) - if (Number(country.month) == currentmonth) { - // Monthly - country.country_name = sanitizeCountryName(country.country_name); - let processedPrediction1 = { - title: `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`, - url: url, - platform: "CoupCast", - description: `The current leader of ${country.country_name} is ${ - country.leader_name - }, who has been in power for ${Number(country.leader_years).toFixed( - 1 - )} years. ${ - country.country_name - } has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${ - country.regime_years - } years.`, - options: [ - { - name: "Yes", - probability: country.month_risk, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - country.month_risk, - type: "PROBABILITY", - }, - ], - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Coupcast", {}), - }, - extra: { - country_name: country.country_name, - regime_type: country.regime_type, - month: country.month, - year: country.year, - leader_name: country.leader_name, - month_risk: country.month_risk, - annual_risk: country.annual_risk, - risk_change_percent: country.risk_change_percent, - regime_years: country.regime_years, - leader_years: country.leader_years, - country_code: country.country_code, - country_abb: country.country_abb, - }, - }; - - // Yearly - let processedPrediction2 = { - title: `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`, - url: url, - platform: "CoupCast", - description: `The current leader of ${country.country_name} is ${ - country.leader_name - }, who has been in power for ${Number(country.leader_years).toFixed( - 1 - )} years. ${ - country.country_name - } has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${ - country.regime_years - } years`, - options: [ - { - name: "Yes", - probability: country.annual_risk, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - country.annual_risk, - type: "PROBABILITY", - }, - ], - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("CoupCast", {}), - }, - extra: { - country_name: country.country_name, - regime_type: country.regime_type, - month: country.month, - year: country.year, - leader_name: country.leader_name, - month_risk: country.month_risk, - annual_risk: country.annual_risk, - risk_change_percent: country.risk_change_percent, - regime_years: country.regime_years, - leader_years: country.leader_years, - country_code: country.country_code, - country_abb: country.country_abb, - }, - }; - - // results.push(processedPrediction1) - // Not pushing monthly - results.push(processedPrediction2); - } - } - await databaseUpsert(results, "coupcast-questions"); - // console.log(results) - console.log("Done"); -} - -/* Body */ - -export async function coupcast() { - let csvContent = await axios - .get(coupCastEndpoint) - .then((query) => query.data); - await Papa.parse(csvContent, { - header: true, - complete: async (results) => { - console.log("Downloaded", results.data.length, "records."); - /* console.log( - JSON.stringify( - unique(results.data.map(country => country.country_name)), - null, - 4 - ) - )*/ - // console.log(results.data) - await processArray(results.data); - }, - }); - await sleep(1000); // needed to wait for Papaparse's callback to be executed. -} -// coupcast() diff --git a/src/backend/platforms/deprecated/csetforetell-fetch.js b/src/backend/platforms/deprecated/csetforetell-fetch.js deleted file mode 100644 index 065f66f..0000000 --- a/src/backend/platforms/deprecated/csetforetell-fetch.js +++ /dev/null @@ -1,280 +0,0 @@ -/* Imports */ -import axios from "axios"; -import { Tabletojson } from "tabletojson"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { applyIfSecretExists } from "../utils/getSecrets"; -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; - -/* Definitions */ -let htmlEndPoint = "https://www.cset-foretell.com/questions?page="; -String.prototype.replaceAll = function replaceAll(search, replace) { - return this.split(search).join(replace); -}; -const DEBUG_MODE = "on"; // "off" -const SLEEP_TIME_RANDOM = 100; //5000 // miliseconds -const SLEEP_TIME_EXTRA = 0; //1000 -/* Support functions */ - -async function fetchPage(page, cookie) { - console.log(`Page #${page}`); - if (page == 1) { - cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :( - } - let urlEndpoint = htmlEndPoint + page; - console.log(urlEndpoint); - let response = await axios({ - url: urlEndpoint, - method: "GET", - headers: { - "Content-Type": "text/html", - Cookie: cookie, - }, - }).then((res) => res.data); - // console.log(response) - return response; -} - -async function fetchStats(questionUrl, cookie) { - let response = await axios({ - url: questionUrl + "/stats", - method: "GET", - headers: { - "Content-Type": "text/html", - Cookie: cookie, - Referer: questionUrl, - }, - }).then((res) => res.data); - - if (response.includes("Sign up or sign in to forecast")) { - throw Error("Not logged in"); - } - - // Is binary? - let isbinary = response.includes("binary?":true"); - // console.log(`is binary? ${isbinary}`) - let options = []; - if (isbinary) { - // Crowd percentage - let htmlElements = response.split("\n"); - // DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id() - let h3Element = htmlElements.filter((str) => str.includes("

"))[0]; - // DEBUG_MODE == "on" ? console.log(h5elements) : id() - let crowdpercentage = h3Element.split(">")[1].split("<")[0]; - let probability = Number(crowdpercentage.replace("%", "")) / 100; - options.push( - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans - type: "PROBABILITY", - } - ); - } else { - try { - let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table. - // console.log(optionsBody) - let optionsHtmlElement = ""; - let tablesAsJson = Tabletojson.convert(optionsHtmlElement); - let firstTable = tablesAsJson[0]; - options = firstTable.map((element) => ({ - name: element["0"], - probability: Number(element["1"].replace("%", "")) / 100, - type: "PROBABILITY", - })); - } catch (error) { - let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position - let optionsHtmlElement = ""; - let tablesAsJson = Tabletojson.convert(optionsHtmlElement); - let firstTable = tablesAsJson[0]; - if (firstTable) { - options = firstTable.map((element) => ({ - name: element["0"], - probability: Number(element["1"].replace("%", "")) / 100, - type: "PROBABILITY", - })); - } else { - // New type of question, tricky to parse the options - // Just leave options = [] for now. - // https://www.cset-foretell.com/blog/rolling-question-formats - } - } - } - // Description - let descriptionraw = response.split(``)[0]; - let descriptionprocessed2 = descriptionprocessed1.replace(">", ""); - let descriptionprocessed3 = descriptionprocessed2.replace( - "To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", - `` - ); - // console.log(descriptionprocessed3) - let descriptionprocessed4 = descriptionprocessed3.replaceAll( - "\r\n\r\n", - "\n" - ); - let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n"); - let descriptionprocessed6 = descriptionprocessed5.replaceAll(""", `"`); - let descriptionprocessed7 = descriptionprocessed6.replaceAll("'", "'"); - let descriptionprocessed8 = toMarkdown(descriptionprocessed7); - let description = descriptionprocessed8; - - // Number of forecasts - //console.log(response) - //console.log(response.split("prediction_sets_count":")[1]) - let numforecasts = response - .split("prediction_sets_count":")[1] - .split(",")[0]; - // console.log(numforecasts) - - // Number of predictors - let numforecasters = response - .split("predictors_count":")[1] - .split(",")[0]; - // console.log(numpredictors) - - let result = { - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: Number(numforecasts), - numforecasters: Number(numforecasters), - stars: calculateStars("CSET-foretell", { numforecasts }), - }, - }; - - return result; -} - -function isSignedIn(html) { - let isSignedInBool = !( - html.includes("You need to sign in or sign up before continuing") || - html.includes("Sign up") - ); - if (!isSignedInBool) { - console.log("Error: Not signed in."); - } - console.log(`Signed in? ${isSignedInBool}`); - return isSignedInBool; -} - -function isEnd(html) { - let isEndBool = html.includes("No questions match your filter"); - if (isEndBool) { - //console.log(html) - } - console.log(`IsEnd? ${isEndBool}`); - return isEndBool; -} - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/* Body */ - -async function csetforetell_inner(cookie) { - let i = 1; - let response = await fetchPage(i, cookie); - let results = []; - let init = Date.now(); - // console.log("Downloading... This might take a couple of minutes. Results will be shown.") - while (!isEnd(response) && isSignedIn(response)) { - let htmlLines = response.split("\n"); - // let h4elements = htmlLines.filter(str => str.includes("
'); - let url = elementSplit[0].split('", "") - .replace("
", "") - .replace("", ""); - await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable - - try { - let moreinfo = await fetchStats(url, cookie); - let question = { - title: title, - url: url, - platform: "CSET-foretell", - ...moreinfo, - }; - if ( - i % 30 == 0 && - !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") - ) { - console.log( - `Page #${i}` && - !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") - ); - console.log(question); - } - results.push(question); - if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") { - console.log(url); - console.log(question); - } - } catch (error) { - console.log(error); - console.log( - `We encountered some error when fetching the URL: ${url}, so it won't appear on the final json` - ); - } - } - - i++; - //i=Number(i)+1 - - console.log( - "Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers" - ); - await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable - - try { - response = await fetchPage(i, cookie); - } catch (error) { - console.log(error); - console.log( - `The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages` - ); - } - } - if (results.length > 0) { - await databaseUpsert(results, "csetforetell-questions"); - } else { - console.log("Not updating results, as process was not signed in"); - } - - let end = Date.now(); - let difference = end - init; - console.log( - `Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.` - ); -} - -export async function csetforetell() { - let cookie = process.env.CSETFORETELL_COOKIE; - await applyIfSecretExists(cookie, csetforetell_inner); -} diff --git a/src/backend/platforms/deprecated/elicit-fetch.js b/src/backend/platforms/deprecated/elicit-fetch.js deleted file mode 100644 index 9515a71..0000000 --- a/src/backend/platforms/deprecated/elicit-fetch.js +++ /dev/null @@ -1,121 +0,0 @@ -/* Imports */ -import axios from "axios"; -import Papa from "papaparse"; -import readline from "readline"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { calculateStars } from "../utils/stars"; - -/* Definitions */ -let elicitEndpoint = - "https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community"; - -/* Support functions */ -let avg = (array) => - array.reduce((a, b) => Number(a) + Number(b)) / array.length; -let unique = (arr) => [...new Set(arr)]; -let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -async function processArray(arrayQuestions) { - let questions = arrayQuestions.map((question) => question.questionTitle); - let uniqueQuestions = unique(questions); - let questionsObj = {}; - uniqueQuestions.forEach((questionTitle) => { - questionsObj[questionTitle] = { - title: questionTitle, - forecasters: [], - forecasts: [], - }; - }); - arrayQuestions.forEach((question) => { - // console.log(question.questionTitle) - let questionTitle = question.questionTitle; - let correspondingQuestion = questionsObj[questionTitle]; - let forecasters = correspondingQuestion.forecasters.concat( - question.predictionCreator - ); - let forecasts = correspondingQuestion.forecasts.concat(question.prediction); - questionsObj[questionTitle] = { - forecasters, - forecasts, - }; - }); - let onlyQuestionsWithMoreThan; - - let results = []; - for (let question in questionsObj) { - let title = question; - - let forecasters = questionsObj[question].forecasters; - - let numforecasters = unique(forecasters).length; - if (numforecasters >= 10) { - let url = `https://elicit.org/binary?binaryQuestions.search=${title.replace( - / /g, - "%20" - )}&binaryQuestions.sortBy=popularity&limit=20&offset=0`; - let forecasts = questionsObj[question].forecasts; - - //console.log(forecasts) - //console.log(avg(forecasts)) - let probability = avg(forecasts) / 100; - let numforecasts = forecasts.length; - let standardObj = { - title: title, - url: url, - platform: "Elicit", - options: [ - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability, - type: "PROBABILITY", - }, - ], - timestamp: new Date().toISOString(), - qualityindicators: { - numforecasts: Number(numforecasts), - numforecasters: Number(numforecasters), - stars: calculateStars("Elicit", {}), - }, - }; - results.push(standardObj); - } - } - await databaseUpsert(results, "elicit-questions"); - - console.log("Done"); -} - -async function awaitdownloadconfirmation(message, callback) { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - rl.question(message, (answer) => { - //console.log("Received"); - rl.close(); - - callback(); - }); -} - -/* Body */ - -export async function elicit() { - let csvContent = await axios.get(elicitEndpoint).then((query) => query.data); - await Papa.parse(csvContent, { - header: true, - complete: async (results) => { - console.log("Downloaded", results.data.length, "records."); - //resolve(results.data); - //console.log(results.data) - await processArray(results.data); - }, - }); - await sleep(5000); // needed to wait for Papaparse's callback to be executed. -} -//elicit() diff --git a/src/backend/platforms/deprecated/estimize-fetch.js b/src/backend/platforms/deprecated/estimize-fetch.js deleted file mode 100644 index 8f70cd5..0000000 --- a/src/backend/platforms/deprecated/estimize-fetch.js +++ /dev/null @@ -1,35 +0,0 @@ -import fs from "fs"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { calculateStars } from "../utils/stars"; - -export async function estimize() { - let data = fs.readFileSync( - "./input/s-and-p-500-companies/companies.csv", - "utf8" - ); - - let splitData = data.split("\n"); - let results = []; - for (let datum of splitData) { - if (datum != "") { - //console.log(datum) - let datumSplit = datum.split(","); - let companyStickerSymbol = datumSplit[0]; - let companyName = datumSplit[1]; - let standardObj = { - title: `Estimates for ${companyName} (${companyStickerSymbol})`, - url: `https://www.estimize.com/${companyStickerSymbol.toLowerCase()}`, - platform: "Estimize", - description: `A link to Estimize's forecasts for *${companyName}* (sticker symbol ${companyStickerSymbol}). Viewing them requires making a prediction, Wall Street estimates are provided for free`, - options: [], - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Estimize", {}), - }, - }; - results.push(standardObj); - } - } - await databaseUpsert(results, "estimize-questions"); -} -//estimize() diff --git a/src/backend/platforms/deprecated/hypermind-fetch.js b/src/backend/platforms/deprecated/hypermind-fetch.js deleted file mode 100644 index dba6bed..0000000 --- a/src/backend/platforms/deprecated/hypermind-fetch.js +++ /dev/null @@ -1,201 +0,0 @@ -/* Imports */ -import axios from "axios"; -import https from "https"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { applyIfSecretExists } from "../utils/getSecrets"; -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; - -/* Definitions */ -let hypermindEnpoint1 = "https://predict.hypermind.com/dash/jsx.json"; -let hypermindEnpoint2 = "https://prod.hypermind.com/ngdp-jsx/jsx.json"; -const insecureHttpsAgent = new https.Agent({ - rejectUnauthorized: false, // (NOTE: this will disable client verification) -}); - -/* Support Functions */ -String.prototype.replaceAll = function replaceAll(search, replace) { - return this.split(search).join(replace); -}; - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/* Fetchers */ -async function fetchHypermindData1(slug) { - let jsx = `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`; - // console.log(jsx) - let response = await await axios(hypermindEnpoint1, { - credentials: "omit", - headers: { - "User-Agent": - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0", - Accept: - "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Cache-Control": "max-age=0", - }, - referrer: `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`, - data: jsx, - method: "POST", - mode: "cors", - httpsAgent: insecureHttpsAgent, - }).then((response) => response.data[0].questions); - //console.log(response) - return response; -} - -async function fetchHypermindDataShowcases(slug, cookie) { - let response = await axios(hypermindEnpoint2, { - credentials: "include", - headers: { - "User-Agent": "", - Accept: "*/*", - "Accept-Language": "en-US,en;q=0.5", - "Content-Type": "application/json; charset=UTF-8", - //"Cookie": cookie - }, - referrer: - "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", - data: `[["showcase","getShowcase",{"showcase":"${slug}","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":false}}]]`, - method: "POST", - mode: "cors", - httpsAgent: insecureHttpsAgent, - }) - .then((resp) => resp.data[0].items) - .then((items) => items.filter((item) => item.type == "IFP")) - .then((items) => items.map((item) => item.IFP)); - - // console.log(response) - // response.forEach(item => delete item.crowdFcstHist) - return response; -} - -/* Body */ -async function hypermind_inner(cookie) { - // Hypermind panelists and competitors; dashboard type two: "showcase" - // https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=SLUG - // E.g., https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=AI2023 - let slugs2 = ["AI2030", "Covid19", "DOSES", "H5N8", "NGDP", "JSAI", "AI2023"]; // [] - let results2 = []; - for (let slug of slugs2) { - console.log(slug); - await sleep(1000 + Math.random() * 1000); - let response = await fetchHypermindDataShowcases(slug); - let objs = response.map((result) => { - let descriptionraw = result.props.details.split("
")[0]; - let descriptionprocessed1 = toMarkdown(descriptionraw); - let descriptionprocessed2 = descriptionprocessed1.replaceAll( - "![image] ()", - "" - ); - let descriptionprocessed3 = descriptionprocessed2.replaceAll( - " Forecasting Schedule ", - "" - ); - let descriptionprocessed4 = descriptionprocessed3 - .replaceAll("\n", " ") - .replaceAll(" ", " "); - let descriptionprocessed5 = descriptionprocessed4.replaceAll( - "Context:", - "" - ); - let description = - descriptionprocessed5 || toMarkdown(result.props.details); - return { - title: result.props.title, - url: - "https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=" + - slug, - platform: "Hypermind", - description: description, - options: [], - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Hypermind", {}), - numforecasters: Number(result.fcsterCnt), - }, - }; - }); - // console.log(objs) - results2.push(...objs); - } - - // Prediction markets; dashboard type one. - // https://predict.hypermind.com/dash/dash/dash.html?list=SLUG - // e.g., https://predict.hypermind.com/dash/dash/dash.html?list=POL - let slugs1 = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]; // [] - let results1 = []; - - for (let slug of slugs1) { - console.log(slug); - await sleep(2000 + Math.random() * 2000); - let result = await fetchHypermindData1(slug); - let objs = result.map((res) => { - let descriptionraw = res.props.details; - let descriptionprocessed1 = descriptionraw.split("%%fr")[0]; - let descriptionprocessed2 = descriptionprocessed1.replaceAll( - "
", - "\n" - ); - let descriptionprocessed3 = descriptionprocessed2.replace("%%en:", ""); - let descriptionprocessed4 = descriptionprocessed3.replace( - `Shares of the correct outcome will be worth 100ℍ, while the others will be worthless (0ℍ).

`, - "" - ); - let descriptionprocessed5 = toMarkdown(descriptionprocessed4); - let description = descriptionprocessed5 - .replaceAll("\n", " ") - .replaceAll(" ", " "); - //console.log(res.otcms) - //let percentage = (res.otcms.length==2) ? Number(res.otcms[0].price).toFixed(0) +"%" : "none" - let options = res.otcms.map((option) => ({ - name: option.props.title.split("%%fr")[0].replaceAll("%%en:", ""), - probability: Number(option.price) / 100, - type: "PROBABILITY", - })); - return { - title: res.props.title.split("%%fr")[0].replace("%%en:", ""), - url: "https://predict.hypermind.com/dash/dash/dash.html?list=" + slug, - platform: "Hypermind", - description: description, - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Hypermind", {}), - // "numforecasters": res.fcsterCnt - }, - }; - }); - // console.log(objs) - results1.push(...objs); - } - - let resultsTotal = [...results1, ...results2]; - - let distinctTitles = []; - let resultsTotalUnique = []; - for (let result of resultsTotal) { - if (!distinctTitles.includes(result["title"])) { - resultsTotalUnique.push(result); - distinctTitles.push(result["title"]); - } - } - // console.log(resultsTotal) - // console.log(resultsTotalUnique) - console.log(resultsTotalUnique.length, "results"); - await databaseUpsert(resultsTotalUnique, "hypermind-questions"); -} -//hypermind() - -export async function hypermind() { - let cookie = process.env.HYPERMINDCOOKIE; - await applyIfSecretExists(cookie, hypermind_inner); -} diff --git a/src/backend/platforms/deprecated/ladbrokes-fetch.js b/src/backend/platforms/deprecated/ladbrokes-fetch.js deleted file mode 100644 index be06be7..0000000 --- a/src/backend/platforms/deprecated/ladbrokes-fetch.js +++ /dev/null @@ -1,142 +0,0 @@ -/* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { calculateStars } from "../utils/stars"; - -/* Definitions */ -let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`; -let enpointDrillDown = (id) => - `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`; - -//

response.data); - return response; -} - -let processResults = async (json) => { - let results = []; - let children = json.SSResponse.children; - children.pop(); - let ids = children.map((child) => child.event.id); - let markets = []; - for (let id of ids) { - let marketsContainer = await fetchUrl(enpointDrillDown(id)); - let marketsObj = marketsContainer.SSResponse.children[0].event; - let newMarkets = marketsObj.children; - newMarkets = newMarkets.map((market) => ({ - ...market.market, - parent: marketsObj.name, - })); - markets.push(...newMarkets); - } - let normalMarkets = markets.filter( - (market) => !market.name.includes("Specials") - ); - //console.log(normalMarkets) - - for (let normalMarket of normalMarkets) { - let title = normalMarket.parent + ": " + normalMarket.name; - title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge - let options = normalMarket.children.map((child) => { - let name = child.outcome.name; - - let priceData = child.outcome.children[0].price; - let priceDecimal = Number(priceData.priceDec); - let probability = 1 / priceDecimal; - let option = { - name: name, - probability: probability, - type: "PROBABILITY", - }; - return option; - }); - // normalize probabilities - let totalValue = options - .map((element) => Number(element.probability)) - .reduce((a, b) => a + b, 0); - options = options.map((element) => ({ - ...element, - probability: Number(element.probability) / totalValue, - })); - - // Filter very unlikely probabilities: Not here, but on the front end - // options = options.filter(element => element.probability > 0.02) - - let obj = { - title: title, - url: "https://sports.ladbrokes.com/sport/politics/outrights", - platform: "Ladbrokes", - description: "", - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Ladbrokes", {}), - }, - }; - results.push(obj); - } - - let specialMarkets = markets.filter((market) => - market.name.includes("Specials") - ); - for (let specialMarket of specialMarkets) { - //let title = specialMarket.parent + ": " + specialMarket.name - //console.log(title) - specialMarket.children.forEach((child) => { - let name = specialMarket.parent.includes("Specials") - ? child.outcome.name - : specialMarket.parent + ": " + child.outcome.name; - name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge - let priceData = child.outcome.children[0].price; - let priceDecimal = Number(priceData.priceDec); - let probability = 1 / priceDecimal; - let obj = { - title: name, - url: "https://sports.ladbrokes.com/sport/politics/outrights", - platform: "LadBrokes", - options: [ - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability, - type: "PROBABILITY", - }, - ], - qualityindicators: { - stars: calculateStars("Ladbrokes", {}), - }, - }; - results.push(obj); - }); - } - return results; -}; - -/* Body */ -export async function ladbrokes() { - let response = await fetchUrl(endpointPolitics); - let results = await processResults(response); - await databaseUpsert(results, "ladbrokes-questions"); - console.log("Done"); -} -//ladbrokes() diff --git a/src/backend/platforms/deprecated/omen-fetch.js b/src/backend/platforms/deprecated/omen-fetch.js deleted file mode 100644 index 67f00c6..0000000 --- a/src/backend/platforms/deprecated/omen-fetch.js +++ /dev/null @@ -1,97 +0,0 @@ -/* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../../utils/database-wrapper"; -import { calculateStars } from "../../utils/stars"; - -/* Definitions */ -let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen"; -// "https://gateway.thegraph.com/api/[api-key]/subgraphs/id/0x0503024fcc5e1bd834530e69d592dbb6e8c03968-0" -// 'https://api.thegraph.com/subgraphs/name/protofire/omen' -// https://github.com/protofire/omen-subgraph -// https://thegraph.com/explorer/subgraph/protofire/omen - -async function fetchAllContractData() { - let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 50; // last 30 days - let response = await axios({ - url: graphQLendpoint, - method: "POST", - headers: { "Content-Type": "application/json" }, - data: JSON.stringify({ - query: ` - { - fixedProductMarketMakers(first: 1000, - where: { - lastActiveDay_gt: ${daysSinceEra} - } - ){ - id - lastActiveDay - question{ - title - } - outcomeSlotCount - outcomeTokenMarginalPrices - usdVolume - usdLiquidityMeasure - resolutionTimestamp - } - } - `, - }), - }) - .then((res) => res.data) - .then((res) => res.data.fixedProductMarketMakers); - console.log(response); - return response; -} - -async function fetch_all() { - let allData = await fetchAllContractData(); - let results = []; - for (let data of allData) { - if ( - (data.question != null) & - (data.usdLiquidityMeasure != "0") & - (data.resolutionTimestamp == null) & - (data.question.title != "ssdds") - ) { - // console.log(data) - // console.log(data.usdLiquidityMeasure) - let options = data.outcomeTokenMarginalPrices.map((price, slotnum) => { - let name = `Option ${slotnum}`; - if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 0) - name = "Yes"; - if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 1) - name = "No"; - return { - name: name, - probability: Number(price), - type: "PROBABILITY", - }; - }); - - let obj = { - title: data.question.title, - url: "https://omen.eth.link/#/" + data.id, - platform: "Omen", - description: "", - options: options, - timestamp: new Date().toISOString(), - qualityindicators: { - stars: calculateStars("Omen", {}), - }, - }; - // console.log(obj) - results.push(obj); - } - } - return results; -} - -/* Body */ -export async function omen() { - let results = await fetch_all(); - await databaseUpsert(results, "omen-questions"); - console.log("Done"); -} -//omen() diff --git a/src/backend/platforms/deprecated/williamhill-fetch.js b/src/backend/platforms/deprecated/williamhill-fetch.js deleted file mode 100644 index cf7c783..0000000 --- a/src/backend/platforms/deprecated/williamhill-fetch.js +++ /dev/null @@ -1,147 +0,0 @@ -/* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../utils/database-wrapper"; -import { calculateStars } from "../utils/stars"; - -/* Definitions */ -let endpoint = "https://sports.williamhill.com/betting/en-gb/politics"; - -//
response.data); - return response; -} - -let processResults = (html) => { - let results = []; - - let chunks = html.split( - '

' - ); - kamalamarkets.shift(); - for (let kamalamarket of kamalamarkets) { - let title = kamalamarket.split("")[0]; - let numerator = Number(kamalamarket.split('data-num="')[1].split('"')[0]); - let denominator = Number( - kamalamarket.split('data-denom="')[1].split('"')[0] - ); - let probability = denominator / (numerator + denominator); - let obj = { - title: title, - url: "https://sports.williamhill.com/betting/en-gb/politics", - platform: "WilliamHill", - description: "", - timestamp: new Date().toISOString(), - options: [ - { - name: "Yes", - probability: probability, - type: "PROBABILITY", - }, - { - name: "No", - probability: 1 - probability, - type: "PROBABILITY", - }, - ], - qualityindicators: { - stars: calculateStars("WilliamHill", {}), - }, - }; - results.push(obj); - } - chunks.shift(); - - // Deal with the other markets - for (let chunk of chunks) { - let title = chunk.split('"')[0]; - let title2 = chunk.split(' title2.length ? title : title2; - - let options = []; - let alternatives = chunk.split( - '

' - ); - alternatives.shift(); - for (let alternative of alternatives) { - let optionName = alternative.split("")[0]; - let numerator = Number(alternative.split('data-num="')[1].split('"')[0]); - let denominator = Number( - alternative.split('data-denom="')[1].split('"')[0] - ); - let option = { - name: optionName, - probability: denominator / (numerator + denominator), - type: "PROBABILITY", - }; - options.push(option); - } - - // normalize probabilities - let totalValue = options - .map((element) => Number(element.probability)) - .reduce((a, b) => a + b, 0); - options = options.map((element) => ({ - ...element, - probability: Number(element.probability) / totalValue, - })); - // Filter very unlikely probabilities: Not here, but on the front end - // options = options.filter(element => element.probability > 0.02) - - let obj = { - title: title, - url: "https://sports.williamhill.com/betting/en-gb/politics", - platform: "WilliamHill", - options: options, - qualityindicators: { - stars: calculateStars("WilliamHill", {}), - }, - }; - results = results.filter( - (result) => result.title.length > 4 && result.title != "2024 or later" - ); - // removes some predictions because hard to parse. - results.push(obj); - } - - //console.log(results) - return results; -}; - -let processhtml2 = (html) => { - html.split(); -}; - -/* Body */ -export async function williamhill() { - let response = await fetchUrl( - "https://sports.williamhill.com/betting/en-gb/politics" - ); - let results = processResults(response); - await databaseUpsert(results, "williamhill-questions"); - console.log(results.sort((a, b) => a.title > b.title)); - console.log("Done"); -} -//williamhill() diff --git a/src/backend/platforms/example-fetch.js b/src/backend/platforms/example-fetch.ts similarity index 86% rename from src/backend/platforms/example-fetch.js rename to src/backend/platforms/example-fetch.ts index f4e78e0..ce1d05a 100644 --- a/src/backend/platforms/example-fetch.js +++ b/src/backend/platforms/example-fetch.ts @@ -1,7 +1,8 @@ /* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; -import { calculateStars } from "../utils/stars"; +import axios from 'axios'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { calculateStars } from '../utils/stars'; /* Definitions */ let endpoint = "https://example.com/"; @@ -44,7 +45,9 @@ async function processPredictions(predictions) { options: options, timestamp: new Date().toISOString(), qualityindicators: { - stars: calculateStars("Example", { some: somex, factors: factors }), + stars: calculateStars("Example", { + /* some: somex, factors: factors */ + }), other: prediction.otherx, indicators: prediction.indicatorx, }, diff --git a/src/backend/platforms/fantasyscotus-fetch.js b/src/backend/platforms/fantasyscotus-fetch.ts similarity index 91% rename from src/backend/platforms/fantasyscotus-fetch.js rename to src/backend/platforms/fantasyscotus-fetch.ts index 83d6c54..bf08364 100644 --- a/src/backend/platforms/fantasyscotus-fetch.js +++ b/src/backend/platforms/fantasyscotus-fetch.ts @@ -1,7 +1,8 @@ /* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; -import { calculateStars } from "../utils/stars"; +import axios from 'axios'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { calculateStars } from '../utils/stars'; /* Definitions */ let unixtime = new Date().getTime(); @@ -9,8 +10,8 @@ let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount= async function fetchData() { let response = await axios({ + method: "GET", url: endpoint, - credentials: "omit", headers: { "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", @@ -19,11 +20,11 @@ async function fetchData() { "Content-Type": "application/x-www-form-urlencoded", "X-Requested-With": "XMLHttpRequest", }, - referrer: "https://fantasyscotus.net/case/list/", - method: "GET", - mode: "cors", + // referrer: "https://fantasyscotus.net/case/list/", + // credentials: "omit", + // mode: "cors", }).then((res) => res.data); - //console.log(response) + return response; } @@ -31,8 +32,8 @@ async function getPredictionsData(caseUrl) { let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`; //console.log(newCaseUrl) let predictions = await axios({ + method: "GET", url: newCaseUrl, - credentials: "include", headers: { "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", @@ -41,9 +42,9 @@ async function getPredictionsData(caseUrl) { "Content-Type": "application/x-www-form-urlencoded", "X-Requested-With": "XMLHttpRequest", }, - referrer: newCaseUrl, - method: "GET", - mode: "cors", + // referrer: newCaseUrl, + // credentials: "include", + // mode: "cors", }).then((res) => res.data); let predictionsAffirm = predictions.filter( @@ -102,7 +103,6 @@ async function processData(data) { stars: calculateStars("FantasySCOTUS", {}), }, }; - // console.log(eventObject) results.push(eventObject); } } diff --git a/src/backend/platforms/foretold-fetch.js b/src/backend/platforms/foretold-fetch.ts similarity index 100% rename from src/backend/platforms/foretold-fetch.js rename to src/backend/platforms/foretold-fetch.ts diff --git a/src/backend/platforms/givewellopenphil-fetch.js b/src/backend/platforms/givewellopenphil-fetch.ts similarity index 89% rename from src/backend/platforms/givewellopenphil-fetch.js rename to src/backend/platforms/givewellopenphil-fetch.ts index baab7fb..c8e3f70 100644 --- a/src/backend/platforms/givewellopenphil-fetch.js +++ b/src/backend/platforms/givewellopenphil-fetch.ts @@ -1,8 +1,9 @@ /* Imports */ -import axios from "axios"; -import fs from "fs"; -import { databaseUpsert } from "../database/database-wrapper"; -import { calculateStars } from "../utils/stars"; +import axios from 'axios'; +import fs from 'fs'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { calculateStars } from '../utils/stars'; /* Support functions */ async function fetchPage(url) { @@ -66,7 +67,9 @@ async function main1() { // main1() async function main2() { - let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json"); + let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json", { + encoding: "utf-8", + }); let data = JSON.parse(rawdata); let dataWithDate = data.map((datum) => ({ ...datum, diff --git a/src/backend/platforms/goodjudgment-fetch.js b/src/backend/platforms/goodjudgment-fetch.ts similarity index 92% rename from src/backend/platforms/goodjudgment-fetch.js rename to src/backend/platforms/goodjudgment-fetch.ts index a46015d..e52d248 100644 --- a/src/backend/platforms/goodjudgment-fetch.js +++ b/src/backend/platforms/goodjudgment-fetch.ts @@ -1,10 +1,11 @@ /* Imports */ -import axios from "axios"; -import { Tabletojson } from "tabletojson"; -import tunnel from "tunnel"; -import { databaseUpsert } from "../database/database-wrapper"; -import { hash } from "../utils/hash"; -import { calculateStars } from "../utils/stars"; +import axios from 'axios'; +import { Tabletojson } from 'tabletojson'; +import tunnel from 'tunnel'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { hash } from '../utils/hash'; +import { calculateStars } from '../utils/stars'; /* Definitions */ let endpoint = "https://goodjudgment.io/superforecasts/"; @@ -48,8 +49,8 @@ export async function goodjudgment() { headers: { "User-Agent": "Chrome", }, - agent, - port: 80, + // agent, + // port: 80, }) .then((query) => query.data); diff --git a/src/backend/platforms/goodjudmentopen-fetch.js b/src/backend/platforms/goodjudmentopen-fetch.ts similarity index 93% rename from src/backend/platforms/goodjudmentopen-fetch.js rename to src/backend/platforms/goodjudmentopen-fetch.ts index 0acaa0f..a52f770 100644 --- a/src/backend/platforms/goodjudmentopen-fetch.js +++ b/src/backend/platforms/goodjudmentopen-fetch.ts @@ -1,10 +1,11 @@ /* Imports */ -import axios from "axios"; -import { Tabletojson } from "tabletojson"; -import { databaseUpsert } from "../database/database-wrapper"; -import { applyIfSecretExists } from "../utils/getSecrets"; -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; +import axios from 'axios'; +import { Tabletojson } from 'tabletojson'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { applyIfSecretExists } from '../utils/getSecrets'; +import { calculateStars } from '../utils/stars'; +import toMarkdown from '../utils/toMarkdown'; /* Definitions */ let htmlEndPoint = "https://www.gjopen.com/questions?page="; @@ -14,8 +15,8 @@ let annoyingPromptUrls = [ "https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread", "https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", ]; -const DEBUG_MODE = "off"; // "on" -const id = (x) => x; +const DEBUG_MODE: "on" | "off" = "off"; // "on" +const id = () => 0; /* Support functions */ @@ -121,7 +122,9 @@ async function fetchStats(questionUrl, cookie) { maxProbability, }), }, - }; + // this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now + } as any; + return result; } diff --git a/src/backend/platforms/infer-fetch.js b/src/backend/platforms/infer-fetch.ts similarity index 95% rename from src/backend/platforms/infer-fetch.js rename to src/backend/platforms/infer-fetch.ts index cc13a65..b49b561 100644 --- a/src/backend/platforms/infer-fetch.js +++ b/src/backend/platforms/infer-fetch.ts @@ -1,17 +1,18 @@ /* Imports */ -import axios from "axios"; -import { Tabletojson } from "tabletojson"; -import { databaseUpsert } from "../database/database-wrapper"; -import { applyIfSecretExists } from "../utils/getSecrets"; -import { calculateStars } from "../utils/stars"; -import toMarkdown from "../utils/toMarkdown"; +import axios from 'axios'; +import { Tabletojson } from 'tabletojson'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { applyIfSecretExists } from '../utils/getSecrets'; +import { calculateStars } from '../utils/stars'; +import toMarkdown from '../utils/toMarkdown'; /* Definitions */ let htmlEndPoint = "https://www.infer-pub.com/questions"; String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }; -const DEBUG_MODE = "off"; // "off" +const DEBUG_MODE: "on" | "off" = "off"; // "off" const SLEEP_TIME_RANDOM = 7000; // miliseconds const SLEEP_TIME_EXTRA = 2000; /* Support functions */ @@ -229,10 +230,7 @@ async function infer_inner(cookie) { i % 30 == 0 && !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") ) { - console.log( - `Page #${i}` && - !(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") - ); + console.log(`Page #${i}`); console.log(question); } results.push(question); diff --git a/src/backend/platforms/kalshi-fetch.js b/src/backend/platforms/kalshi-fetch.ts similarity index 100% rename from src/backend/platforms/kalshi-fetch.js rename to src/backend/platforms/kalshi-fetch.ts diff --git a/src/backend/platforms/manifoldmarkets-fetch.js b/src/backend/platforms/manifoldmarkets-fetch.ts similarity index 100% rename from src/backend/platforms/manifoldmarkets-fetch.js rename to src/backend/platforms/manifoldmarkets-fetch.ts diff --git a/src/backend/platforms/metaculus-fetch.js b/src/backend/platforms/metaculus-fetch.ts similarity index 100% rename from src/backend/platforms/metaculus-fetch.js rename to src/backend/platforms/metaculus-fetch.ts diff --git a/src/backend/platforms/polymarket-fetch.js b/src/backend/platforms/polymarket-fetch.ts similarity index 100% rename from src/backend/platforms/polymarket-fetch.js rename to src/backend/platforms/polymarket-fetch.ts diff --git a/src/backend/platforms/predictit-fetch.js b/src/backend/platforms/predictit-fetch.ts similarity index 100% rename from src/backend/platforms/predictit-fetch.js rename to src/backend/platforms/predictit-fetch.ts diff --git a/src/backend/platforms/rootclaim-fetch.js b/src/backend/platforms/rootclaim-fetch.ts similarity index 100% rename from src/backend/platforms/rootclaim-fetch.js rename to src/backend/platforms/rootclaim-fetch.ts diff --git a/src/backend/platforms/smarkets-fetch.js b/src/backend/platforms/smarkets-fetch.ts similarity index 90% rename from src/backend/platforms/smarkets-fetch.js rename to src/backend/platforms/smarkets-fetch.ts index ca14366..579e66d 100644 --- a/src/backend/platforms/smarkets-fetch.js +++ b/src/backend/platforms/smarkets-fetch.ts @@ -1,12 +1,13 @@ /* Imports */ -import axios from "axios"; -import { databaseUpsert } from "../database/database-wrapper"; -import { calculateStars } from "../utils/stars"; +import axios from 'axios'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { calculateStars } from '../utils/stars'; /* Definitions */ let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/"; let VERBOSE = false; -let empty = (x) => x; +let empty = () => 0; /* Support functions */ async function fetchEvents(url) { @@ -82,7 +83,7 @@ export async function smarkets() { })); VERBOSE ? console.log("Markets fetched") : empty(); VERBOSE ? console.log(event.id) : empty(); - VERBOSE ? console.log(market) : empty(); + VERBOSE ? console.log(eventMarkets) : empty(); markets.push(...eventMarkets); //let lastPrices = await fetchPrices(market.id) } @@ -102,29 +103,29 @@ export async function smarkets() { ? console.log("Prices: ", prices["last_executed_prices"][market.id]) : empty(); - let options = {}; + let optionsObj = {}; for (let contract of contracts["contracts"]) { - options[contract.id] = { name: contract.name }; + optionsObj[contract.id] = { name: contract.name }; } for (let price of prices["last_executed_prices"][market.id]) { - options[price.contract_id] = { - ...options[price.contract_id], + optionsObj[price.contract_id] = { + ...optionsObj[price.contract_id], probability: price.last_executed_price ? Number(price.last_executed_price) : null, type: "PROBABILITY", }; } - options = Object.values(options); + let options: any[] = Object.values(optionsObj); // monkey patch the case where there are only two options and only one has traded. if ( options.length == 2 && options.map((option) => option.probability).includes(null) ) { let nonNullPrice = - option[0].probability == null - ? option[1].probability - : option[0].probability; + options[0].probability == null + ? options[1].probability + : options[0].probability; options = options.map((option) => { let probability = option.probability; return { diff --git a/src/backend/platforms/wildeford-fetch.js b/src/backend/platforms/wildeford-fetch.ts similarity index 92% rename from src/backend/platforms/wildeford-fetch.js rename to src/backend/platforms/wildeford-fetch.ts index 3e1bc34..9814d8c 100644 --- a/src/backend/platforms/wildeford-fetch.js +++ b/src/backend/platforms/wildeford-fetch.ts @@ -1,10 +1,11 @@ /* Imports */ // import axios from "axios" -import { GoogleSpreadsheet } from "google-spreadsheet"; -import { databaseUpsert } from "../database/database-wrapper"; -import { applyIfSecretExists } from "../utils/getSecrets"; -import { hash } from "../utils/hash"; -import { calculateStars } from "../utils/stars"; +import { GoogleSpreadsheet } from 'google-spreadsheet'; + +import { databaseUpsert } from '../database/database-wrapper'; +import { applyIfSecretExists } from '../utils/getSecrets'; +import { hash } from '../utils/hash'; +import { calculateStars } from '../utils/stars'; /* Definitions */ const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL @@ -98,7 +99,7 @@ async function processPredictions(predictions) { Date.parse(prediction["Prediction Date"] + "Z") ).toISOString(), qualityindicators: { - stars: calculateStars("Peter Wildeford"), + stars: calculateStars("Peter Wildeford", null), }, }; return result; diff --git a/src/backend/utils/algolia.js b/src/backend/utils/algolia.ts similarity index 91% rename from src/backend/utils/algolia.js rename to src/backend/utils/algolia.ts index d5b6bc6..8c76e47 100644 --- a/src/backend/utils/algolia.js +++ b/src/backend/utils/algolia.ts @@ -1,6 +1,7 @@ -import algoliasearch from "algoliasearch"; -import { databaseReadWithReadCredentials } from "../database/database-wrapper"; -import { mergeEverythingInner } from "../flow/mergeEverything"; +import algoliasearch from 'algoliasearch'; + +import { databaseReadWithReadCredentials } from '../database/database-wrapper'; +import { mergeEverythingInner } from '../flow/mergeEverything'; let cookie = process.env.ALGOLIA_MASTER_API_KEY; const client = algoliasearch("96UD3NTQ7L", cookie); diff --git a/src/backend/utils/evaluations/pullForecastsToCSVForRating.js b/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts similarity index 91% rename from src/backend/utils/evaluations/pullForecastsToCSVForRating.js rename to src/backend/utils/evaluations/pullForecastsToCSVForRating.ts index 144a1bb..504f56a 100644 --- a/src/backend/utils/evaluations/pullForecastsToCSVForRating.js +++ b/src/backend/utils/evaluations/pullForecastsToCSVForRating.ts @@ -1,6 +1,7 @@ /* Imports */ -import fs from "fs"; -import { databaseReadWithReadCredentials } from "../database-wrapper"; +import fs from 'fs'; + +import { databaseReadWithReadCredentials } from '../../database/database-wrapper'; /* Definitions */ diff --git a/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.js b/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts similarity index 93% rename from src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.js rename to src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts index a06e6c9..909410a 100644 --- a/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.js +++ b/src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts @@ -1,6 +1,7 @@ /* Imports */ -import fs from "fs"; -import { databaseReadWithReadCredentials } from "../database-wrapper"; +import fs from 'fs'; + +import { databaseReadWithReadCredentials } from '../../database/database-wrapper'; /* Definitions */ diff --git a/src/backend/utils/getSecrets.js b/src/backend/utils/getSecrets.ts similarity index 100% rename from src/backend/utils/getSecrets.js rename to src/backend/utils/getSecrets.ts diff --git a/src/backend/utils/hash.js b/src/backend/utils/hash.ts similarity index 51% rename from src/backend/utils/hash.js rename to src/backend/utils/hash.ts index 906f709..cb04b08 100644 --- a/src/backend/utils/hash.js +++ b/src/backend/utils/hash.ts @@ -1,4 +1,4 @@ -import crypto from "crypto"; +import crypto from 'crypto'; -export const hash = (string) => +export const hash = (string: string) => crypto.createHash("sha256").update(string).digest("hex").slice(0, 10); diff --git a/src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.js b/src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.ts similarity index 91% rename from src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.js rename to src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.ts index d7c36b6..4cbc51a 100644 --- a/src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.js +++ b/src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.ts @@ -1,11 +1,12 @@ /* Imports */ -import fs from "fs"; +import fs from 'fs'; /* Definitions */ let locationData = "../../input"; /* Body */ let rawdata = fs.readFileSync( - `${locationData}/givewellopenphil-questions.json` + `${locationData}/givewellopenphil-questions.json`, + { encoding: "utf-8" } ); let data = JSON.parse(rawdata); diff --git a/src/backend/utils/misc/process-forecasts-from-old-xrisk.js b/src/backend/utils/misc/process-forecasts-from-old-xrisk.ts similarity index 92% rename from src/backend/utils/misc/process-forecasts-from-old-xrisk.js rename to src/backend/utils/misc/process-forecasts-from-old-xrisk.ts index 2a15e04..705e4e1 100644 --- a/src/backend/utils/misc/process-forecasts-from-old-xrisk.js +++ b/src/backend/utils/misc/process-forecasts-from-old-xrisk.ts @@ -1,12 +1,13 @@ /* Imports */ -import fs from "fs"; +import fs from 'fs'; /* Definitions */ let locationData = "../../data/"; /* Body */ let rawdata = fs.readFileSync( - "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json" + "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json", + { encoding: "utf-8" } ); let data = JSON.parse(rawdata); diff --git a/src/backend/utils/misc/process-forecasts-from-xrisk.js b/src/backend/utils/misc/process-forecasts-from-xrisk.ts similarity index 89% rename from src/backend/utils/misc/process-forecasts-from-xrisk.js rename to src/backend/utils/misc/process-forecasts-from-xrisk.ts index c9245f9..09c9058 100644 --- a/src/backend/utils/misc/process-forecasts-from-xrisk.js +++ b/src/backend/utils/misc/process-forecasts-from-xrisk.ts @@ -1,12 +1,13 @@ /* Imports */ -import fs from "fs"; +import fs from 'fs'; /* Definitions */ let locationData = "../../data/"; /* Body */ let rawdata = fs.readFileSync( - "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json" + "/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json", + { encoding: "utf-8" } ); let data = JSON.parse(rawdata); diff --git a/src/backend/utils/misc/process-forecasts-into-elicit.js b/src/backend/utils/misc/process-forecasts-into-elicit.ts similarity index 94% rename from src/backend/utils/misc/process-forecasts-into-elicit.js rename to src/backend/utils/misc/process-forecasts-into-elicit.ts index 08b7bc9..766974a 100644 --- a/src/backend/utils/misc/process-forecasts-into-elicit.js +++ b/src/backend/utils/misc/process-forecasts-into-elicit.ts @@ -1,6 +1,7 @@ /* Imports */ -import fs from "fs"; -import { databaseReadWithReadCredentials } from "../database-wrapper"; +import fs from 'fs'; + +import { databaseReadWithReadCredentials } from '../../database/database-wrapper'; /* Definitions */ let locationData = "./data/"; diff --git a/src/backend/utils/misc/process-forecasts-template.js b/src/backend/utils/misc/process-forecasts-template.ts similarity index 58% rename from src/backend/utils/misc/process-forecasts-template.js rename to src/backend/utils/misc/process-forecasts-template.ts index ab1961e..cdb7dac 100644 --- a/src/backend/utils/misc/process-forecasts-template.js +++ b/src/backend/utils/misc/process-forecasts-template.ts @@ -1,11 +1,13 @@ /* Imports */ -import fs from "fs"; +import fs from 'fs'; /* Definitions */ let locationData = "./data/"; /* Body */ -let rawdata = fs.readFileSync("../data/merged-questions.json"); +let rawdata = fs.readFileSync("../data/merged-questions.json", { + encoding: "utf-8", +}); let data = JSON.parse(rawdata); let results = []; @@ -13,5 +15,5 @@ for (let datum of data) { // do something } -let string = JSON.stringify(result, null, 2); +let string = JSON.stringify(results, null, 2); fs.writeFileSync("../data/output.txt", string); diff --git a/src/backend/utils/misc/test.js b/src/backend/utils/misc/test.ts similarity index 87% rename from src/backend/utils/misc/test.js rename to src/backend/utils/misc/test.ts index d0c6488..5118f15 100644 --- a/src/backend/utils/misc/test.js +++ b/src/backend/utils/misc/test.ts @@ -1,5 +1,4 @@ -import fs from "fs"; -import axios from "axios"; +import axios from 'axios'; let elicitEndpoint = "https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community"; diff --git a/src/backend/utils/roughSize.js b/src/backend/utils/roughSize.ts similarity index 100% rename from src/backend/utils/roughSize.js rename to src/backend/utils/roughSize.ts diff --git a/src/backend/utils/stars.js b/src/backend/utils/stars.ts similarity index 100% rename from src/backend/utils/stars.js rename to src/backend/utils/stars.ts diff --git a/src/backend/utils/toMarkdown.js b/src/backend/utils/toMarkdown.ts similarity index 100% rename from src/backend/utils/toMarkdown.js rename to src/backend/utils/toMarkdown.ts diff --git a/src/pages/api/platform/[platform].js b/src/pages/api/platform/[platform].js deleted file mode 100644 index 0e88130..0000000 --- a/src/pages/api/platform/[platform].js +++ /dev/null @@ -1,6 +0,0 @@ -// see: https://nextjs.org/docs/api-routes/dynamic-api-routes - -export default function handler(req, res) { - const { platform } = req.query; - res.end(`Platform: ${platform}`); -} diff --git a/src/pages/api/platform/index.js b/src/pages/api/platform/index.js deleted file mode 100644 index e69de29..0000000 diff --git a/src/pages/api/question/[question].js b/src/pages/api/question/[question].js deleted file mode 100644 index 134c9b8..0000000 --- a/src/pages/api/question/[question].js +++ /dev/null @@ -1,6 +0,0 @@ -// see: https://nextjs.org/docs/api-routes/dynamic-api-routes - -export default function handler(req, res) { - const { question } = req.query; - res.end(`Platform: ${question}`); -} diff --git a/src/pages/api/question/index.js b/src/pages/api/question/index.js deleted file mode 100644 index e69de29..0000000 diff --git a/src/pages/secretDashboard.js b/src/pages/secretDashboard.tsx similarity index 91% rename from src/pages/secretDashboard.js rename to src/pages/secretDashboard.tsx index be72d46..bcabe18 100644 --- a/src/pages/secretDashboard.js +++ b/src/pages/secretDashboard.tsx @@ -1,11 +1,11 @@ /* Imports */ // React -import { useRouter } from "next/router"; // https://nextjs.org/docs/api-reference/next/router -import React, { useState } from "react"; -// Data -import displayForecasts from "../web/display/displayForecasts"; -import { getDashboardForecastsByDashboardId } from "../web/worker/getDashboardForecasts"; +import { useRouter } from 'next/router'; // https://nextjs.org/docs/api-reference/next/router +import { useState } from 'react'; + +import displayForecasts from '../web/display/displayForecasts'; +import { getDashboardForecastsByDashboardId } from '../web/worker/getDashboardForecasts'; /* get Props */ diff --git a/src/web/worker/getForecasts.js b/src/web/worker/getForecasts.js deleted file mode 100644 index 3ae49dd..0000000 --- a/src/web/worker/getForecasts.js +++ /dev/null @@ -1,90 +0,0 @@ -/* -import {MongoClient} from 'mongodb'; -// import pkg from 'mongodb'; -// const {MongoClient} = pkg; - -async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){ - const url = "mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database. - - const client = new MongoClient(url, { - useNewUrlParser: true, - useUnifiedTopology: true, - }); - - let documentContents - try { - await client.connect(); - // console.log(`Connected correctly to server to read ${documentName}`); - const db = client.db(databaseName); - - // Use the collection "data" - const collection = db.collection(collectionName); - - // Search options - const query = { "name": documentName }; - const options = { - // sort matched documents in descending order by rating - sort: { rating: -1 }, - }; - - // Insert a single document, wait for promise so we can read it back - // const p = await collection.insertOne(metaforecastDocument); - const document = await collection.findOne(query, options); - documentContents = document.contentsArray - } catch (err) { - console.log(err.stack); - } - finally { - await client.close(); - } - // console.log(documentContents.slice(0,10)); - return documentContents -} - -export async function getForecasts() { - const mongodbDocument = await mongoRead("metaforecasts") - // console.log(mongodbDocument) - return mongodbDocument -} - -// getForecasts() - -//---------------------------------------// - -//- Previous methods: Graphql -// -import { GraphQLClient } from "graphql-request"; -// import { request } from 'graphql-request' - -const graphcms = new GraphQLClient( - "https://api.baseql.com/airtable/graphql/apptDvDsHzSDEDARC" -); - -export async function getForecasts0() { - return await graphcms.request( - ` - query { - metaforecasts{ - id - title - url - platform - binaryQuestion - percentage - forecasts - description - stars - } - }` - ); -} - -//- Previous methods: Github JSON -// -import axios from "axios" - -export async function getForecasts1() { - const { data } = await axios.get(`https://raw.githubusercontent.com/QURIresearch/metaforecast/master/data/metaforecasts.json?ver=${Math.random().toFixed(10)}`) // this is, for now, a hack - //console.log(data) - return data -} - -*/ diff --git a/src/web/worker/searchGuesstimate.js b/src/web/worker/searchGuesstimate.ts similarity index 88% rename from src/web/worker/searchGuesstimate.js rename to src/web/worker/searchGuesstimate.ts index a1b5869..1ddb3e5 100644 --- a/src/web/worker/searchGuesstimate.js +++ b/src/web/worker/searchGuesstimate.ts @@ -1,5 +1,5 @@ /* Imports */ -import axios from "axios"; +import axios from 'axios'; /* Definitions */ let urlEndPoint = @@ -10,15 +10,15 @@ let urlEndPoint = export default function searchGuesstimate(query) { let response = axios({ url: urlEndPoint, - credentials: "omit", + // credentials: "omit", headers: { // "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0", Accept: "application/json", "Accept-Language": "en-US,en;q=0.5", "content-type": "application/x-www-form-urlencoded", }, - referrer: - "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c", + // referrer: + // "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c", data: `{\"params\":\"query=${query.replace( / /g, "%20" diff --git a/src/web/worker/searchWithAlgolia.js b/src/web/worker/searchWithAlgolia.ts similarity index 98% rename from src/web/worker/searchWithAlgolia.js rename to src/web/worker/searchWithAlgolia.ts index b0d4279..1c5955b 100644 --- a/src/web/worker/searchWithAlgolia.js +++ b/src/web/worker/searchWithAlgolia.ts @@ -1,4 +1,4 @@ -import algoliasearch from "algoliasearch"; +import algoliasearch from 'algoliasearch'; const client = algoliasearch("96UD3NTQ7L", "618dbd0092971388cfd43aac1ae5f1f5"); // Only search. const index = client.initIndex("metaforecast"); @@ -89,7 +89,7 @@ export default async function searchWithAlgolia({ //facetFilters: buildFacetFilter({filterByPlatforms}), getRankingInfo: true, }); - let results = response.hits; + let results: any[] = response.hits; console.log( "searchWithAlgolia.js/searchWithAlgolia/queryString", queryString diff --git a/src/web/worker/uploadToImgur.js b/src/web/worker/uploadToImgur.ts similarity index 86% rename from src/web/worker/uploadToImgur.js rename to src/web/worker/uploadToImgur.ts index 81eec8d..81fe3fe 100644 --- a/src/web/worker/uploadToImgur.js +++ b/src/web/worker/uploadToImgur.ts @@ -1,8 +1,8 @@ // import fetch from "fetch" -import axios from "axios"; +import axios, { AxiosRequestConfig } from 'axios'; export async function uploadToImgur(dataURL, handleGettingImgurlImage) { - let request = { + let request: AxiosRequestConfig = { method: "post", url: "https://api.imgur.com/3/image", headers: { @@ -12,7 +12,7 @@ export async function uploadToImgur(dataURL, handleGettingImgurlImage) { type: "base64", image: dataURL.split(",")[1], }, - redirect: "follow", + // redirect: "follow", }; let url; try { diff --git a/tsconfig.json b/tsconfig.json index eaaa522..42bfacc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,5 @@ { "compilerOptions": { - "allowJs": true, "target": "es6", "module": "CommonJS", "outDir": "out", @@ -23,7 +22,6 @@ "include": [ "next-env.d.ts", "src/**/*.ts", - "src/**/*.js", "src/**/*.tsx" ], "exclude": [