quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #23 from QURIresearch/more-typescript

Browse Source 
feat: More typescript & cleanups
This commit is contained in:
Nuño Sempere 2022-03-27 22:45:25 -04:00 committed by GitHub
commit 5b9b36d2ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
70 changed files with 181 additions and 1762 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/backend/database/database-wrapper.js → src/backend/database/database-wrapper.ts
View File

@ -1,9 +1,5 @@
import {
mongoRead,
mongoReadWithReadCredentials,
mongoUpsert,
} from "./mongo-wrapper";
import { pgRead, pgReadWithReadCredentials, pgUpsert } from "./pg-wrapper";
import { mongoRead, mongoReadWithReadCredentials, mongoUpsert } from './mongo-wrapper';
import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper';
export async function databaseUpsert({ contents, group }) {
// No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
@ -146,7 +142,7 @@ export async function databaseReadWithReadCredentials({ group }) {
break;
default:
mongoDocName = `${group}-questions`;
responseMongo = mongoRemongoReadWithReadCredentialsad(
responseMongo = mongoReadWithReadCredentials(
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"

6
src/backend/database/mongo-wrapper.js → src/backend/database/mongo-wrapper.ts
View File

@ -1,5 +1,7 @@
import pkg from "mongodb";
import { roughSizeOfObject } from "../utils/roughSize";
import pkg from 'mongodb';
import { roughSizeOfObject } from '../utils/roughSize';
const { MongoClient } = pkg;
export async function mongoUpsert(

10
src/backend/database/pg-wrapper.js → src/backend/database/pg-wrapper.ts
View File

@ -1,7 +1,9 @@
import pkg from "pg";
import { platformNames } from "../platforms/all/platformNames";
import { hash } from "../utils/hash";
import { roughSizeOfObject } from "../utils/roughSize";
import pkg from 'pg';
import { platformNames } from '../platforms/all/platformNames';
import { hash } from '../utils/hash';
import { roughSizeOfObject } from '../utils/roughSize';
const { Pool } = pkg;
// Definitions

11
src/backend/flow/doEverything.js → src/backend/flow/doEverything.ts
View File

@ -1,8 +1,9 @@
import { platformFetchers } from "../platforms/all-platforms";
import { rebuildAlgoliaDatabase } from "../utils/algolia";
import { updateHistory } from "./history/updateHistory";
import { mergeEverything } from "./mergeEverything";
import { rebuildNetlifySiteWithNewData } from "./rebuildNetliftySiteWithNewData";
import { platformFetchers } from '../platforms/all-platforms';
import { rebuildAlgoliaDatabase } from '../utils/algolia';
import { updateHistory } from './history/updateHistory';
import { mergeEverything } from './mergeEverything';
import { rebuildNetlifySiteWithNewData } from './rebuildNetliftySiteWithNewData';
/* Do everything */
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));

3
src/backend/flow/doEverythingForScheduler.js
View File

@ -1,3 +0,0 @@
import { doEverything } from "./doEverything";
doEverything();

3
src/backend/flow/doEverythingForScheduler.ts Normal file
View File

@ -0,0 +1,3 @@
import { doEverything } from './doEverything';
doEverything();

110
src/backend/flow/history/old/addToHistory.js
View File

@ -1,110 +0,0 @@
import {
databaseReadWithReadCredentials,
databaseUpsert,
} from "../../database/database-wrapper";
let isEmptyArray = (arr) => arr.length == 0;
export async function addToHistory() {
let currentDate = new Date();
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_");
let currentJSONwithMetaculus = await databaseReadWithReadCredentials({
group: "combined",
});
let currentJSON = currentJSONwithMetaculus.filter(
(element) =>
element.platform != "Metaculus" && element.platform != "Estimize"
); // without Metaculus
// console.log(currentJSON.slice(0,20))
// console.log(currentJSON)
let historyJSON = await databaseReadWithReadCredentials({ group: "history" });
// console.log(historyJSON)
let currentForecastsWithAHistory = currentJSON.filter(
(element) =>
!isEmptyArray(
historyJSON.filter(
(historyElement) =>
historyElement.title == element.title &&
historyElement.url == element.url
)
)
);
// console.log(currentForecastsWithAHistory)
let currentForecastsWithoutAHistory = currentJSON.filter((element) =>
isEmptyArray(
historyJSON.filter(
(historyElement) =>
historyElement.title == element.title &&
historyElement.url == element.url
)
)
);
// console.log(currentForecastsWithoutAHistory)
// Add both types of forecast
let newHistoryJSON = [];
for (let historyElement of historyJSON) {
let correspondingNewElementArray = currentForecastsWithAHistory.filter(
(element) =>
historyElement.title == element.title &&
historyElement.url == element.url
);
// console.log(correspondingNewElement)
if (!isEmptyArray(correspondingNewElementArray)) {
let correspondingNewElement = correspondingNewElementArray[0];
let timeStampOfNewElement = correspondingNewElement.timestamp;
let doesHistoryAlreadyContainElement = historyElement.history
.map((element) => element.timestamp)
.includes(timeStampOfNewElement);
if (!doesHistoryAlreadyContainElement) {
let historyWithNewElement = historyElement["history"].concat({
timestamp: correspondingNewElement.timestamp,
options: correspondingNewElement.options,
qualityindicators: correspondingNewElement.qualityindicators,
});
let newHistoryElement = {
...correspondingNewElement,
history: historyWithNewElement,
};
// If some element (like the description) changes, we keep the new one.
newHistoryJSON.push(newHistoryElement);
} else {
newHistoryJSON.push(historyElement);
}
} else {
// console.log(historyElement)
newHistoryJSON.push(historyElement);
}
}
for (let currentForecast of currentForecastsWithoutAHistory) {
let newHistoryElement = {
...currentForecast,
history: [
{
timestamp: currentForecast.timestamp,
options: currentForecast.options,
qualityindicators: currentForecast.qualityindicators,
},
],
};
delete newHistoryElement.timestamp;
delete newHistoryElement.options;
delete newHistoryElement.qualityindicators;
newHistoryJSON.push(newHistoryElement);
}
await databaseUpsert({ contents: newHistoryJSON, group: "history" });
// console.log(newHistoryJSON.slice(0,5))
// writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
// writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
//console.log(newHistoryJSON)
/*
let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
*/
//console.log(new Date().toISOString())
}
// updateHistory()

36
src/backend/flow/history/old/createHistoryForMonth.js
View File

@ -1,36 +0,0 @@
import { databaseRead, databaseUpsert } from "../../database/database-wrapper";
export async function createHistoryForMonth() {
let currentDate = new Date();
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_");
let metaforecasts = await databaseRead({ group: "combined" });
let metaforecastsHistorySeed = metaforecasts
.map((element) => {
// let moreoriginsdata = element.author ? ({author: element.author}) : ({})
return {
title: element.title,
url: element.url,
platform: element.platform,
moreoriginsdata: element.moreoriginsdata || {},
description: element.description,
history: [
{
timestamp: element.timestamp,
options: element.options,
qualityindicators: element.qualityindicators,
},
],
extra: element.extra || {},
};
})
.filter(
(element) =>
element.platform != "Metaculus" && element.platform != "Estimize"
);
//console.log(metaforecastsHistorySeed)
await databaseUpsert({
contents: metaforecastsHistorySeed,
group: "history",
});
}
////createInitialHistory()

29
src/backend/flow/history/old/createInitialHistoryWithMetaculus.js
View File

@ -1,29 +0,0 @@
import { databaseRead, databaseUpsert } from "../database-wrapper";
let createInitialHistory = async () => {
let metaforecasts = await databaseRead({ group: "combined" });
let metaforecastsHistorySeed = metaforecasts.map((element) => {
// let moreoriginsdata = element.author ? ({author: element.author}) : ({})
return {
title: element.title,
url: element.url,
platform: element.platform,
moreoriginsdata: element.moreoriginsdata || {},
description: element.description,
history: [
{
timestamp: element.timestamp,
options: element.options,
qualityindicators: element.qualityindicators,
},
],
extra: element.extra || {},
};
});
console.log(metaforecastsHistorySeed);
await databaseUpsert({
contents: metaforecastsHistorySeed,
group: "history",
});
};
createInitialHistory();

21
src/backend/flow/history/old/updateHistory.js
View File

@ -1,21 +0,0 @@
import { addToHistory } from "./addToHistory";
import { createHistoryForMonth } from "./createHistoryForMonth";
export async function updateHistoryOld() {
let currentDate = new Date();
let dayOfMonth = currentDate.getDate();
if (dayOfMonth == 1) {
console.log(
`Creating history for the month ${currentDate.toISOString().slice(0, 7)}`
);
await createHistoryForMonth();
} else {
console.log(`Updating history for ${currentDate.toISOString()}`);
await addToHistory();
}
}
export async function updateHistory() {
let currentDate = new Date();
let year = currentDate.toISOString().slice(0, 4);
}

5
src/backend/flow/history/updateHistory.js → src/backend/flow/history/updateHistory.ts
View File

@ -1,7 +1,4 @@
import {
databaseReadWithReadCredentials,
databaseUpsert,
} from "../../database/database-wrapper";
import { databaseReadWithReadCredentials, databaseUpsert } from '../../database/database-wrapper';
export async function updateHistory() {
let latest = await databaseReadWithReadCredentials({ group: "combined" });

5
src/backend/flow/mergeEverything.js → src/backend/flow/mergeEverything.ts
View File

@ -1,5 +1,6 @@
import { databaseRead, databaseUpsert } from "../database/database-wrapper";
import { platformNames } from "../platforms/all-platforms";
import { databaseRead, databaseUpsert } from '../database/database-wrapper';
import { platformNames } from '../platforms/all-platforms';
/* Merge everything */
export async function mergeEverythingInner() {

5
src/backend/flow/rebuildNetliftySiteWithNewData.js → src/backend/flow/rebuildNetliftySiteWithNewData.ts
View File

@ -1,5 +1,6 @@
import axios from "axios";
import { applyIfSecretExists } from "../utils/getSecrets";
import axios from 'axios';
import { applyIfSecretExists } from '../utils/getSecrets';
async function rebuildNetlifySiteWithNewData_inner(cookie) {
let payload = {};

8
src/backend/manual/manualDownload.js → src/backend/manual/manualDownload.ts
View File

@ -1,6 +1,8 @@
import "dotenv/config";
import fs from "fs";
import { databaseReadWithReadCredentials } from "../database/database-wrapper";
import 'dotenv/config';
import fs from 'fs';
import { databaseReadWithReadCredentials } from '../database/database-wrapper';
let main = async () => {
let json = await databaseReadWithReadCredentials({ group: "combined" });

3
src/backend/manual/manualInitialize.js
View File

@ -1,3 +0,0 @@
import { pgInitialize } from "../database/pg-wrapper";
pgInitialize();

3
src/backend/manual/manualInitialize.ts Normal file
View File

@ -0,0 +1,3 @@
import { pgInitialize } from '../database/pg-wrapper';
pgInitialize();

9
src/backend/manual/manualSendToMongo.js → src/backend/manual/manualSendToMongo.ts
View File

@ -1,5 +1,6 @@
import fs from "fs";
import { databaseUpsert } from "../database/database-wrapper";
import fs from 'fs';
import { databaseUpsert } from '../database/database-wrapper';
/* This is necessary for estimize, the database of x-risk estimates, and for the OpenPhil/GiveWell predictions. Unlike the others, I'm not fetching them constantly, but only once. */
@ -9,7 +10,9 @@ let suffixMongo = "-questions";
let main = async () => {
for (let file of pushManualFiles) {
let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`);
let fileRaw = fs.readFileSync(`./input/${file + suffixFiles}`, {
encoding: "utf-8",
});
let fileContents = JSON.parse(fileRaw);
console.log(fileContents);
await databaseUpsert({ contents: fileContents, group: file });

2
src/backend/manual/pullSuperforecastsManually.js → src/backend/manual/pullSuperforecastsManually.ts
View File

@ -1,5 +1,5 @@
/* Imports */
import { goodjudgment } from "../platforms/goodjudgment-fetch";
import { goodjudgment } from '../platforms/goodjudgment-fetch';
/* Definitions */

0
src/backend/platforms/all-platforms.js → src/backend/platforms/all-platforms.ts
View File

28
src/backend/platforms/all/platformFetchers.js → src/backend/platforms/all/platformFetchers.ts
View File

@ -1,17 +1,17 @@
import { betfair } from "../betfair-fetch";
import { fantasyscotus } from "../fantasyscotus-fetch";
import { foretold } from "../foretold-fetch";
import { goodjudgment } from "../goodjudgment-fetch";
import { goodjudgmentopen } from "../goodjudmentopen-fetch";
import { infer } from "../infer-fetch";
import { kalshi } from "../kalshi-fetch";
import { manifoldmarkets } from "../manifoldmarkets-fetch";
import { metaculus } from "../metaculus-fetch";
import { polymarket } from "../polymarket-fetch";
import { predictit } from "../predictit-fetch";
import { rootclaim } from "../rootclaim-fetch";
import { smarkets } from "../smarkets-fetch";
import { wildeford } from "../wildeford-fetch";
import { betfair } from '../betfair-fetch';
import { fantasyscotus } from '../fantasyscotus-fetch';
import { foretold } from '../foretold-fetch';
import { goodjudgment } from '../goodjudgment-fetch';
import { goodjudgmentopen } from '../goodjudmentopen-fetch';
import { infer } from '../infer-fetch';
import { kalshi } from '../kalshi-fetch';
import { manifoldmarkets } from '../manifoldmarkets-fetch';
import { metaculus } from '../metaculus-fetch';
import { polymarket } from '../polymarket-fetch';
import { predictit } from '../predictit-fetch';
import { rootclaim } from '../rootclaim-fetch';
import { smarkets } from '../smarkets-fetch';
import { wildeford } from '../wildeford-fetch';
/* Deprecated
import { astralcodexten } from "../platforms/astralcodexten-fetch"

0
src/backend/platforms/all/platformNames.js → src/backend/platforms/all/platformNames.ts
View File

0
src/backend/platforms/betfair-fetch.js → src/backend/platforms/betfair-fetch.ts
View File

97
src/backend/platforms/deprecated/astralcodexten-fetch.js
View File

@ -1,97 +0,0 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../utils/database-wrapper";
import { calculateStars } from "../utils/stars";
/* Definitions */
let graphQLendpoint = "https://api.foretold.io/graphql";
let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"];
/* Support functions */
async function fetchAllCommunityQuestions(communityId) {
let response = await axios({
url: graphQLendpoint,
method: "POST",
headers: { "Content-Type": "application/json" },
data: JSON.stringify({
query: `
query {
measurables(
channelId: "${communityId}",
states: OPEN,
first: 500
){
total
edges{
node{
id
name
valueType
measurementCount
previousAggregate{
value{
percentage
}
}
}
}
}
}
`,
}),
})
.then((res) => res.data)
.then((res) => res.data.measurables.edges);
//console.log(response)
return response;
}
/* Body */
export async function astralcodexten() {
let results = [];
for (let community of ScottAlexanderPredictions) {
let questions = await fetchAllCommunityQuestions(community);
questions = questions.map((question) => question.node);
questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
questions.forEach((question) => {
let options = [];
if (question.valueType == "PERCENTAGE") {
let probability = question.previousAggregate.value.percentage;
options = [
{
name: "Yes",
probability: probability / 100,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - probability / 100,
type: "PROBABILITY",
},
];
}
let result = {
title: question.name.split(". ")[1],
url: `https://www.foretold.io/c/${community}/m/${question.id}`,
platform: "AstralCodexTen",
description: "...by the end of 2021",
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number((question.measurementCount + 1) / 2),
stars: calculateStars("AstralCodexTen", {}),
},
/*liquidity: liquidity.toFixed(2),
tradevolume: tradevolume.toFixed(2),
address: obj.address*/
};
// console.log(result)
results.push(result);
});
}
await databaseUpsert(results, "astralcodexten-questions");
// console.log(results)
console.log("Done");
}
// astralcodexten()

184
src/backend/platforms/deprecated/coupcast-fetch.js
View File

@ -1,184 +0,0 @@
/* Imports */
import axios from "axios";
import Papa from "papaparse";
import { databaseUpsert } from "../utils/database-wrapper";
import { calculateStars } from "../utils/stars";
/* Definitions */
let coupCastEndpoint =
"https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv";
var datenow = new Date();
var currentmonth = datenow.getMonth() + 1;
dd;
/* Support functions */
let unique = (arr) => [...new Set(arr)];
let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
let sanitizeCountryName = (country_name) => {
let sanitized_name;
switch (country_name) {
case "Cen African Rep":
sanitized_name = "Central African Republic";
break;
case "Congo-Brz":
sanitized_name = "Republic of the Congo";
break;
case "Congo/Zaire":
sanitized_name = "Democratic Republic of the Congo";
break;
case "Czech Rep":
sanitized_name = "Czech Republic";
break;
case "Dominican Rep":
sanitized_name = "Dominican Republic";
break;
case "Korea North":
sanitized_name = "North Korea";
break;
case "Korea South":
sanitized_name = "South Korea";
break;
case "UKG":
sanitized_name = "UK";
break;
default:
sanitized_name = country_name;
}
return sanitized_name;
};
async function processArray(countryArray) {
let results = [];
for (let country of countryArray) {
let url = `https://www.oneearthfuture.org/activities/coup-cast`;
// We don't really want the prediction for all months; one is enough
// console.log(country.month)
if (Number(country.month) == currentmonth) {
// Monthly
country.country_name = sanitizeCountryName(country.country_name);
let processedPrediction1 = {
title: `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
url: url,
platform: "CoupCast",
description: `The current leader of ${country.country_name} is ${
country.leader_name
}, who has been in power for ${Number(country.leader_years).toFixed(
1
)} years. ${
country.country_name
} has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${
country.regime_years
} years.`,
options: [
{
name: "Yes",
probability: country.month_risk,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - country.month_risk,
type: "PROBABILITY",
},
],
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Coupcast", {}),
},
extra: {
country_name: country.country_name,
regime_type: country.regime_type,
month: country.month,
year: country.year,
leader_name: country.leader_name,
month_risk: country.month_risk,
annual_risk: country.annual_risk,
risk_change_percent: country.risk_change_percent,
regime_years: country.regime_years,
leader_years: country.leader_years,
country_code: country.country_code,
country_abb: country.country_abb,
},
};
// Yearly
let processedPrediction2 = {
title: `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
url: url,
platform: "CoupCast",
description: `The current leader of ${country.country_name} is ${
country.leader_name
}, who has been in power for ${Number(country.leader_years).toFixed(
1
)} years. ${
country.country_name
} has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${
country.regime_years
} years`,
options: [
{
name: "Yes",
probability: country.annual_risk,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - country.annual_risk,
type: "PROBABILITY",
},
],
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("CoupCast", {}),
},
extra: {
country_name: country.country_name,
regime_type: country.regime_type,
month: country.month,
year: country.year,
leader_name: country.leader_name,
month_risk: country.month_risk,
annual_risk: country.annual_risk,
risk_change_percent: country.risk_change_percent,
regime_years: country.regime_years,
leader_years: country.leader_years,
country_code: country.country_code,
country_abb: country.country_abb,
},
};
// results.push(processedPrediction1)
// Not pushing monthly
results.push(processedPrediction2);
}
}
await databaseUpsert(results, "coupcast-questions");
// console.log(results)
console.log("Done");
}
/* Body */
export async function coupcast() {
let csvContent = await axios
.get(coupCastEndpoint)
.then((query) => query.data);
await Papa.parse(csvContent, {
header: true,
complete: async (results) => {
console.log("Downloaded", results.data.length, "records.");
/* console.log(
JSON.stringify(
unique(results.data.map(country => country.country_name)),
null,
4
)
)*/
// console.log(results.data)
await processArray(results.data);
},
});
await sleep(1000); // needed to wait for Papaparse's callback to be executed.
}
// coupcast()

280
src/backend/platforms/deprecated/csetforetell-fetch.js
View File

@ -1,280 +0,0 @@
/* Imports */
import axios from "axios";
import { Tabletojson } from "tabletojson";
import { databaseUpsert } from "../utils/database-wrapper";
import { applyIfSecretExists } from "../utils/getSecrets";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
/* Definitions */
let htmlEndPoint = "https://www.cset-foretell.com/questions?page=";
String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace);
};
const DEBUG_MODE = "on"; // "off"
const SLEEP_TIME_RANDOM = 100; //5000 // miliseconds
const SLEEP_TIME_EXTRA = 0; //1000
/* Support functions */
async function fetchPage(page, cookie) {
console.log(`Page #${page}`);
if (page == 1) {
cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :(
}
let urlEndpoint = htmlEndPoint + page;
console.log(urlEndpoint);
let response = await axios({
url: urlEndpoint,
method: "GET",
headers: {
"Content-Type": "text/html",
Cookie: cookie,
},
}).then((res) => res.data);
// console.log(response)
return response;
}
async function fetchStats(questionUrl, cookie) {
let response = await axios({
url: questionUrl + "/stats",
method: "GET",
headers: {
"Content-Type": "text/html",
Cookie: cookie,
Referer: questionUrl,
},
}).then((res) => res.data);
if (response.includes("Sign up or sign in to forecast")) {
throw Error("Not logged in");
}
// Is binary?
let isbinary = response.includes("binary?":true");
// console.log(`is binary? ${isbinary}`)
let options = [];
if (isbinary) {
// Crowd percentage
let htmlElements = response.split("\n");
// DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
// DEBUG_MODE == "on" ? console.log(h5elements) : id()
let crowdpercentage = h3Element.split(">")[1].split("<")[0];
let probability = Number(crowdpercentage.replace("%", "")) / 100;
options.push(
{
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
name: "No",
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
type: "PROBABILITY",
}
);
} else {
try {
let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table.
// console.log(optionsBody)
let optionsHtmlElement = "<table" + optionsBody + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0];
options = firstTable.map((element) => ({
name: element["0"],
probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY",
}));
} catch (error) {
let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position
let optionsHtmlElement = "<table" + optionsBody + "table>";
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
let firstTable = tablesAsJson[0];
if (firstTable) {
options = firstTable.map((element) => ({
name: element["0"],
probability: Number(element["1"].replace("%", "")) / 100,
type: "PROBABILITY",
}));
} else {
// New type of question, tricky to parse the options
// Just leave options = [] for now.
// https://www.cset-foretell.com/blog/rolling-question-formats
}
}
}
// Description
let descriptionraw = response.split(`<meta name="description" content="`)[1];
let descriptionprocessed1 = descriptionraw.split(`">`)[0];
let descriptionprocessed2 = descriptionprocessed1.replace(">", "");
let descriptionprocessed3 = descriptionprocessed2.replace(
"To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.",
``
);
// console.log(descriptionprocessed3)
let descriptionprocessed4 = descriptionprocessed3.replaceAll(
"\r\n\r\n",
"\n"
);
let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n");
let descriptionprocessed6 = descriptionprocessed5.replaceAll("&quot;", `"`);
let descriptionprocessed7 = descriptionprocessed6.replaceAll("&#39;", "'");
let descriptionprocessed8 = toMarkdown(descriptionprocessed7);
let description = descriptionprocessed8;
// Number of forecasts
//console.log(response)
//console.log(response.split("prediction_sets_count&quot;:")[1])
let numforecasts = response
.split("prediction_sets_count&quot;:")[1]
.split(",")[0];
// console.log(numforecasts)
// Number of predictors
let numforecasters = response
.split("predictors_count&quot;:")[1]
.split(",")[0];
// console.log(numpredictors)
let result = {
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(numforecasts),
numforecasters: Number(numforecasters),
stars: calculateStars("CSET-foretell", { numforecasts }),
},
};
return result;
}
function isSignedIn(html) {
let isSignedInBool = !(
html.includes("You need to sign in or sign up before continuing") ||
html.includes("Sign up")
);
if (!isSignedInBool) {
console.log("Error: Not signed in.");
}
console.log(`Signed in? ${isSignedInBool}`);
return isSignedInBool;
}
function isEnd(html) {
let isEndBool = html.includes("No questions match your filter");
if (isEndBool) {
//console.log(html)
}
console.log(`IsEnd? ${isEndBool}`);
return isEndBool;
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/* Body */
async function csetforetell_inner(cookie) {
let i = 1;
let response = await fetchPage(i, cookie);
let results = [];
let init = Date.now();
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while (!isEnd(response) && isSignedIn(response)) {
let htmlLines = response.split("\n");
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
let questionHrefs = htmlLines.filter((str) =>
str.includes("https://www.cset-foretell.com/questions/")
);
// console.log(questionHrefs)
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
//console.log(response)
console.log("questionHrefs: ");
console.log(questionHrefs);
}
//console.log("")
//console.log("")
//console.log(h4elements)
for (let questionHref of questionHrefs) {
//console.log(h4element)
let elementSplit = questionHref.split('"><span>');
let url = elementSplit[0].split('<a href="')[1];
let title = elementSplit[1]
.replace("</h4>", "")
.replace("</h5>", "")
.replace("</span></a>", "");
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
let moreinfo = await fetchStats(url, cookie);
let question = {
title: title,
url: url,
platform: "CSET-foretell",
...moreinfo,
};
if (
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(
`Page #${i}` &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
);
console.log(question);
}
results.push(question);
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
console.log(url);
console.log(question);
}
} catch (error) {
console.log(error);
console.log(
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
);
}
}
i++;
//i=Number(i)+1
console.log(
"Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers"
);
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
try {
response = await fetchPage(i, cookie);
} catch (error) {
console.log(error);
console.log(
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
);
}
}
if (results.length > 0) {
await databaseUpsert(results, "csetforetell-questions");
} else {
console.log("Not updating results, as process was not signed in");
}
let end = Date.now();
let difference = end - init;
console.log(
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
);
}
export async function csetforetell() {
let cookie = process.env.CSETFORETELL_COOKIE;
await applyIfSecretExists(cookie, csetforetell_inner);
}

121
src/backend/platforms/deprecated/elicit-fetch.js
View File

@ -1,121 +0,0 @@
/* Imports */
import axios from "axios";
import Papa from "papaparse";
import readline from "readline";
import { databaseUpsert } from "../utils/database-wrapper";
import { calculateStars } from "../utils/stars";
/* Definitions */
let elicitEndpoint =
"https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";
/* Support functions */
let avg = (array) =>
array.reduce((a, b) => Number(a) + Number(b)) / array.length;
let unique = (arr) => [...new Set(arr)];
let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function processArray(arrayQuestions) {
let questions = arrayQuestions.map((question) => question.questionTitle);
let uniqueQuestions = unique(questions);
let questionsObj = {};
uniqueQuestions.forEach((questionTitle) => {
questionsObj[questionTitle] = {
title: questionTitle,
forecasters: [],
forecasts: [],
};
});
arrayQuestions.forEach((question) => {
// console.log(question.questionTitle)
let questionTitle = question.questionTitle;
let correspondingQuestion = questionsObj[questionTitle];
let forecasters = correspondingQuestion.forecasters.concat(
question.predictionCreator
);
let forecasts = correspondingQuestion.forecasts.concat(question.prediction);
questionsObj[questionTitle] = {
forecasters,
forecasts,
};
});
let onlyQuestionsWithMoreThan;
let results = [];
for (let question in questionsObj) {
let title = question;
let forecasters = questionsObj[question].forecasters;
let numforecasters = unique(forecasters).length;
if (numforecasters >= 10) {
let url = `https://elicit.org/binary?binaryQuestions.search=${title.replace(
/ /g,
"%20"
)}&binaryQuestions.sortBy=popularity&limit=20&offset=0`;
let forecasts = questionsObj[question].forecasts;
//console.log(forecasts)
//console.log(avg(forecasts))
let probability = avg(forecasts) / 100;
let numforecasts = forecasts.length;
let standardObj = {
title: title,
url: url,
platform: "Elicit",
options: [
{
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
],
timestamp: new Date().toISOString(),
qualityindicators: {
numforecasts: Number(numforecasts),
numforecasters: Number(numforecasters),
stars: calculateStars("Elicit", {}),
},
};
results.push(standardObj);
}
}
await databaseUpsert(results, "elicit-questions");
console.log("Done");
}
async function awaitdownloadconfirmation(message, callback) {
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
});
rl.question(message, (answer) => {
//console.log("Received");
rl.close();
callback();
});
}
/* Body */
export async function elicit() {
let csvContent = await axios.get(elicitEndpoint).then((query) => query.data);
await Papa.parse(csvContent, {
header: true,
complete: async (results) => {
console.log("Downloaded", results.data.length, "records.");
//resolve(results.data);
//console.log(results.data)
await processArray(results.data);
},
});
await sleep(5000); // needed to wait for Papaparse's callback to be executed.
}
//elicit()

35
src/backend/platforms/deprecated/estimize-fetch.js
View File

@ -1,35 +0,0 @@
import fs from "fs";
import { databaseUpsert } from "../utils/database-wrapper";
import { calculateStars } from "../utils/stars";
export async function estimize() {
let data = fs.readFileSync(
"./input/s-and-p-500-companies/companies.csv",
"utf8"
);
let splitData = data.split("\n");
let results = [];
for (let datum of splitData) {
if (datum != "") {
//console.log(datum)
let datumSplit = datum.split(",");
let companyStickerSymbol = datumSplit[0];
let companyName = datumSplit[1];
let standardObj = {
title: `Estimates for ${companyName} (${companyStickerSymbol})`,
url: `https://www.estimize.com/${companyStickerSymbol.toLowerCase()}`,
platform: "Estimize",
description: `A link to Estimize's forecasts for *${companyName}* (sticker symbol ${companyStickerSymbol}). Viewing them requires making a prediction, Wall Street estimates are provided for free`,
options: [],
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Estimize", {}),
},
};
results.push(standardObj);
}
}
await databaseUpsert(results, "estimize-questions");
}
//estimize()

201
src/backend/platforms/deprecated/hypermind-fetch.js
View File

@ -1,201 +0,0 @@
/* Imports */
import axios from "axios";
import https from "https";
import { databaseUpsert } from "../utils/database-wrapper";
import { applyIfSecretExists } from "../utils/getSecrets";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
/* Definitions */
let hypermindEnpoint1 = "https://predict.hypermind.com/dash/jsx.json";
let hypermindEnpoint2 = "https://prod.hypermind.com/ngdp-jsx/jsx.json";
const insecureHttpsAgent = new https.Agent({
rejectUnauthorized: false, // (NOTE: this will disable client verification)
});
/* Support Functions */
String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace);
};
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/* Fetchers */
async function fetchHypermindData1(slug) {
let jsx = `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`;
// console.log(jsx)
let response = await await axios(hypermindEnpoint1, {
credentials: "omit",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Cache-Control": "max-age=0",
},
referrer: `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`,
data: jsx,
method: "POST",
mode: "cors",
httpsAgent: insecureHttpsAgent,
}).then((response) => response.data[0].questions);
//console.log(response)
return response;
}
async function fetchHypermindDataShowcases(slug, cookie) {
let response = await axios(hypermindEnpoint2, {
credentials: "include",
headers: {
"User-Agent": "",
Accept: "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8",
//"Cookie": cookie
},
referrer:
"https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
data: `[["showcase","getShowcase",{"showcase":"${slug}","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":false}}]]`,
method: "POST",
mode: "cors",
httpsAgent: insecureHttpsAgent,
})
.then((resp) => resp.data[0].items)
.then((items) => items.filter((item) => item.type == "IFP"))
.then((items) => items.map((item) => item.IFP));
// console.log(response)
// response.forEach(item => delete item.crowdFcstHist)
return response;
}
/* Body */
async function hypermind_inner(cookie) {
// Hypermind panelists and competitors; dashboard type two: "showcase"
// https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=SLUG
// E.g., https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=AI2023
let slugs2 = ["AI2030", "Covid19", "DOSES", "H5N8", "NGDP", "JSAI", "AI2023"]; // []
let results2 = [];
for (let slug of slugs2) {
console.log(slug);
await sleep(1000 + Math.random() * 1000);
let response = await fetchHypermindDataShowcases(slug);
let objs = response.map((result) => {
let descriptionraw = result.props.details.split("<hr size=1>")[0];
let descriptionprocessed1 = toMarkdown(descriptionraw);
let descriptionprocessed2 = descriptionprocessed1.replaceAll(
"![image] ()",
""
);
let descriptionprocessed3 = descriptionprocessed2.replaceAll(
" Forecasting Schedule ",
""
);
let descriptionprocessed4 = descriptionprocessed3
.replaceAll("\n", " ")
.replaceAll(" ", " ");
let descriptionprocessed5 = descriptionprocessed4.replaceAll(
"Context:",
""
);
let description =
descriptionprocessed5 || toMarkdown(result.props.details);
return {
title: result.props.title,
url:
"https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=" +
slug,
platform: "Hypermind",
description: description,
options: [],
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Hypermind", {}),
numforecasters: Number(result.fcsterCnt),
},
};
});
// console.log(objs)
results2.push(...objs);
}
// Prediction markets; dashboard type one.
// https://predict.hypermind.com/dash/dash/dash.html?list=SLUG
// e.g., https://predict.hypermind.com/dash/dash/dash.html?list=POL
let slugs1 = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]; // []
let results1 = [];
for (let slug of slugs1) {
console.log(slug);
await sleep(2000 + Math.random() * 2000);
let result = await fetchHypermindData1(slug);
let objs = result.map((res) => {
let descriptionraw = res.props.details;
let descriptionprocessed1 = descriptionraw.split("%%fr")[0];
let descriptionprocessed2 = descriptionprocessed1.replaceAll(
"<BR>",
"\n"
);
let descriptionprocessed3 = descriptionprocessed2.replace("%%en:", "");
let descriptionprocessed4 = descriptionprocessed3.replace(
`Shares of the correct outcome will be worth 100<sup>ℍ</sup>, while the others will be worthless (0<sup>ℍ</sup>).<p>`,
""
);
let descriptionprocessed5 = toMarkdown(descriptionprocessed4);
let description = descriptionprocessed5
.replaceAll("\n", " ")
.replaceAll(" ", " ");
//console.log(res.otcms)
//let percentage = (res.otcms.length==2) ? Number(res.otcms[0].price).toFixed(0) +"%" : "none"
let options = res.otcms.map((option) => ({
name: option.props.title.split("%%fr")[0].replaceAll("%%en:", ""),
probability: Number(option.price) / 100,
type: "PROBABILITY",
}));
return {
title: res.props.title.split("%%fr")[0].replace("%%en:", ""),
url: "https://predict.hypermind.com/dash/dash/dash.html?list=" + slug,
platform: "Hypermind",
description: description,
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Hypermind", {}),
// "numforecasters": res.fcsterCnt
},
};
});
// console.log(objs)
results1.push(...objs);
}
let resultsTotal = [...results1, ...results2];
let distinctTitles = [];
let resultsTotalUnique = [];
for (let result of resultsTotal) {
if (!distinctTitles.includes(result["title"])) {
resultsTotalUnique.push(result);
distinctTitles.push(result["title"]);
}
}
// console.log(resultsTotal)
// console.log(resultsTotalUnique)
console.log(resultsTotalUnique.length, "results");
await databaseUpsert(resultsTotalUnique, "hypermind-questions");
}
//hypermind()
export async function hypermind() {
let cookie = process.env.HYPERMINDCOOKIE;
await applyIfSecretExists(cookie, hypermind_inner);
}

142
src/backend/platforms/deprecated/ladbrokes-fetch.js
View File

@ -1,142 +0,0 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../utils/database-wrapper";
import { calculateStars } from "../utils/stars";
/* Definitions */
let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`;
let enpointDrillDown = (id) =>
`https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`;
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
/* Support functions */
async function fetchUrl(url) {
let response = await axios(url, {
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0",
},
method: "GET",
mode: "cors",
}).then((response) => response.data);
return response;
}
let processResults = async (json) => {
let results = [];
let children = json.SSResponse.children;
children.pop();
let ids = children.map((child) => child.event.id);
let markets = [];
for (let id of ids) {
let marketsContainer = await fetchUrl(enpointDrillDown(id));
let marketsObj = marketsContainer.SSResponse.children[0].event;
let newMarkets = marketsObj.children;
newMarkets = newMarkets.map((market) => ({
...market.market,
parent: marketsObj.name,
}));
markets.push(...newMarkets);
}
let normalMarkets = markets.filter(
(market) => !market.name.includes("Specials")
);
//console.log(normalMarkets)
for (let normalMarket of normalMarkets) {
let title = normalMarket.parent + ": " + normalMarket.name;
title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
let options = normalMarket.children.map((child) => {
let name = child.outcome.name;
let priceData = child.outcome.children[0].price;
let priceDecimal = Number(priceData.priceDec);
let probability = 1 / priceDecimal;
let option = {
name: name,
probability: probability,
type: "PROBABILITY",
};
return option;
});
// normalize probabilities
let totalValue = options
.map((element) => Number(element.probability))
.reduce((a, b) => a + b, 0);
options = options.map((element) => ({
...element,
probability: Number(element.probability) / totalValue,
}));
// Filter very unlikely probabilities: Not here, but on the front end
// options = options.filter(element => element.probability > 0.02)
let obj = {
title: title,
url: "https://sports.ladbrokes.com/sport/politics/outrights",
platform: "Ladbrokes",
description: "",
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Ladbrokes", {}),
},
};
results.push(obj);
}
let specialMarkets = markets.filter((market) =>
market.name.includes("Specials")
);
for (let specialMarket of specialMarkets) {
//let title = specialMarket.parent + ": " + specialMarket.name
//console.log(title)
specialMarket.children.forEach((child) => {
let name = specialMarket.parent.includes("Specials")
? child.outcome.name
: specialMarket.parent + ": " + child.outcome.name;
name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
let priceData = child.outcome.children[0].price;
let priceDecimal = Number(priceData.priceDec);
let probability = 1 / priceDecimal;
let obj = {
title: name,
url: "https://sports.ladbrokes.com/sport/politics/outrights",
platform: "LadBrokes",
options: [
{
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
],
qualityindicators: {
stars: calculateStars("Ladbrokes", {}),
},
};
results.push(obj);
});
}
return results;
};
/* Body */
export async function ladbrokes() {
let response = await fetchUrl(endpointPolitics);
let results = await processResults(response);
await databaseUpsert(results, "ladbrokes-questions");
console.log("Done");
}
//ladbrokes()

97
src/backend/platforms/deprecated/omen-fetch.js
View File

@ -1,97 +0,0 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../../utils/database-wrapper";
import { calculateStars } from "../../utils/stars";
/* Definitions */
let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen";
// "https://gateway.thegraph.com/api/[api-key]/subgraphs/id/0x0503024fcc5e1bd834530e69d592dbb6e8c03968-0"
// 'https://api.thegraph.com/subgraphs/name/protofire/omen'
// https://github.com/protofire/omen-subgraph
// https://thegraph.com/explorer/subgraph/protofire/omen
async function fetchAllContractData() {
let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 50; // last 30 days
let response = await axios({
url: graphQLendpoint,
method: "POST",
headers: { "Content-Type": "application/json" },
data: JSON.stringify({
query: `
{
fixedProductMarketMakers(first: 1000,
where: {
lastActiveDay_gt: ${daysSinceEra}
}
){
id
lastActiveDay
question{
title
}
outcomeSlotCount
outcomeTokenMarginalPrices
usdVolume
usdLiquidityMeasure
resolutionTimestamp
}
}
`,
}),
})
.then((res) => res.data)
.then((res) => res.data.fixedProductMarketMakers);
console.log(response);
return response;
}
async function fetch_all() {
let allData = await fetchAllContractData();
let results = [];
for (let data of allData) {
if (
(data.question != null) &
(data.usdLiquidityMeasure != "0") &
(data.resolutionTimestamp == null) &
(data.question.title != "ssdds")
) {
// console.log(data)
// console.log(data.usdLiquidityMeasure)
let options = data.outcomeTokenMarginalPrices.map((price, slotnum) => {
let name = `Option ${slotnum}`;
if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 0)
name = "Yes";
if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 1)
name = "No";
return {
name: name,
probability: Number(price),
type: "PROBABILITY",
};
});
let obj = {
title: data.question.title,
url: "https://omen.eth.link/#/" + data.id,
platform: "Omen",
description: "",
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Omen", {}),
},
};
// console.log(obj)
results.push(obj);
}
}
return results;
}
/* Body */
export async function omen() {
let results = await fetch_all();
await databaseUpsert(results, "omen-questions");
console.log("Done");
}
//omen()

147
src/backend/platforms/deprecated/williamhill-fetch.js
View File

@ -1,147 +0,0 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../utils/database-wrapper";
import { calculateStars } from "../utils/stars";
/* Definitions */
let endpoint = "https://sports.williamhill.com/betting/en-gb/politics";
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
/* Support functions */
async function fetchUrl(url) {
let response = await axios(url, {
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Cache-Control": "max-age=0",
},
method: "GET",
mode: "cors",
}).then((response) => response.data);
return response;
}
let processResults = (html) => {
let results = [];
let chunks = html.split(
'<header class="header-dropdown header-dropdown--large -expanded" data-id="'
);
chunks.shift();
// Kamala Special
let kamalaspecial = chunks[0];
let kamalamarkets = kamalaspecial.split(
'<div class="btmarket__selection"><p class="btmarket__name"><span>'
);
kamalamarkets.shift();
for (let kamalamarket of kamalamarkets) {
let title = kamalamarket.split("</span>")[0];
let numerator = Number(kamalamarket.split('data-num="')[1].split('"')[0]);
let denominator = Number(
kamalamarket.split('data-denom="')[1].split('"')[0]
);
let probability = denominator / (numerator + denominator);
let obj = {
title: title,
url: "https://sports.williamhill.com/betting/en-gb/politics",
platform: "WilliamHill",
description: "",
timestamp: new Date().toISOString(),
options: [
{
name: "Yes",
probability: probability,
type: "PROBABILITY",
},
{
name: "No",
probability: 1 - probability,
type: "PROBABILITY",
},
],
qualityindicators: {
stars: calculateStars("WilliamHill", {}),
},
};
results.push(obj);
}
chunks.shift();
// Deal with the other markets
for (let chunk of chunks) {
let title = chunk.split('"')[0];
let title2 = chunk.split('<a title="')[1].split('"')[0];
title = title.length > title2.length ? title : title2;
let options = [];
let alternatives = chunk.split(
'<div class="btmarket__selection"><p class="btmarket__name"><span>'
);
alternatives.shift();
for (let alternative of alternatives) {
let optionName = alternative.split("</span>")[0];
let numerator = Number(alternative.split('data-num="')[1].split('"')[0]);
let denominator = Number(
alternative.split('data-denom="')[1].split('"')[0]
);
let option = {
name: optionName,
probability: denominator / (numerator + denominator),
type: "PROBABILITY",
};
options.push(option);
}
// normalize probabilities
let totalValue = options
.map((element) => Number(element.probability))
.reduce((a, b) => a + b, 0);
options = options.map((element) => ({
...element,
probability: Number(element.probability) / totalValue,
}));
// Filter very unlikely probabilities: Not here, but on the front end
// options = options.filter(element => element.probability > 0.02)
let obj = {
title: title,
url: "https://sports.williamhill.com/betting/en-gb/politics",
platform: "WilliamHill",
options: options,
qualityindicators: {
stars: calculateStars("WilliamHill", {}),
},
};
results = results.filter(
(result) => result.title.length > 4 && result.title != "2024 or later"
);
// removes some predictions because hard to parse.
results.push(obj);
}
//console.log(results)
return results;
};
let processhtml2 = (html) => {
html.split();
};
/* Body */
export async function williamhill() {
let response = await fetchUrl(
"https://sports.williamhill.com/betting/en-gb/politics"
);
let results = processResults(response);
await databaseUpsert(results, "williamhill-questions");
console.log(results.sort((a, b) => a.title > b.title));
console.log("Done");
}
//williamhill()

11
src/backend/platforms/example-fetch.js → src/backend/platforms/example-fetch.ts
View File

@ -1,7 +1,8 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../database/database-wrapper";
import { calculateStars } from "../utils/stars";
import axios from 'axios';
import { databaseUpsert } from '../database/database-wrapper';
import { calculateStars } from '../utils/stars';
/* Definitions */
let endpoint = "https://example.com/";
@ -44,7 +45,9 @@ async function processPredictions(predictions) {
options: options,
timestamp: new Date().toISOString(),
qualityindicators: {
stars: calculateStars("Example", { some: somex, factors: factors }),
stars: calculateStars("Example", {
/* some: somex, factors: factors */
}),
other: prediction.otherx,
indicators: prediction.indicatorx,
},

26
src/backend/platforms/fantasyscotus-fetch.js → src/backend/platforms/fantasyscotus-fetch.ts
View File

@ -1,7 +1,8 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../database/database-wrapper";
import { calculateStars } from "../utils/stars";
import axios from 'axios';
import { databaseUpsert } from '../database/database-wrapper';
import { calculateStars } from '../utils/stars';
/* Definitions */
let unixtime = new Date().getTime();
@ -9,8 +10,8 @@ let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=
async function fetchData() {
let response = await axios({
method: "GET",
url: endpoint,
credentials: "omit",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
@ -19,11 +20,11 @@ async function fetchData() {
"Content-Type": "application/x-www-form-urlencoded",
"X-Requested-With": "XMLHttpRequest",
},
referrer: "https://fantasyscotus.net/case/list/",
method: "GET",
mode: "cors",
// referrer: "https://fantasyscotus.net/case/list/",
// credentials: "omit",
// mode: "cors",
}).then((res) => res.data);
//console.log(response)
return response;
}
@ -31,8 +32,8 @@ async function getPredictionsData(caseUrl) {
let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`;
//console.log(newCaseUrl)
let predictions = await axios({
method: "GET",
url: newCaseUrl,
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
@ -41,9 +42,9 @@ async function getPredictionsData(caseUrl) {
"Content-Type": "application/x-www-form-urlencoded",
"X-Requested-With": "XMLHttpRequest",
},
referrer: newCaseUrl,
method: "GET",
mode: "cors",
// referrer: newCaseUrl,
// credentials: "include",
// mode: "cors",
}).then((res) => res.data);
let predictionsAffirm = predictions.filter(
@ -102,7 +103,6 @@ async function processData(data) {
stars: calculateStars("FantasySCOTUS", {}),
},
};
// console.log(eventObject)
results.push(eventObject);
}
}

0
src/backend/platforms/foretold-fetch.js → src/backend/platforms/foretold-fetch.ts
View File

13
src/backend/platforms/givewellopenphil-fetch.js → src/backend/platforms/givewellopenphil-fetch.ts
View File

@ -1,8 +1,9 @@
/* Imports */
import axios from "axios";
import fs from "fs";
import { databaseUpsert } from "../database/database-wrapper";
import { calculateStars } from "../utils/stars";
import axios from 'axios';
import fs from 'fs';
import { databaseUpsert } from '../database/database-wrapper';
import { calculateStars } from '../utils/stars';
/* Support functions */
async function fetchPage(url) {
@ -66,7 +67,9 @@ async function main1() {
// main1()
async function main2() {
let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json");
let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json", {
encoding: "utf-8",
});
let data = JSON.parse(rawdata);
let dataWithDate = data.map((datum) => ({
...datum,

17
src/backend/platforms/goodjudgment-fetch.js → src/backend/platforms/goodjudgment-fetch.ts
View File

@ -1,10 +1,11 @@
/* Imports */
import axios from "axios";
import { Tabletojson } from "tabletojson";
import tunnel from "tunnel";
import { databaseUpsert } from "../database/database-wrapper";
import { hash } from "../utils/hash";
import { calculateStars } from "../utils/stars";
import axios from 'axios';
import { Tabletojson } from 'tabletojson';
import tunnel from 'tunnel';
import { databaseUpsert } from '../database/database-wrapper';
import { hash } from '../utils/hash';
import { calculateStars } from '../utils/stars';
/* Definitions */
let endpoint = "https://goodjudgment.io/superforecasts/";
@ -48,8 +49,8 @@ export async function goodjudgment() {
headers: {
"User-Agent": "Chrome",
},
agent,
port: 80,
// agent,
// port: 80,
})
.then((query) => query.data);

21
src/backend/platforms/goodjudmentopen-fetch.js → src/backend/platforms/goodjudmentopen-fetch.ts
View File

@ -1,10 +1,11 @@
/* Imports */
import axios from "axios";
import { Tabletojson } from "tabletojson";
import { databaseUpsert } from "../database/database-wrapper";
import { applyIfSecretExists } from "../utils/getSecrets";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
import axios from 'axios';
import { Tabletojson } from 'tabletojson';
import { databaseUpsert } from '../database/database-wrapper';
import { applyIfSecretExists } from '../utils/getSecrets';
import { calculateStars } from '../utils/stars';
import toMarkdown from '../utils/toMarkdown';
/* Definitions */
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
@ -14,8 +15,8 @@ let annoyingPromptUrls = [
"https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
"https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
];
const DEBUG_MODE = "off"; // "on"
const id = (x) => x;
const DEBUG_MODE: "on" | "off" = "off"; // "on"
const id = () => 0;
/* Support functions */
@ -121,7 +122,9 @@ async function fetchStats(questionUrl, cookie) {
maxProbability,
}),
},
};
// this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now
} as any;
return result;
}

20
src/backend/platforms/infer-fetch.js → src/backend/platforms/infer-fetch.ts
View File

@ -1,17 +1,18 @@
/* Imports */
import axios from "axios";
import { Tabletojson } from "tabletojson";
import { databaseUpsert } from "../database/database-wrapper";
import { applyIfSecretExists } from "../utils/getSecrets";
import { calculateStars } from "../utils/stars";
import toMarkdown from "../utils/toMarkdown";
import axios from 'axios';
import { Tabletojson } from 'tabletojson';
import { databaseUpsert } from '../database/database-wrapper';
import { applyIfSecretExists } from '../utils/getSecrets';
import { calculateStars } from '../utils/stars';
import toMarkdown from '../utils/toMarkdown';
/* Definitions */
let htmlEndPoint = "https://www.infer-pub.com/questions";
String.prototype.replaceAll = function replaceAll(search, replace) {
return this.split(search).join(replace);
};
const DEBUG_MODE = "off"; // "off"
const DEBUG_MODE: "on" | "off" = "off"; // "off"
const SLEEP_TIME_RANDOM = 7000; // miliseconds
const SLEEP_TIME_EXTRA = 2000;
/* Support functions */
@ -229,10 +230,7 @@ async function infer_inner(cookie) {
i % 30 == 0 &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
) {
console.log(
`Page #${i}` &&
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
);
console.log(`Page #${i}`);
console.log(question);
}
results.push(question);

0
src/backend/platforms/kalshi-fetch.js → src/backend/platforms/kalshi-fetch.ts
View File

0
src/backend/platforms/manifoldmarkets-fetch.js → src/backend/platforms/manifoldmarkets-fetch.ts
View File

0
src/backend/platforms/metaculus-fetch.js → src/backend/platforms/metaculus-fetch.ts
View File

0
src/backend/platforms/polymarket-fetch.js → src/backend/platforms/polymarket-fetch.ts
View File

0
src/backend/platforms/predictit-fetch.js → src/backend/platforms/predictit-fetch.ts
View File

0
src/backend/platforms/rootclaim-fetch.js → src/backend/platforms/rootclaim-fetch.ts
View File

27
src/backend/platforms/smarkets-fetch.js → src/backend/platforms/smarkets-fetch.ts
View File

@ -1,12 +1,13 @@
/* Imports */
import axios from "axios";
import { databaseUpsert } from "../database/database-wrapper";
import { calculateStars } from "../utils/stars";
import axios from 'axios';
import { databaseUpsert } from '../database/database-wrapper';
import { calculateStars } from '../utils/stars';
/* Definitions */
let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
let VERBOSE = false;
let empty = (x) => x;
let empty = () => 0;
/* Support functions */
async function fetchEvents(url) {
@ -82,7 +83,7 @@ export async function smarkets() {
}));
VERBOSE ? console.log("Markets fetched") : empty();
VERBOSE ? console.log(event.id) : empty();
VERBOSE ? console.log(market) : empty();
VERBOSE ? console.log(eventMarkets) : empty();
markets.push(...eventMarkets);
//let lastPrices = await fetchPrices(market.id)
}
@ -102,29 +103,29 @@ export async function smarkets() {
? console.log("Prices: ", prices["last_executed_prices"][market.id])
: empty();
let options = {};
let optionsObj = {};
for (let contract of contracts["contracts"]) {
options[contract.id] = { name: contract.name };
optionsObj[contract.id] = { name: contract.name };
}
for (let price of prices["last_executed_prices"][market.id]) {
options[price.contract_id] = {
...options[price.contract_id],
optionsObj[price.contract_id] = {
...optionsObj[price.contract_id],
probability: price.last_executed_price
? Number(price.last_executed_price)
: null,
type: "PROBABILITY",
};
}
options = Object.values(options);
let options: any[] = Object.values(optionsObj);
// monkey patch the case where there are only two options and only one has traded.
if (
options.length == 2 &&
options.map((option) => option.probability).includes(null)
) {
let nonNullPrice =
option[0].probability == null
? option[1].probability
: option[0].probability;
options[0].probability == null
? options[1].probability
: options[0].probability;
options = options.map((option) => {
let probability = option.probability;
return {

13
src/backend/platforms/wildeford-fetch.js → src/backend/platforms/wildeford-fetch.ts
View File

@ -1,10 +1,11 @@
/* Imports */
// import axios from "axios"
import { GoogleSpreadsheet } from "google-spreadsheet";
import { databaseUpsert } from "../database/database-wrapper";
import { applyIfSecretExists } from "../utils/getSecrets";
import { hash } from "../utils/hash";
import { calculateStars } from "../utils/stars";
import { GoogleSpreadsheet } from 'google-spreadsheet';
import { databaseUpsert } from '../database/database-wrapper';
import { applyIfSecretExists } from '../utils/getSecrets';
import { hash } from '../utils/hash';
import { calculateStars } from '../utils/stars';
/* Definitions */
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
@ -98,7 +99,7 @@ async function processPredictions(predictions) {
Date.parse(prediction["Prediction Date"] + "Z")
).toISOString(),
qualityindicators: {
stars: calculateStars("Peter Wildeford"),
stars: calculateStars("Peter Wildeford", null),
},
};
return result;

7
src/backend/utils/algolia.js → src/backend/utils/algolia.ts
View File

@ -1,6 +1,7 @@
import algoliasearch from "algoliasearch";
import { databaseReadWithReadCredentials } from "../database/database-wrapper";
import { mergeEverythingInner } from "../flow/mergeEverything";
import algoliasearch from 'algoliasearch';
import { databaseReadWithReadCredentials } from '../database/database-wrapper';
import { mergeEverythingInner } from '../flow/mergeEverything';
let cookie = process.env.ALGOLIA_MASTER_API_KEY;
const client = algoliasearch("96UD3NTQ7L", cookie);

5
src/backend/utils/evaluations/pullForecastsToCSVForRating.js → src/backend/utils/evaluations/pullForecastsToCSVForRating.ts
View File

@ -1,6 +1,7 @@
/* Imports */
import fs from "fs";
import { databaseReadWithReadCredentials } from "../database-wrapper";
import fs from 'fs';
import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
/* Definitions */

5
src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.js → src/backend/utils/evaluations/pullMetaculusForecastsToCSVForRating.ts
View File

@ -1,6 +1,7 @@
/* Imports */
import fs from "fs";
import { databaseReadWithReadCredentials } from "../database-wrapper";
import fs from 'fs';
import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
/* Definitions */

0
src/backend/utils/getSecrets.js → src/backend/utils/getSecrets.ts
View File

4
src/backend/utils/hash.js → src/backend/utils/hash.ts
View File

@ -1,4 +1,4 @@
import crypto from "crypto";
import crypto from 'crypto';
export const hash = (string) =>
export const hash = (string: string) =>
crypto.createHash("sha256").update(string).digest("hex").slice(0, 10);

5
src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.js → src/backend/utils/misc/process-forecasts-from-old-givewellopenphil.ts
View File

@ -1,11 +1,12 @@
/* Imports */
import fs from "fs";
import fs from 'fs';
/* Definitions */
let locationData = "../../input";
/* Body */
let rawdata = fs.readFileSync(
`${locationData}/givewellopenphil-questions.json`
`${locationData}/givewellopenphil-questions.json`,
{ encoding: "utf-8" }
);
let data = JSON.parse(rawdata);

5
src/backend/utils/misc/process-forecasts-from-old-xrisk.js → src/backend/utils/misc/process-forecasts-from-old-xrisk.ts
View File

@ -1,12 +1,13 @@
/* Imports */
import fs from "fs";
import fs from 'fs';
/* Definitions */
let locationData = "../../data/";
/* Body */
let rawdata = fs.readFileSync(
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json"
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json",
{ encoding: "utf-8" }
);
let data = JSON.parse(rawdata);

5
src/backend/utils/misc/process-forecasts-from-xrisk.js → src/backend/utils/misc/process-forecasts-from-xrisk.ts
View File

@ -1,12 +1,13 @@
/* Imports */
import fs from "fs";
import fs from 'fs';
/* Definitions */
let locationData = "../../data/";
/* Body */
let rawdata = fs.readFileSync(
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json"
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json",
{ encoding: "utf-8" }
);
let data = JSON.parse(rawdata);

5
src/backend/utils/misc/process-forecasts-into-elicit.js → src/backend/utils/misc/process-forecasts-into-elicit.ts
View File

@ -1,6 +1,7 @@
/* Imports */
import fs from "fs";
import { databaseReadWithReadCredentials } from "../database-wrapper";
import fs from 'fs';
import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
/* Definitions */
let locationData = "./data/";

8
src/backend/utils/misc/process-forecasts-template.js → src/backend/utils/misc/process-forecasts-template.ts
View File

@ -1,11 +1,13 @@
/* Imports */
import fs from "fs";
import fs from 'fs';
/* Definitions */
let locationData = "./data/";
/* Body */
let rawdata = fs.readFileSync("../data/merged-questions.json");
let rawdata = fs.readFileSync("../data/merged-questions.json", {
encoding: "utf-8",
});
let data = JSON.parse(rawdata);
let results = [];
@ -13,5 +15,5 @@ for (let datum of data) {
// do something
}
let string = JSON.stringify(result, null, 2);
let string = JSON.stringify(results, null, 2);
fs.writeFileSync("../data/output.txt", string);

3
src/backend/utils/misc/test.js → src/backend/utils/misc/test.ts
View File

@ -1,5 +1,4 @@
import fs from "fs";
import axios from "axios";
import axios from 'axios';
let elicitEndpoint =
"https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";

0
src/backend/utils/roughSize.js → src/backend/utils/roughSize.ts
View File

0
src/backend/utils/stars.js → src/backend/utils/stars.ts
View File

0
src/backend/utils/toMarkdown.js → src/backend/utils/toMarkdown.ts
View File

6
src/pages/api/platform/[platform].js
View File

@ -1,6 +0,0 @@
// see: https://nextjs.org/docs/api-routes/dynamic-api-routes
export default function handler(req, res) {
const { platform } = req.query;
res.end(`Platform: ${platform}`);
}

0
src/pages/api/platform/index.js
View File

6
src/pages/api/question/[question].js
View File

@ -1,6 +0,0 @@
// see: https://nextjs.org/docs/api-routes/dynamic-api-routes
export default function handler(req, res) {
const { question } = req.query;
res.end(`Platform: ${question}`);
}

0
src/pages/api/question/index.js
View File

10
src/pages/secretDashboard.js → src/pages/secretDashboard.tsx
View File

@ -1,11 +1,11 @@
/* Imports */
// React
import { useRouter } from "next/router"; // https://nextjs.org/docs/api-reference/next/router
import React, { useState } from "react";
// Data
import displayForecasts from "../web/display/displayForecasts";
import { getDashboardForecastsByDashboardId } from "../web/worker/getDashboardForecasts";
import { useRouter } from 'next/router'; // https://nextjs.org/docs/api-reference/next/router
import { useState } from 'react';
import displayForecasts from '../web/display/displayForecasts';
import { getDashboardForecastsByDashboardId } from '../web/worker/getDashboardForecasts';
/* get Props */

90
src/web/worker/getForecasts.js
View File

@ -1,90 +0,0 @@
/*
import {MongoClient} from 'mongodb';
// import pkg from 'mongodb';
// const {MongoClient} = pkg;
async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
const url = "mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database.
const client = new MongoClient(url, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
let documentContents
try {
await client.connect();
// console.log(`Connected correctly to server to read ${documentName}`);
const db = client.db(databaseName);
// Use the collection "data"
const collection = db.collection(collectionName);
// Search options
const query = { "name": documentName };
const options = {
// sort matched documents in descending order by rating
sort: { rating: -1 },
};
// Insert a single document, wait for promise so we can read it back
// const p = await collection.insertOne(metaforecastDocument);
const document = await collection.findOne(query, options);
documentContents = document.contentsArray
} catch (err) {
console.log(err.stack);
}
finally {
await client.close();
}
// console.log(documentContents.slice(0,10));
return documentContents
}
export async function getForecasts() {
const mongodbDocument = await mongoRead("metaforecasts")
// console.log(mongodbDocument)
return mongodbDocument
}
// getForecasts()
//---------------------------------------//
//- Previous methods: Graphql -//
import { GraphQLClient } from "graphql-request";
// import { request } from 'graphql-request'
const graphcms = new GraphQLClient(
"https://api.baseql.com/airtable/graphql/apptDvDsHzSDEDARC"
);
export async function getForecasts0() {
return await graphcms.request(
`
query {
metaforecasts{
id
title
url
platform
binaryQuestion
percentage
forecasts
description
stars
}
}`
);
}
//- Previous methods: Github JSON -//
import axios from "axios"
export async function getForecasts1() {
const { data } = await axios.get(`https://raw.githubusercontent.com/QURIresearch/metaforecast/master/data/metaforecasts.json?ver=${Math.random().toFixed(10)}`) // this is, for now, a hack
//console.log(data)
return data
}
*/

8
src/web/worker/searchGuesstimate.js → src/web/worker/searchGuesstimate.ts
View File

@ -1,5 +1,5 @@
/* Imports */
import axios from "axios";
import axios from 'axios';
/* Definitions */
let urlEndPoint =
@ -10,15 +10,15 @@ let urlEndPoint =
export default function searchGuesstimate(query) {
let response = axios({
url: urlEndPoint,
credentials: "omit",
// credentials: "omit",
headers: {
// "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0",
Accept: "application/json",
"Accept-Language": "en-US,en;q=0.5",
"content-type": "application/x-www-form-urlencoded",
},
referrer:
"https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
// referrer:
// "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
data: `{\"params\":\"query=${query.replace(
/ /g,
"%20"

4
src/web/worker/searchWithAlgolia.js → src/web/worker/searchWithAlgolia.ts
View File

@ -1,4 +1,4 @@
import algoliasearch from "algoliasearch";
import algoliasearch from 'algoliasearch';
const client = algoliasearch("96UD3NTQ7L", "618dbd0092971388cfd43aac1ae5f1f5"); // Only search.
const index = client.initIndex("metaforecast");
@ -89,7 +89,7 @@ export default async function searchWithAlgolia({
//facetFilters: buildFacetFilter({filterByPlatforms}),
getRankingInfo: true,
});
let results = response.hits;
let results: any[] = response.hits;
console.log(
"searchWithAlgolia.js/searchWithAlgolia/queryString",
queryString

6
src/web/worker/uploadToImgur.js → src/web/worker/uploadToImgur.ts
View File

@ -1,8 +1,8 @@
// import fetch from "fetch"
import axios from "axios";
import axios, { AxiosRequestConfig } from 'axios';
export async function uploadToImgur(dataURL, handleGettingImgurlImage) {
let request = {
let request: AxiosRequestConfig = {
method: "post",
url: "https://api.imgur.com/3/image",
headers: {
@ -12,7 +12,7 @@ export async function uploadToImgur(dataURL, handleGettingImgurlImage) {
type: "base64",
image: dataURL.split(",")[1],
},
redirect: "follow",
// redirect: "follow",
};
let url;
try {

2
tsconfig.json
View File

@ -1,6 +1,5 @@
{
"compilerOptions": {
"allowJs": true,
"target": "es6",
"module": "CommonJS",
"outDir": "out",
@ -23,7 +22,6 @@
"include": [
"next-env.d.ts",
"src/**/*.ts",
"src/**/*.js",
"src/**/*.tsx"
],
"exclude": [