feat: convert all js to ts
allowJs is set to false in tsconfig.json now; please write all future
code with typescript.
cleanups:
- removed platforms/deprecated
- removed flow/history/old
- see https://github.com/QURIresearch/metaforecast/issues/22
- commented some invalid axios options
- minor fixes with mismatching function arguments
- commented invalid mongo call in databaseReadWithReadCredentials
- {encoding: 'utf-8'} in readFileSync calls
This commit is contained in:
Vyacheslav Matyukhin
parent
41b314b84a
commit
164f996f41
|
|
@ -142,11 +142,11 @@ export async function databaseReadWithReadCredentials({ group }) {
|
|||
break;
|
||||
default:
|
||||
mongoDocName = `${group}-questions`;
|
||||
responseMongo = mongoRemongoReadWithReadCredentialsad(
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
// responseMongo = mongoRemongoReadWithReadCredentialsad(
|
||||
// mongoDocName,
|
||||
// "metaforecastCollection",
|
||||
// "metaforecastDatabase"
|
||||
// );
|
||||
responsePg = await pgReadWithReadCredentials({
|
||||
schema: "latest",
|
||||
tableName: group,
|
||||
|
|
|
|||
|
|
@ -1,110 +0,0 @@
|
|||
import {
|
||||
databaseReadWithReadCredentials,
|
||||
databaseUpsert,
|
||||
} from "../../database/database-wrapper";
|
||||
let isEmptyArray = (arr) => arr.length == 0;
|
||||
|
||||
export async function addToHistory() {
|
||||
let currentDate = new Date();
|
||||
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_");
|
||||
|
||||
let currentJSONwithMetaculus = await databaseReadWithReadCredentials({
|
||||
group: "combined",
|
||||
});
|
||||
let currentJSON = currentJSONwithMetaculus.filter(
|
||||
(element) =>
|
||||
element.platform != "Metaculus" && element.platform != "Estimize"
|
||||
); // without Metaculus
|
||||
// console.log(currentJSON.slice(0,20))
|
||||
// console.log(currentJSON)
|
||||
let historyJSON = await databaseReadWithReadCredentials({ group: "history" });
|
||||
// console.log(historyJSON)
|
||||
|
||||
let currentForecastsWithAHistory = currentJSON.filter(
|
||||
(element) =>
|
||||
!isEmptyArray(
|
||||
historyJSON.filter(
|
||||
(historyElement) =>
|
||||
historyElement.title == element.title &&
|
||||
historyElement.url == element.url
|
||||
)
|
||||
)
|
||||
);
|
||||
// console.log(currentForecastsWithAHistory)
|
||||
|
||||
let currentForecastsWithoutAHistory = currentJSON.filter((element) =>
|
||||
isEmptyArray(
|
||||
historyJSON.filter(
|
||||
(historyElement) =>
|
||||
historyElement.title == element.title &&
|
||||
historyElement.url == element.url
|
||||
)
|
||||
)
|
||||
);
|
||||
// console.log(currentForecastsWithoutAHistory)
|
||||
|
||||
// Add both types of forecast
|
||||
let newHistoryJSON = [];
|
||||
for (let historyElement of historyJSON) {
|
||||
let correspondingNewElementArray = currentForecastsWithAHistory.filter(
|
||||
(element) =>
|
||||
historyElement.title == element.title &&
|
||||
historyElement.url == element.url
|
||||
);
|
||||
// console.log(correspondingNewElement)
|
||||
if (!isEmptyArray(correspondingNewElementArray)) {
|
||||
let correspondingNewElement = correspondingNewElementArray[0];
|
||||
let timeStampOfNewElement = correspondingNewElement.timestamp;
|
||||
let doesHistoryAlreadyContainElement = historyElement.history
|
||||
.map((element) => element.timestamp)
|
||||
.includes(timeStampOfNewElement);
|
||||
if (!doesHistoryAlreadyContainElement) {
|
||||
let historyWithNewElement = historyElement["history"].concat({
|
||||
timestamp: correspondingNewElement.timestamp,
|
||||
options: correspondingNewElement.options,
|
||||
qualityindicators: correspondingNewElement.qualityindicators,
|
||||
});
|
||||
let newHistoryElement = {
|
||||
...correspondingNewElement,
|
||||
history: historyWithNewElement,
|
||||
};
|
||||
// If some element (like the description) changes, we keep the new one.
|
||||
newHistoryJSON.push(newHistoryElement);
|
||||
} else {
|
||||
newHistoryJSON.push(historyElement);
|
||||
}
|
||||
} else {
|
||||
// console.log(historyElement)
|
||||
newHistoryJSON.push(historyElement);
|
||||
}
|
||||
}
|
||||
|
||||
for (let currentForecast of currentForecastsWithoutAHistory) {
|
||||
let newHistoryElement = {
|
||||
...currentForecast,
|
||||
history: [
|
||||
{
|
||||
timestamp: currentForecast.timestamp,
|
||||
options: currentForecast.options,
|
||||
qualityindicators: currentForecast.qualityindicators,
|
||||
},
|
||||
],
|
||||
};
|
||||
delete newHistoryElement.timestamp;
|
||||
delete newHistoryElement.options;
|
||||
delete newHistoryElement.qualityindicators;
|
||||
newHistoryJSON.push(newHistoryElement);
|
||||
}
|
||||
|
||||
await databaseUpsert({ contents: newHistoryJSON, group: "history" });
|
||||
|
||||
// console.log(newHistoryJSON.slice(0,5))
|
||||
// writeFileSync("metaforecast_history.json", JSON.stringify(newHistoryJSON, null, 2))
|
||||
// writefile(JSON.stringify(newHistoryJSON, null, 2), "metaforecasts_history", "", ".json")
|
||||
//console.log(newHistoryJSON)
|
||||
/*
|
||||
let forecastsAlreadyInHistory = currentJSON.filter(element => !isEmptyArray(historyJSON.filter(historyElement => historyElement.title == element.title && historyElement.url == element.url )))
|
||||
*/
|
||||
//console.log(new Date().toISOString())
|
||||
}
|
||||
// updateHistory()
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
import { databaseRead, databaseUpsert } from "../../database/database-wrapper";
|
||||
|
||||
export async function createHistoryForMonth() {
|
||||
let currentDate = new Date();
|
||||
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_");
|
||||
let metaforecasts = await databaseRead({ group: "combined" });
|
||||
let metaforecastsHistorySeed = metaforecasts
|
||||
.map((element) => {
|
||||
// let moreoriginsdata = element.author ? ({author: element.author}) : ({})
|
||||
return {
|
||||
title: element.title,
|
||||
url: element.url,
|
||||
platform: element.platform,
|
||||
moreoriginsdata: element.moreoriginsdata || {},
|
||||
description: element.description,
|
||||
history: [
|
||||
{
|
||||
timestamp: element.timestamp,
|
||||
options: element.options,
|
||||
qualityindicators: element.qualityindicators,
|
||||
},
|
||||
],
|
||||
extra: element.extra || {},
|
||||
};
|
||||
})
|
||||
.filter(
|
||||
(element) =>
|
||||
element.platform != "Metaculus" && element.platform != "Estimize"
|
||||
);
|
||||
//console.log(metaforecastsHistorySeed)
|
||||
await databaseUpsert({
|
||||
contents: metaforecastsHistorySeed,
|
||||
group: "history",
|
||||
});
|
||||
}
|
||||
////createInitialHistory()
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
import { databaseRead, databaseUpsert } from "../database-wrapper";
|
||||
|
||||
let createInitialHistory = async () => {
|
||||
let metaforecasts = await databaseRead({ group: "combined" });
|
||||
let metaforecastsHistorySeed = metaforecasts.map((element) => {
|
||||
// let moreoriginsdata = element.author ? ({author: element.author}) : ({})
|
||||
return {
|
||||
title: element.title,
|
||||
url: element.url,
|
||||
platform: element.platform,
|
||||
moreoriginsdata: element.moreoriginsdata || {},
|
||||
description: element.description,
|
||||
history: [
|
||||
{
|
||||
timestamp: element.timestamp,
|
||||
options: element.options,
|
||||
qualityindicators: element.qualityindicators,
|
||||
},
|
||||
],
|
||||
extra: element.extra || {},
|
||||
};
|
||||
});
|
||||
console.log(metaforecastsHistorySeed);
|
||||
await databaseUpsert({
|
||||
contents: metaforecastsHistorySeed,
|
||||
group: "history",
|
||||
});
|
||||
};
|
||||
createInitialHistory();
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
import { addToHistory } from "./addToHistory";
|
||||
import { createHistoryForMonth } from "./createHistoryForMonth";
|
||||
|
||||
export async function updateHistoryOld() {
|
||||
let currentDate = new Date();
|
||||
let dayOfMonth = currentDate.getDate();
|
||||
if (dayOfMonth == 1) {
|
||||
console.log(
|
||||
`Creating history for the month ${currentDate.toISOString().slice(0, 7)}`
|
||||
);
|
||||
await createHistoryForMonth();
|
||||
} else {
|
||||
console.log(`Updating history for ${currentDate.toISOString()}`);
|
||||
await addToHistory();
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateHistory() {
|
||||
let currentDate = new Date();
|
||||
let year = currentDate.toISOString().slice(0, 4);
|
||||
}
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from 'axios';
|
||||
|
||||
import { databaseUpsert } from '../utils/database-wrapper';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Definitions */
|
||||
let graphQLendpoint = "https://api.foretold.io/graphql";
|
||||
let ScottAlexanderPredictions = ["6eebf79b-4b6f-487b-a6a5-748d82524637"];
|
||||
|
||||
/* Support functions */
|
||||
async function fetchAllCommunityQuestions(communityId) {
|
||||
let response = await axios({
|
||||
url: graphQLendpoint,
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
data: JSON.stringify({
|
||||
query: `
|
||||
query {
|
||||
measurables(
|
||||
channelId: "${communityId}",
|
||||
states: OPEN,
|
||||
first: 500
|
||||
){
|
||||
total
|
||||
edges{
|
||||
node{
|
||||
id
|
||||
name
|
||||
valueType
|
||||
measurementCount
|
||||
previousAggregate{
|
||||
value{
|
||||
percentage
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`,
|
||||
}),
|
||||
})
|
||||
.then((res) => res.data)
|
||||
.then((res) => res.data.measurables.edges);
|
||||
//console.log(response)
|
||||
return response;
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
export async function astralcodexten() {
|
||||
let results = [];
|
||||
for (let community of ScottAlexanderPredictions) {
|
||||
let questions = await fetchAllCommunityQuestions(community);
|
||||
questions = questions.map((question) => question.node);
|
||||
questions = questions.filter((question) => question.previousAggregate); // Questions without any predictions
|
||||
questions.forEach((question) => {
|
||||
let options = [];
|
||||
if (question.valueType == "PERCENTAGE") {
|
||||
let probability = question.previousAggregate.value.percentage;
|
||||
options = [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability / 100,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - probability / 100,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
];
|
||||
}
|
||||
let result = {
|
||||
title: question.name.split(". ")[1],
|
||||
url: `https://www.foretold.io/c/${community}/m/${question.id}`,
|
||||
platform: "AstralCodexTen",
|
||||
description: "...by the end of 2021",
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number((question.measurementCount + 1) / 2),
|
||||
stars: calculateStars("AstralCodexTen", {}),
|
||||
},
|
||||
/*liquidity: liquidity.toFixed(2),
|
||||
tradevolume: tradevolume.toFixed(2),
|
||||
address: obj.address*/
|
||||
};
|
||||
// console.log(result)
|
||||
results.push(result);
|
||||
});
|
||||
}
|
||||
await databaseUpsert(results, "astralcodexten-questions");
|
||||
// console.log(results)
|
||||
console.log("Done");
|
||||
}
|
||||
// astralcodexten()
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import Papa from "papaparse";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
|
||||
/* Definitions */
|
||||
let coupCastEndpoint =
|
||||
"https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv";
|
||||
var datenow = new Date();
|
||||
var currentmonth = datenow.getMonth() + 1;
|
||||
dd;
|
||||
/* Support functions */
|
||||
let unique = (arr) => [...new Set(arr)];
|
||||
let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
let sanitizeCountryName = (country_name) => {
|
||||
let sanitized_name;
|
||||
switch (country_name) {
|
||||
case "Cen African Rep":
|
||||
sanitized_name = "Central African Republic";
|
||||
break;
|
||||
case "Congo-Brz":
|
||||
sanitized_name = "Republic of the Congo";
|
||||
break;
|
||||
case "Congo/Zaire":
|
||||
sanitized_name = "Democratic Republic of the Congo";
|
||||
break;
|
||||
case "Czech Rep":
|
||||
sanitized_name = "Czech Republic";
|
||||
break;
|
||||
case "Dominican Rep":
|
||||
sanitized_name = "Dominican Republic";
|
||||
break;
|
||||
case "Korea North":
|
||||
sanitized_name = "North Korea";
|
||||
break;
|
||||
case "Korea South":
|
||||
sanitized_name = "South Korea";
|
||||
break;
|
||||
case "UKG":
|
||||
sanitized_name = "UK";
|
||||
break;
|
||||
default:
|
||||
sanitized_name = country_name;
|
||||
}
|
||||
return sanitized_name;
|
||||
};
|
||||
|
||||
async function processArray(countryArray) {
|
||||
let results = [];
|
||||
for (let country of countryArray) {
|
||||
let url = `https://www.oneearthfuture.org/activities/coup-cast`;
|
||||
|
||||
// We don't really want the prediction for all months; one is enough
|
||||
// console.log(country.month)
|
||||
if (Number(country.month) == currentmonth) {
|
||||
// Monthly
|
||||
country.country_name = sanitizeCountryName(country.country_name);
|
||||
let processedPrediction1 = {
|
||||
title: `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
|
||||
url: url,
|
||||
platform: "CoupCast",
|
||||
description: `The current leader of ${country.country_name} is ${
|
||||
country.leader_name
|
||||
}, who has been in power for ${Number(country.leader_years).toFixed(
|
||||
1
|
||||
)} years. ${
|
||||
country.country_name
|
||||
} has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${
|
||||
country.regime_years
|
||||
} years.`,
|
||||
options: [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: country.month_risk,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - country.month_risk,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
],
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Coupcast", {}),
|
||||
},
|
||||
extra: {
|
||||
country_name: country.country_name,
|
||||
regime_type: country.regime_type,
|
||||
month: country.month,
|
||||
year: country.year,
|
||||
leader_name: country.leader_name,
|
||||
month_risk: country.month_risk,
|
||||
annual_risk: country.annual_risk,
|
||||
risk_change_percent: country.risk_change_percent,
|
||||
regime_years: country.regime_years,
|
||||
leader_years: country.leader_years,
|
||||
country_code: country.country_code,
|
||||
country_abb: country.country_abb,
|
||||
},
|
||||
};
|
||||
|
||||
// Yearly
|
||||
let processedPrediction2 = {
|
||||
title: `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
|
||||
url: url,
|
||||
platform: "CoupCast",
|
||||
description: `The current leader of ${country.country_name} is ${
|
||||
country.leader_name
|
||||
}, who has been in power for ${Number(country.leader_years).toFixed(
|
||||
1
|
||||
)} years. ${
|
||||
country.country_name
|
||||
} has a ${country.regime_type.toLowerCase()} regime type which has lasted for ${
|
||||
country.regime_years
|
||||
} years`,
|
||||
options: [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: country.annual_risk,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - country.annual_risk,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
],
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("CoupCast", {}),
|
||||
},
|
||||
extra: {
|
||||
country_name: country.country_name,
|
||||
regime_type: country.regime_type,
|
||||
month: country.month,
|
||||
year: country.year,
|
||||
leader_name: country.leader_name,
|
||||
month_risk: country.month_risk,
|
||||
annual_risk: country.annual_risk,
|
||||
risk_change_percent: country.risk_change_percent,
|
||||
regime_years: country.regime_years,
|
||||
leader_years: country.leader_years,
|
||||
country_code: country.country_code,
|
||||
country_abb: country.country_abb,
|
||||
},
|
||||
};
|
||||
|
||||
// results.push(processedPrediction1)
|
||||
// Not pushing monthly
|
||||
results.push(processedPrediction2);
|
||||
}
|
||||
}
|
||||
await databaseUpsert(results, "coupcast-questions");
|
||||
// console.log(results)
|
||||
console.log("Done");
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
export async function coupcast() {
|
||||
let csvContent = await axios
|
||||
.get(coupCastEndpoint)
|
||||
.then((query) => query.data);
|
||||
await Papa.parse(csvContent, {
|
||||
header: true,
|
||||
complete: async (results) => {
|
||||
console.log("Downloaded", results.data.length, "records.");
|
||||
/* console.log(
|
||||
JSON.stringify(
|
||||
unique(results.data.map(country => country.country_name)),
|
||||
null,
|
||||
4
|
||||
)
|
||||
)*/
|
||||
// console.log(results.data)
|
||||
await processArray(results.data);
|
||||
},
|
||||
});
|
||||
await sleep(1000); // needed to wait for Papaparse's callback to be executed.
|
||||
}
|
||||
// coupcast()
|
||||
|
|
@ -1,280 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { Tabletojson } from "tabletojson";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { applyIfSecretExists } from "../utils/getSecrets";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
|
||||
/* Definitions */
|
||||
let htmlEndPoint = "https://www.cset-foretell.com/questions?page=";
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
};
|
||||
const DEBUG_MODE = "on"; // "off"
|
||||
const SLEEP_TIME_RANDOM = 100; //5000 // miliseconds
|
||||
const SLEEP_TIME_EXTRA = 0; //1000
|
||||
/* Support functions */
|
||||
|
||||
async function fetchPage(page, cookie) {
|
||||
console.log(`Page #${page}`);
|
||||
if (page == 1) {
|
||||
cookie = cookie.split(";")[0]; // Interesting that it otherwise doesn't work :(
|
||||
}
|
||||
let urlEndpoint = htmlEndPoint + page;
|
||||
console.log(urlEndpoint);
|
||||
let response = await axios({
|
||||
url: urlEndpoint,
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "text/html",
|
||||
Cookie: cookie,
|
||||
},
|
||||
}).then((res) => res.data);
|
||||
// console.log(response)
|
||||
return response;
|
||||
}
|
||||
|
||||
async function fetchStats(questionUrl, cookie) {
|
||||
let response = await axios({
|
||||
url: questionUrl + "/stats",
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "text/html",
|
||||
Cookie: cookie,
|
||||
Referer: questionUrl,
|
||||
},
|
||||
}).then((res) => res.data);
|
||||
|
||||
if (response.includes("Sign up or sign in to forecast")) {
|
||||
throw Error("Not logged in");
|
||||
}
|
||||
|
||||
// Is binary?
|
||||
let isbinary = response.includes("binary?":true");
|
||||
// console.log(`is binary? ${isbinary}`)
|
||||
let options = [];
|
||||
if (isbinary) {
|
||||
// Crowd percentage
|
||||
let htmlElements = response.split("\n");
|
||||
// DEBUG_MODE == "on" ? htmlLines.forEach(line => console.log(line)) : id()
|
||||
let h3Element = htmlElements.filter((str) => str.includes("<h3>"))[0];
|
||||
// DEBUG_MODE == "on" ? console.log(h5elements) : id()
|
||||
let crowdpercentage = h3Element.split(">")[1].split("<")[0];
|
||||
let probability = Number(crowdpercentage.replace("%", "")) / 100;
|
||||
options.push(
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: +(1 - probability).toFixed(2), // avoids floating point shenanigans
|
||||
type: "PROBABILITY",
|
||||
}
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
let optionsBody = response.split("tbody")[1]; // Previously [1], then previously [3] but they added a new table.
|
||||
// console.log(optionsBody)
|
||||
let optionsHtmlElement = "<table" + optionsBody + "table>";
|
||||
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
|
||||
let firstTable = tablesAsJson[0];
|
||||
options = firstTable.map((element) => ({
|
||||
name: element["0"],
|
||||
probability: Number(element["1"].replace("%", "")) / 100,
|
||||
type: "PROBABILITY",
|
||||
}));
|
||||
} catch (error) {
|
||||
let optionsBody = response.split("tbody")[3]; // Catch if the error is related to table position
|
||||
let optionsHtmlElement = "<table" + optionsBody + "table>";
|
||||
let tablesAsJson = Tabletojson.convert(optionsHtmlElement);
|
||||
let firstTable = tablesAsJson[0];
|
||||
if (firstTable) {
|
||||
options = firstTable.map((element) => ({
|
||||
name: element["0"],
|
||||
probability: Number(element["1"].replace("%", "")) / 100,
|
||||
type: "PROBABILITY",
|
||||
}));
|
||||
} else {
|
||||
// New type of question, tricky to parse the options
|
||||
// Just leave options = [] for now.
|
||||
// https://www.cset-foretell.com/blog/rolling-question-formats
|
||||
}
|
||||
}
|
||||
}
|
||||
// Description
|
||||
let descriptionraw = response.split(`<meta name="description" content="`)[1];
|
||||
let descriptionprocessed1 = descriptionraw.split(`">`)[0];
|
||||
let descriptionprocessed2 = descriptionprocessed1.replace(">", "");
|
||||
let descriptionprocessed3 = descriptionprocessed2.replace(
|
||||
"To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.",
|
||||
``
|
||||
);
|
||||
// console.log(descriptionprocessed3)
|
||||
let descriptionprocessed4 = descriptionprocessed3.replaceAll(
|
||||
"\r\n\r\n",
|
||||
"\n"
|
||||
);
|
||||
let descriptionprocessed5 = descriptionprocessed4.replaceAll("\n\n", "\n");
|
||||
let descriptionprocessed6 = descriptionprocessed5.replaceAll(""", `"`);
|
||||
let descriptionprocessed7 = descriptionprocessed6.replaceAll("'", "'");
|
||||
let descriptionprocessed8 = toMarkdown(descriptionprocessed7);
|
||||
let description = descriptionprocessed8;
|
||||
|
||||
// Number of forecasts
|
||||
//console.log(response)
|
||||
//console.log(response.split("prediction_sets_count":")[1])
|
||||
let numforecasts = response
|
||||
.split("prediction_sets_count":")[1]
|
||||
.split(",")[0];
|
||||
// console.log(numforecasts)
|
||||
|
||||
// Number of predictors
|
||||
let numforecasters = response
|
||||
.split("predictors_count":")[1]
|
||||
.split(",")[0];
|
||||
// console.log(numpredictors)
|
||||
|
||||
let result = {
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number(numforecasts),
|
||||
numforecasters: Number(numforecasters),
|
||||
stars: calculateStars("CSET-foretell", { numforecasts }),
|
||||
},
|
||||
};
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function isSignedIn(html) {
|
||||
let isSignedInBool = !(
|
||||
html.includes("You need to sign in or sign up before continuing") ||
|
||||
html.includes("Sign up")
|
||||
);
|
||||
if (!isSignedInBool) {
|
||||
console.log("Error: Not signed in.");
|
||||
}
|
||||
console.log(`Signed in? ${isSignedInBool}`);
|
||||
return isSignedInBool;
|
||||
}
|
||||
|
||||
function isEnd(html) {
|
||||
let isEndBool = html.includes("No questions match your filter");
|
||||
if (isEndBool) {
|
||||
//console.log(html)
|
||||
}
|
||||
console.log(`IsEnd? ${isEndBool}`);
|
||||
return isEndBool;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
async function csetforetell_inner(cookie) {
|
||||
let i = 1;
|
||||
let response = await fetchPage(i, cookie);
|
||||
let results = [];
|
||||
let init = Date.now();
|
||||
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
|
||||
while (!isEnd(response) && isSignedIn(response)) {
|
||||
let htmlLines = response.split("\n");
|
||||
// let h4elements = htmlLines.filter(str => str.includes("<h5> <a href=") || str.includes("<h4> <a href="))
|
||||
let questionHrefs = htmlLines.filter((str) =>
|
||||
str.includes("https://www.cset-foretell.com/questions/")
|
||||
);
|
||||
// console.log(questionHrefs)
|
||||
|
||||
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
|
||||
//console.log(response)
|
||||
console.log("questionHrefs: ");
|
||||
console.log(questionHrefs);
|
||||
}
|
||||
|
||||
//console.log("")
|
||||
//console.log("")
|
||||
//console.log(h4elements)
|
||||
|
||||
for (let questionHref of questionHrefs) {
|
||||
//console.log(h4element)
|
||||
|
||||
let elementSplit = questionHref.split('"><span>');
|
||||
let url = elementSplit[0].split('<a href="')[1];
|
||||
let title = elementSplit[1]
|
||||
.replace("</h4>", "")
|
||||
.replace("</h5>", "")
|
||||
.replace("</span></a>", "");
|
||||
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
|
||||
|
||||
try {
|
||||
let moreinfo = await fetchStats(url, cookie);
|
||||
let question = {
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "CSET-foretell",
|
||||
...moreinfo,
|
||||
};
|
||||
if (
|
||||
i % 30 == 0 &&
|
||||
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
|
||||
) {
|
||||
console.log(
|
||||
`Page #${i}` &&
|
||||
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
|
||||
);
|
||||
console.log(question);
|
||||
}
|
||||
results.push(question);
|
||||
if (process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on") {
|
||||
console.log(url);
|
||||
console.log(question);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.log(
|
||||
`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
i++;
|
||||
//i=Number(i)+1
|
||||
|
||||
console.log(
|
||||
"Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers"
|
||||
);
|
||||
await sleep(Math.random() * SLEEP_TIME_RANDOM + SLEEP_TIME_EXTRA); // don't be as noticeable
|
||||
|
||||
try {
|
||||
response = await fetchPage(i, cookie);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.log(
|
||||
`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (results.length > 0) {
|
||||
await databaseUpsert(results, "csetforetell-questions");
|
||||
} else {
|
||||
console.log("Not updating results, as process was not signed in");
|
||||
}
|
||||
|
||||
let end = Date.now();
|
||||
let difference = end - init;
|
||||
console.log(
|
||||
`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`
|
||||
);
|
||||
}
|
||||
|
||||
export async function csetforetell() {
|
||||
let cookie = process.env.CSETFORETELL_COOKIE;
|
||||
await applyIfSecretExists(cookie, csetforetell_inner);
|
||||
}
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import Papa from "papaparse";
|
||||
import readline from "readline";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
|
||||
/* Definitions */
|
||||
let elicitEndpoint =
|
||||
"https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";
|
||||
|
||||
/* Support functions */
|
||||
let avg = (array) =>
|
||||
array.reduce((a, b) => Number(a) + Number(b)) / array.length;
|
||||
let unique = (arr) => [...new Set(arr)];
|
||||
let sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function processArray(arrayQuestions) {
|
||||
let questions = arrayQuestions.map((question) => question.questionTitle);
|
||||
let uniqueQuestions = unique(questions);
|
||||
let questionsObj = {};
|
||||
uniqueQuestions.forEach((questionTitle) => {
|
||||
questionsObj[questionTitle] = {
|
||||
title: questionTitle,
|
||||
forecasters: [],
|
||||
forecasts: [],
|
||||
};
|
||||
});
|
||||
arrayQuestions.forEach((question) => {
|
||||
// console.log(question.questionTitle)
|
||||
let questionTitle = question.questionTitle;
|
||||
let correspondingQuestion = questionsObj[questionTitle];
|
||||
let forecasters = correspondingQuestion.forecasters.concat(
|
||||
question.predictionCreator
|
||||
);
|
||||
let forecasts = correspondingQuestion.forecasts.concat(question.prediction);
|
||||
questionsObj[questionTitle] = {
|
||||
forecasters,
|
||||
forecasts,
|
||||
};
|
||||
});
|
||||
let onlyQuestionsWithMoreThan;
|
||||
|
||||
let results = [];
|
||||
for (let question in questionsObj) {
|
||||
let title = question;
|
||||
|
||||
let forecasters = questionsObj[question].forecasters;
|
||||
|
||||
let numforecasters = unique(forecasters).length;
|
||||
if (numforecasters >= 10) {
|
||||
let url = `https://elicit.org/binary?binaryQuestions.search=${title.replace(
|
||||
/ /g,
|
||||
"%20"
|
||||
)}&binaryQuestions.sortBy=popularity&limit=20&offset=0`;
|
||||
let forecasts = questionsObj[question].forecasts;
|
||||
|
||||
//console.log(forecasts)
|
||||
//console.log(avg(forecasts))
|
||||
let probability = avg(forecasts) / 100;
|
||||
let numforecasts = forecasts.length;
|
||||
let standardObj = {
|
||||
title: title,
|
||||
url: url,
|
||||
platform: "Elicit",
|
||||
options: [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
],
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
numforecasts: Number(numforecasts),
|
||||
numforecasters: Number(numforecasters),
|
||||
stars: calculateStars("Elicit", {}),
|
||||
},
|
||||
};
|
||||
results.push(standardObj);
|
||||
}
|
||||
}
|
||||
await databaseUpsert(results, "elicit-questions");
|
||||
|
||||
console.log("Done");
|
||||
}
|
||||
|
||||
async function awaitdownloadconfirmation(message, callback) {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
rl.question(message, (answer) => {
|
||||
//console.log("Received");
|
||||
rl.close();
|
||||
|
||||
callback();
|
||||
});
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
export async function elicit() {
|
||||
let csvContent = await axios.get(elicitEndpoint).then((query) => query.data);
|
||||
await Papa.parse(csvContent, {
|
||||
header: true,
|
||||
complete: async (results) => {
|
||||
console.log("Downloaded", results.data.length, "records.");
|
||||
//resolve(results.data);
|
||||
//console.log(results.data)
|
||||
await processArray(results.data);
|
||||
},
|
||||
});
|
||||
await sleep(5000); // needed to wait for Papaparse's callback to be executed.
|
||||
}
|
||||
//elicit()
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
import fs from "fs";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
|
||||
export async function estimize() {
|
||||
let data = fs.readFileSync(
|
||||
"./input/s-and-p-500-companies/companies.csv",
|
||||
"utf8"
|
||||
);
|
||||
|
||||
let splitData = data.split("\n");
|
||||
let results = [];
|
||||
for (let datum of splitData) {
|
||||
if (datum != "") {
|
||||
//console.log(datum)
|
||||
let datumSplit = datum.split(",");
|
||||
let companyStickerSymbol = datumSplit[0];
|
||||
let companyName = datumSplit[1];
|
||||
let standardObj = {
|
||||
title: `Estimates for ${companyName} (${companyStickerSymbol})`,
|
||||
url: `https://www.estimize.com/${companyStickerSymbol.toLowerCase()}`,
|
||||
platform: "Estimize",
|
||||
description: `A link to Estimize's forecasts for *${companyName}* (sticker symbol ${companyStickerSymbol}). Viewing them requires making a prediction, Wall Street estimates are provided for free`,
|
||||
options: [],
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Estimize", {}),
|
||||
},
|
||||
};
|
||||
results.push(standardObj);
|
||||
}
|
||||
}
|
||||
await databaseUpsert(results, "estimize-questions");
|
||||
}
|
||||
//estimize()
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import https from "https";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { applyIfSecretExists } from "../utils/getSecrets";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
|
||||
/* Definitions */
|
||||
let hypermindEnpoint1 = "https://predict.hypermind.com/dash/jsx.json";
|
||||
let hypermindEnpoint2 = "https://prod.hypermind.com/ngdp-jsx/jsx.json";
|
||||
const insecureHttpsAgent = new https.Agent({
|
||||
rejectUnauthorized: false, // (NOTE: this will disable client verification)
|
||||
});
|
||||
|
||||
/* Support Functions */
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
};
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/* Fetchers */
|
||||
async function fetchHypermindData1(slug) {
|
||||
let jsx = `jsx=%5B%5B%22dataMgr%22%2C%22getGQList%22%2C%7B%22listName%22%3A%20%22${slug}%22%2C%22format%22%3A%20%7B%22props%22%3A%20true%2C%22posts%22%3A%20true%2C%22cond%22%3A%20%7B%22props%22%3A%20true%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%2C%22otcm%22%3A%20%7B%22tradingHistory%22%3A%20true%2C%22props%22%3A%20true%7D%7D%7D%5D%5D`;
|
||||
// console.log(jsx)
|
||||
let response = await await axios(hypermindEnpoint1, {
|
||||
credentials: "omit",
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
|
||||
Accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Cache-Control": "max-age=0",
|
||||
},
|
||||
referrer: `https://predict.hypermind.com/dash/dash/dash.html?list=${slug}`,
|
||||
data: jsx,
|
||||
method: "POST",
|
||||
mode: "cors",
|
||||
httpsAgent: insecureHttpsAgent,
|
||||
}).then((response) => response.data[0].questions);
|
||||
//console.log(response)
|
||||
return response;
|
||||
}
|
||||
|
||||
async function fetchHypermindDataShowcases(slug, cookie) {
|
||||
let response = await axios(hypermindEnpoint2, {
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"User-Agent": "",
|
||||
Accept: "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Content-Type": "application/json; charset=UTF-8",
|
||||
//"Cookie": cookie
|
||||
},
|
||||
referrer:
|
||||
"https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
|
||||
data: `[["showcase","getShowcase",{"showcase":"${slug}","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":false}}]]`,
|
||||
method: "POST",
|
||||
mode: "cors",
|
||||
httpsAgent: insecureHttpsAgent,
|
||||
})
|
||||
.then((resp) => resp.data[0].items)
|
||||
.then((items) => items.filter((item) => item.type == "IFP"))
|
||||
.then((items) => items.map((item) => item.IFP));
|
||||
|
||||
// console.log(response)
|
||||
// response.forEach(item => delete item.crowdFcstHist)
|
||||
return response;
|
||||
}
|
||||
|
||||
/* Body */
|
||||
async function hypermind_inner(cookie) {
|
||||
// Hypermind panelists and competitors; dashboard type two: "showcase"
|
||||
// https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=SLUG
|
||||
// E.g., https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=AI2023
|
||||
let slugs2 = ["AI2030", "Covid19", "DOSES", "H5N8", "NGDP", "JSAI", "AI2023"]; // []
|
||||
let results2 = [];
|
||||
for (let slug of slugs2) {
|
||||
console.log(slug);
|
||||
await sleep(1000 + Math.random() * 1000);
|
||||
let response = await fetchHypermindDataShowcases(slug);
|
||||
let objs = response.map((result) => {
|
||||
let descriptionraw = result.props.details.split("<hr size=1>")[0];
|
||||
let descriptionprocessed1 = toMarkdown(descriptionraw);
|
||||
let descriptionprocessed2 = descriptionprocessed1.replaceAll(
|
||||
"![image] ()",
|
||||
""
|
||||
);
|
||||
let descriptionprocessed3 = descriptionprocessed2.replaceAll(
|
||||
" Forecasting Schedule ",
|
||||
""
|
||||
);
|
||||
let descriptionprocessed4 = descriptionprocessed3
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll(" ", " ");
|
||||
let descriptionprocessed5 = descriptionprocessed4.replaceAll(
|
||||
"Context:",
|
||||
""
|
||||
);
|
||||
let description =
|
||||
descriptionprocessed5 || toMarkdown(result.props.details);
|
||||
return {
|
||||
title: result.props.title,
|
||||
url:
|
||||
"https://prod.hypermind.com/ngdp/fr/showcase2/showcase.html?sc=" +
|
||||
slug,
|
||||
platform: "Hypermind",
|
||||
description: description,
|
||||
options: [],
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Hypermind", {}),
|
||||
numforecasters: Number(result.fcsterCnt),
|
||||
},
|
||||
};
|
||||
});
|
||||
// console.log(objs)
|
||||
results2.push(...objs);
|
||||
}
|
||||
|
||||
// Prediction markets; dashboard type one.
|
||||
// https://predict.hypermind.com/dash/dash/dash.html?list=SLUG
|
||||
// e.g., https://predict.hypermind.com/dash/dash/dash.html?list=POL
|
||||
let slugs1 = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]; // []
|
||||
let results1 = [];
|
||||
|
||||
for (let slug of slugs1) {
|
||||
console.log(slug);
|
||||
await sleep(2000 + Math.random() * 2000);
|
||||
let result = await fetchHypermindData1(slug);
|
||||
let objs = result.map((res) => {
|
||||
let descriptionraw = res.props.details;
|
||||
let descriptionprocessed1 = descriptionraw.split("%%fr")[0];
|
||||
let descriptionprocessed2 = descriptionprocessed1.replaceAll(
|
||||
"<BR>",
|
||||
"\n"
|
||||
);
|
||||
let descriptionprocessed3 = descriptionprocessed2.replace("%%en:", "");
|
||||
let descriptionprocessed4 = descriptionprocessed3.replace(
|
||||
`Shares of the correct outcome will be worth 100<sup>ℍ</sup>, while the others will be worthless (0<sup>ℍ</sup>).<p>`,
|
||||
""
|
||||
);
|
||||
let descriptionprocessed5 = toMarkdown(descriptionprocessed4);
|
||||
let description = descriptionprocessed5
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll(" ", " ");
|
||||
//console.log(res.otcms)
|
||||
//let percentage = (res.otcms.length==2) ? Number(res.otcms[0].price).toFixed(0) +"%" : "none"
|
||||
let options = res.otcms.map((option) => ({
|
||||
name: option.props.title.split("%%fr")[0].replaceAll("%%en:", ""),
|
||||
probability: Number(option.price) / 100,
|
||||
type: "PROBABILITY",
|
||||
}));
|
||||
return {
|
||||
title: res.props.title.split("%%fr")[0].replace("%%en:", ""),
|
||||
url: "https://predict.hypermind.com/dash/dash/dash.html?list=" + slug,
|
||||
platform: "Hypermind",
|
||||
description: description,
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Hypermind", {}),
|
||||
// "numforecasters": res.fcsterCnt
|
||||
},
|
||||
};
|
||||
});
|
||||
// console.log(objs)
|
||||
results1.push(...objs);
|
||||
}
|
||||
|
||||
let resultsTotal = [...results1, ...results2];
|
||||
|
||||
let distinctTitles = [];
|
||||
let resultsTotalUnique = [];
|
||||
for (let result of resultsTotal) {
|
||||
if (!distinctTitles.includes(result["title"])) {
|
||||
resultsTotalUnique.push(result);
|
||||
distinctTitles.push(result["title"]);
|
||||
}
|
||||
}
|
||||
// console.log(resultsTotal)
|
||||
// console.log(resultsTotalUnique)
|
||||
console.log(resultsTotalUnique.length, "results");
|
||||
await databaseUpsert(resultsTotalUnique, "hypermind-questions");
|
||||
}
|
||||
//hypermind()
|
||||
|
||||
export async function hypermind() {
|
||||
let cookie = process.env.HYPERMINDCOOKIE;
|
||||
await applyIfSecretExists(cookie, hypermind_inner);
|
||||
}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
|
||||
/* Definitions */
|
||||
let endpointPolitics = `https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForClass/302,301,300?simpleFilter=event.siteChannels:contains:M&simpleFilter=event.eventSortCode:intersects:TNMT,TR01,TR02,TR03,TR04,TR05,TR06,TR07,TR08,TR09,TR10,TR11,TR12,TR13,TR14,TR15,TR16,TR17,TR18,TR19,TR20&simpleFilter=event.suspendAtTime:greaterThan:${new Date().toISOString()}.000Z&limitRecords=outcome:1&limitRecords=market:1&translationLang=en&responseFormat=json&prune=event&prune=market`;
|
||||
let enpointDrillDown = (id) =>
|
||||
`https://ss-aka-ori.ladbrokes.com/openbet-ssviewer/Drilldown/2.31/EventToOutcomeForEvent/${id}?&translationLang=en&responseFormat=json`;
|
||||
|
||||
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
|
||||
|
||||
/* Support functions */
|
||||
async function fetchUrl(url) {
|
||||
let response = await axios(url, {
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
|
||||
Accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Cache-Control": "max-age=0",
|
||||
},
|
||||
method: "GET",
|
||||
mode: "cors",
|
||||
}).then((response) => response.data);
|
||||
return response;
|
||||
}
|
||||
|
||||
let processResults = async (json) => {
|
||||
let results = [];
|
||||
let children = json.SSResponse.children;
|
||||
children.pop();
|
||||
let ids = children.map((child) => child.event.id);
|
||||
let markets = [];
|
||||
for (let id of ids) {
|
||||
let marketsContainer = await fetchUrl(enpointDrillDown(id));
|
||||
let marketsObj = marketsContainer.SSResponse.children[0].event;
|
||||
let newMarkets = marketsObj.children;
|
||||
newMarkets = newMarkets.map((market) => ({
|
||||
...market.market,
|
||||
parent: marketsObj.name,
|
||||
}));
|
||||
markets.push(...newMarkets);
|
||||
}
|
||||
let normalMarkets = markets.filter(
|
||||
(market) => !market.name.includes("Specials")
|
||||
);
|
||||
//console.log(normalMarkets)
|
||||
|
||||
for (let normalMarket of normalMarkets) {
|
||||
let title = normalMarket.parent + ": " + normalMarket.name;
|
||||
title = title.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
|
||||
let options = normalMarket.children.map((child) => {
|
||||
let name = child.outcome.name;
|
||||
|
||||
let priceData = child.outcome.children[0].price;
|
||||
let priceDecimal = Number(priceData.priceDec);
|
||||
let probability = 1 / priceDecimal;
|
||||
let option = {
|
||||
name: name,
|
||||
probability: probability,
|
||||
type: "PROBABILITY",
|
||||
};
|
||||
return option;
|
||||
});
|
||||
// normalize probabilities
|
||||
let totalValue = options
|
||||
.map((element) => Number(element.probability))
|
||||
.reduce((a, b) => a + b, 0);
|
||||
options = options.map((element) => ({
|
||||
...element,
|
||||
probability: Number(element.probability) / totalValue,
|
||||
}));
|
||||
|
||||
// Filter very unlikely probabilities: Not here, but on the front end
|
||||
// options = options.filter(element => element.probability > 0.02)
|
||||
|
||||
let obj = {
|
||||
title: title,
|
||||
url: "https://sports.ladbrokes.com/sport/politics/outrights",
|
||||
platform: "Ladbrokes",
|
||||
description: "",
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Ladbrokes", {}),
|
||||
},
|
||||
};
|
||||
results.push(obj);
|
||||
}
|
||||
|
||||
let specialMarkets = markets.filter((market) =>
|
||||
market.name.includes("Specials")
|
||||
);
|
||||
for (let specialMarket of specialMarkets) {
|
||||
//let title = specialMarket.parent + ": " + specialMarket.name
|
||||
//console.log(title)
|
||||
specialMarket.children.forEach((child) => {
|
||||
let name = specialMarket.parent.includes("Specials")
|
||||
? child.outcome.name
|
||||
: specialMarket.parent + ": " + child.outcome.name;
|
||||
name = name.replace("Boris Johnson Specials", "Boris Johnson"); // small kludge
|
||||
let priceData = child.outcome.children[0].price;
|
||||
let priceDecimal = Number(priceData.priceDec);
|
||||
let probability = 1 / priceDecimal;
|
||||
let obj = {
|
||||
title: name,
|
||||
url: "https://sports.ladbrokes.com/sport/politics/outrights",
|
||||
platform: "LadBrokes",
|
||||
options: [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
],
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Ladbrokes", {}),
|
||||
},
|
||||
};
|
||||
results.push(obj);
|
||||
});
|
||||
}
|
||||
return results;
|
||||
};
|
||||
|
||||
/* Body */
|
||||
export async function ladbrokes() {
|
||||
let response = await fetchUrl(endpointPolitics);
|
||||
let results = await processResults(response);
|
||||
await databaseUpsert(results, "ladbrokes-questions");
|
||||
console.log("Done");
|
||||
}
|
||||
//ladbrokes()
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { databaseUpsert } from "../../utils/database-wrapper";
|
||||
import { calculateStars } from "../../utils/stars";
|
||||
|
||||
/* Definitions */
|
||||
let graphQLendpoint = "https://api.thegraph.com/subgraphs/name/protofire/omen";
|
||||
// "https://gateway.thegraph.com/api/[api-key]/subgraphs/id/0x0503024fcc5e1bd834530e69d592dbb6e8c03968-0"
|
||||
// 'https://api.thegraph.com/subgraphs/name/protofire/omen'
|
||||
// https://github.com/protofire/omen-subgraph
|
||||
// https://thegraph.com/explorer/subgraph/protofire/omen
|
||||
|
||||
async function fetchAllContractData() {
|
||||
let daysSinceEra = Math.round(Date.now() / (1000 * 24 * 60 * 60)) - 50; // last 30 days
|
||||
let response = await axios({
|
||||
url: graphQLendpoint,
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
data: JSON.stringify({
|
||||
query: `
|
||||
{
|
||||
fixedProductMarketMakers(first: 1000,
|
||||
where: {
|
||||
lastActiveDay_gt: ${daysSinceEra}
|
||||
}
|
||||
){
|
||||
id
|
||||
lastActiveDay
|
||||
question{
|
||||
title
|
||||
}
|
||||
outcomeSlotCount
|
||||
outcomeTokenMarginalPrices
|
||||
usdVolume
|
||||
usdLiquidityMeasure
|
||||
resolutionTimestamp
|
||||
}
|
||||
}
|
||||
`,
|
||||
}),
|
||||
})
|
||||
.then((res) => res.data)
|
||||
.then((res) => res.data.fixedProductMarketMakers);
|
||||
console.log(response);
|
||||
return response;
|
||||
}
|
||||
|
||||
async function fetch_all() {
|
||||
let allData = await fetchAllContractData();
|
||||
let results = [];
|
||||
for (let data of allData) {
|
||||
if (
|
||||
(data.question != null) &
|
||||
(data.usdLiquidityMeasure != "0") &
|
||||
(data.resolutionTimestamp == null) &
|
||||
(data.question.title != "ssdds")
|
||||
) {
|
||||
// console.log(data)
|
||||
// console.log(data.usdLiquidityMeasure)
|
||||
let options = data.outcomeTokenMarginalPrices.map((price, slotnum) => {
|
||||
let name = `Option ${slotnum}`;
|
||||
if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 0)
|
||||
name = "Yes";
|
||||
if (data.outcomeTokenMarginalPrices.length == 2 && slotnum == 1)
|
||||
name = "No";
|
||||
return {
|
||||
name: name,
|
||||
probability: Number(price),
|
||||
type: "PROBABILITY",
|
||||
};
|
||||
});
|
||||
|
||||
let obj = {
|
||||
title: data.question.title,
|
||||
url: "https://omen.eth.link/#/" + data.id,
|
||||
platform: "Omen",
|
||||
description: "",
|
||||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Omen", {}),
|
||||
},
|
||||
};
|
||||
// console.log(obj)
|
||||
results.push(obj);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/* Body */
|
||||
export async function omen() {
|
||||
let results = await fetch_all();
|
||||
await databaseUpsert(results, "omen-questions");
|
||||
console.log("Done");
|
||||
}
|
||||
//omen()
|
||||
|
|
@ -1,147 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { databaseUpsert } from "../utils/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = "https://sports.williamhill.com/betting/en-gb/politics";
|
||||
|
||||
// <header class="header-dropdown header-dropdown--large -expanded" data-id="
|
||||
|
||||
/* Support functions */
|
||||
async function fetchUrl(url) {
|
||||
let response = await axios(url, {
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0",
|
||||
Accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Cache-Control": "max-age=0",
|
||||
},
|
||||
method: "GET",
|
||||
mode: "cors",
|
||||
}).then((response) => response.data);
|
||||
return response;
|
||||
}
|
||||
|
||||
let processResults = (html) => {
|
||||
let results = [];
|
||||
|
||||
let chunks = html.split(
|
||||
'<header class="header-dropdown header-dropdown--large -expanded" data-id="'
|
||||
);
|
||||
chunks.shift();
|
||||
|
||||
// Kamala Special
|
||||
let kamalaspecial = chunks[0];
|
||||
let kamalamarkets = kamalaspecial.split(
|
||||
'<div class="btmarket__selection"><p class="btmarket__name"><span>'
|
||||
);
|
||||
kamalamarkets.shift();
|
||||
for (let kamalamarket of kamalamarkets) {
|
||||
let title = kamalamarket.split("</span>")[0];
|
||||
let numerator = Number(kamalamarket.split('data-num="')[1].split('"')[0]);
|
||||
let denominator = Number(
|
||||
kamalamarket.split('data-denom="')[1].split('"')[0]
|
||||
);
|
||||
let probability = denominator / (numerator + denominator);
|
||||
let obj = {
|
||||
title: title,
|
||||
url: "https://sports.williamhill.com/betting/en-gb/politics",
|
||||
platform: "WilliamHill",
|
||||
description: "",
|
||||
timestamp: new Date().toISOString(),
|
||||
options: [
|
||||
{
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
{
|
||||
name: "No",
|
||||
probability: 1 - probability,
|
||||
type: "PROBABILITY",
|
||||
},
|
||||
],
|
||||
qualityindicators: {
|
||||
stars: calculateStars("WilliamHill", {}),
|
||||
},
|
||||
};
|
||||
results.push(obj);
|
||||
}
|
||||
chunks.shift();
|
||||
|
||||
// Deal with the other markets
|
||||
for (let chunk of chunks) {
|
||||
let title = chunk.split('"')[0];
|
||||
let title2 = chunk.split('<a title="')[1].split('"')[0];
|
||||
title = title.length > title2.length ? title : title2;
|
||||
|
||||
let options = [];
|
||||
let alternatives = chunk.split(
|
||||
'<div class="btmarket__selection"><p class="btmarket__name"><span>'
|
||||
);
|
||||
alternatives.shift();
|
||||
for (let alternative of alternatives) {
|
||||
let optionName = alternative.split("</span>")[0];
|
||||
let numerator = Number(alternative.split('data-num="')[1].split('"')[0]);
|
||||
let denominator = Number(
|
||||
alternative.split('data-denom="')[1].split('"')[0]
|
||||
);
|
||||
let option = {
|
||||
name: optionName,
|
||||
probability: denominator / (numerator + denominator),
|
||||
type: "PROBABILITY",
|
||||
};
|
||||
options.push(option);
|
||||
}
|
||||
|
||||
// normalize probabilities
|
||||
let totalValue = options
|
||||
.map((element) => Number(element.probability))
|
||||
.reduce((a, b) => a + b, 0);
|
||||
options = options.map((element) => ({
|
||||
...element,
|
||||
probability: Number(element.probability) / totalValue,
|
||||
}));
|
||||
// Filter very unlikely probabilities: Not here, but on the front end
|
||||
// options = options.filter(element => element.probability > 0.02)
|
||||
|
||||
let obj = {
|
||||
title: title,
|
||||
url: "https://sports.williamhill.com/betting/en-gb/politics",
|
||||
platform: "WilliamHill",
|
||||
options: options,
|
||||
qualityindicators: {
|
||||
stars: calculateStars("WilliamHill", {}),
|
||||
},
|
||||
};
|
||||
results = results.filter(
|
||||
(result) => result.title.length > 4 && result.title != "2024 or later"
|
||||
);
|
||||
// removes some predictions because hard to parse.
|
||||
results.push(obj);
|
||||
}
|
||||
|
||||
//console.log(results)
|
||||
return results;
|
||||
};
|
||||
|
||||
let processhtml2 = (html) => {
|
||||
html.split();
|
||||
};
|
||||
|
||||
/* Body */
|
||||
export async function williamhill() {
|
||||
let response = await fetchUrl(
|
||||
"https://sports.williamhill.com/betting/en-gb/politics"
|
||||
);
|
||||
let results = processResults(response);
|
||||
await databaseUpsert(results, "williamhill-questions");
|
||||
console.log(results.sort((a, b) => a.title > b.title));
|
||||
console.log("Done");
|
||||
}
|
||||
//williamhill()
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import axios from 'axios';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = "https://example.com/";
|
||||
|
|
@ -44,7 +45,9 @@ async function processPredictions(predictions) {
|
|||
options: options,
|
||||
timestamp: new Date().toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Example", { some: somex, factors: factors }),
|
||||
stars: calculateStars("Example", {
|
||||
/* some: somex, factors: factors */
|
||||
}),
|
||||
other: prediction.otherx,
|
||||
indicators: prediction.indicatorx,
|
||||
},
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import axios from 'axios';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Definitions */
|
||||
let unixtime = new Date().getTime();
|
||||
|
|
@ -9,8 +10,8 @@ let endpoint = `https://fantasyscotus.net/case/list/?filterscount=0&groupscount=
|
|||
|
||||
async function fetchData() {
|
||||
let response = await axios({
|
||||
method: "GET",
|
||||
url: endpoint,
|
||||
credentials: "omit",
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
|
||||
|
|
@ -19,11 +20,11 @@ async function fetchData() {
|
|||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
},
|
||||
referrer: "https://fantasyscotus.net/case/list/",
|
||||
method: "GET",
|
||||
mode: "cors",
|
||||
// referrer: "https://fantasyscotus.net/case/list/",
|
||||
// credentials: "omit",
|
||||
// mode: "cors",
|
||||
}).then((res) => res.data);
|
||||
//console.log(response)
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
|
|
@ -31,8 +32,8 @@ async function getPredictionsData(caseUrl) {
|
|||
let newCaseUrl = `https://fantasyscotus.net/user-predictions${caseUrl}?filterscount=0&groupscount=0&sortdatafield=username&sortorder=asc&pagenum=0&pagesize=20&recordstartindex=0&recordendindex=20&_=${unixtime}`;
|
||||
//console.log(newCaseUrl)
|
||||
let predictions = await axios({
|
||||
method: "GET",
|
||||
url: newCaseUrl,
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
|
||||
|
|
@ -41,9 +42,9 @@ async function getPredictionsData(caseUrl) {
|
|||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
},
|
||||
referrer: newCaseUrl,
|
||||
method: "GET",
|
||||
mode: "cors",
|
||||
// referrer: newCaseUrl,
|
||||
// credentials: "include",
|
||||
// mode: "cors",
|
||||
}).then((res) => res.data);
|
||||
|
||||
let predictionsAffirm = predictions.filter(
|
||||
|
|
@ -102,7 +103,6 @@ async function processData(data) {
|
|||
stars: calculateStars("FantasySCOTUS", {}),
|
||||
},
|
||||
};
|
||||
// console.log(eventObject)
|
||||
results.push(eventObject);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import fs from "fs";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import axios from 'axios';
|
||||
import fs from 'fs';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Support functions */
|
||||
async function fetchPage(url) {
|
||||
|
|
@ -66,7 +67,9 @@ async function main1() {
|
|||
// main1()
|
||||
|
||||
async function main2() {
|
||||
let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json");
|
||||
let rawdata = fs.readFileSync("./input/givewellopenphil-questions.json", {
|
||||
encoding: "utf-8",
|
||||
});
|
||||
let data = JSON.parse(rawdata);
|
||||
let dataWithDate = data.map((datum) => ({
|
||||
...datum,
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { Tabletojson } from "tabletojson";
|
||||
import tunnel from "tunnel";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { hash } from "../utils/hash";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import axios from 'axios';
|
||||
import { Tabletojson } from 'tabletojson';
|
||||
import tunnel from 'tunnel';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { hash } from '../utils/hash';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Definitions */
|
||||
let endpoint = "https://goodjudgment.io/superforecasts/";
|
||||
|
|
@ -48,8 +49,8 @@ export async function goodjudgment() {
|
|||
headers: {
|
||||
"User-Agent": "Chrome",
|
||||
},
|
||||
agent,
|
||||
port: 80,
|
||||
// agent,
|
||||
// port: 80,
|
||||
})
|
||||
.then((query) => query.data);
|
||||
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { Tabletojson } from "tabletojson";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { applyIfSecretExists } from "../utils/getSecrets";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import axios from 'axios';
|
||||
import { Tabletojson } from 'tabletojson';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { applyIfSecretExists } from '../utils/getSecrets';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
import toMarkdown from '../utils/toMarkdown';
|
||||
|
||||
/* Definitions */
|
||||
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
|
||||
|
|
@ -14,8 +15,8 @@ let annoyingPromptUrls = [
|
|||
"https://www.gjopen.com/questions/2246-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters-2022-thread",
|
||||
"https://www.gjopen.com/questions/2237-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen",
|
||||
];
|
||||
const DEBUG_MODE = "off"; // "on"
|
||||
const id = (x) => x;
|
||||
const DEBUG_MODE: "on" | "off" = "off"; // "on"
|
||||
const id = () => 0;
|
||||
|
||||
/* Support functions */
|
||||
|
||||
|
|
@ -121,7 +122,9 @@ async function fetchStats(questionUrl, cookie) {
|
|||
maxProbability,
|
||||
}),
|
||||
},
|
||||
};
|
||||
// this mismatches the code below, and needs to be fixed, but I'm doing typescript conversion and don't want to touch any logic for now
|
||||
} as any;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -1,17 +1,18 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { Tabletojson } from "tabletojson";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { applyIfSecretExists } from "../utils/getSecrets";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import toMarkdown from "../utils/toMarkdown";
|
||||
import axios from 'axios';
|
||||
import { Tabletojson } from 'tabletojson';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { applyIfSecretExists } from '../utils/getSecrets';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
import toMarkdown from '../utils/toMarkdown';
|
||||
|
||||
/* Definitions */
|
||||
let htmlEndPoint = "https://www.infer-pub.com/questions";
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) {
|
||||
return this.split(search).join(replace);
|
||||
};
|
||||
const DEBUG_MODE = "off"; // "off"
|
||||
const DEBUG_MODE: "on" | "off" = "off"; // "off"
|
||||
const SLEEP_TIME_RANDOM = 7000; // miliseconds
|
||||
const SLEEP_TIME_EXTRA = 2000;
|
||||
/* Support functions */
|
||||
|
|
@ -229,10 +230,7 @@ async function infer_inner(cookie) {
|
|||
i % 30 == 0 &&
|
||||
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
|
||||
) {
|
||||
console.log(
|
||||
`Page #${i}` &&
|
||||
!(process.env.DEBUG_MODE == "on" || DEBUG_MODE == "on")
|
||||
);
|
||||
console.log(`Page #${i}`);
|
||||
console.log(question);
|
||||
}
|
||||
results.push(question);
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import axios from 'axios';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Definitions */
|
||||
let htmlEndPointEntrance = "https://api.smarkets.com/v3/events/";
|
||||
let VERBOSE = false;
|
||||
let empty = (x) => x;
|
||||
let empty = () => 0;
|
||||
/* Support functions */
|
||||
|
||||
async function fetchEvents(url) {
|
||||
|
|
@ -82,7 +83,7 @@ export async function smarkets() {
|
|||
}));
|
||||
VERBOSE ? console.log("Markets fetched") : empty();
|
||||
VERBOSE ? console.log(event.id) : empty();
|
||||
VERBOSE ? console.log(market) : empty();
|
||||
VERBOSE ? console.log(eventMarkets) : empty();
|
||||
markets.push(...eventMarkets);
|
||||
//let lastPrices = await fetchPrices(market.id)
|
||||
}
|
||||
|
|
@ -102,29 +103,29 @@ export async function smarkets() {
|
|||
? console.log("Prices: ", prices["last_executed_prices"][market.id])
|
||||
: empty();
|
||||
|
||||
let options = {};
|
||||
let optionsObj = {};
|
||||
for (let contract of contracts["contracts"]) {
|
||||
options[contract.id] = { name: contract.name };
|
||||
optionsObj[contract.id] = { name: contract.name };
|
||||
}
|
||||
for (let price of prices["last_executed_prices"][market.id]) {
|
||||
options[price.contract_id] = {
|
||||
...options[price.contract_id],
|
||||
optionsObj[price.contract_id] = {
|
||||
...optionsObj[price.contract_id],
|
||||
probability: price.last_executed_price
|
||||
? Number(price.last_executed_price)
|
||||
: null,
|
||||
type: "PROBABILITY",
|
||||
};
|
||||
}
|
||||
options = Object.values(options);
|
||||
let options: any[] = Object.values(optionsObj);
|
||||
// monkey patch the case where there are only two options and only one has traded.
|
||||
if (
|
||||
options.length == 2 &&
|
||||
options.map((option) => option.probability).includes(null)
|
||||
) {
|
||||
let nonNullPrice =
|
||||
option[0].probability == null
|
||||
? option[1].probability
|
||||
: option[0].probability;
|
||||
options[0].probability == null
|
||||
? options[1].probability
|
||||
: options[0].probability;
|
||||
options = options.map((option) => {
|
||||
let probability = option.probability;
|
||||
return {
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
/* Imports */
|
||||
// import axios from "axios"
|
||||
import { GoogleSpreadsheet } from "google-spreadsheet";
|
||||
import { databaseUpsert } from "../database/database-wrapper";
|
||||
import { applyIfSecretExists } from "../utils/getSecrets";
|
||||
import { hash } from "../utils/hash";
|
||||
import { calculateStars } from "../utils/stars";
|
||||
import { GoogleSpreadsheet } from 'google-spreadsheet';
|
||||
|
||||
import { databaseUpsert } from '../database/database-wrapper';
|
||||
import { applyIfSecretExists } from '../utils/getSecrets';
|
||||
import { hash } from '../utils/hash';
|
||||
import { calculateStars } from '../utils/stars';
|
||||
|
||||
/* Definitions */
|
||||
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
|
||||
|
|
@ -98,7 +99,7 @@ async function processPredictions(predictions) {
|
|||
Date.parse(prediction["Prediction Date"] + "Z")
|
||||
).toISOString(),
|
||||
qualityindicators: {
|
||||
stars: calculateStars("Peter Wildeford"),
|
||||
stars: calculateStars("Peter Wildeford", null),
|
||||
},
|
||||
};
|
||||
return result;
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import algoliasearch from "algoliasearch";
|
||||
import { databaseReadWithReadCredentials } from "../database/database-wrapper";
|
||||
import { mergeEverythingInner } from "../flow/mergeEverything";
|
||||
import algoliasearch from 'algoliasearch';
|
||||
|
||||
import { databaseReadWithReadCredentials } from '../database/database-wrapper';
|
||||
import { mergeEverythingInner } from '../flow/mergeEverything';
|
||||
|
||||
let cookie = process.env.ALGOLIA_MASTER_API_KEY;
|
||||
const client = algoliasearch("96UD3NTQ7L", cookie);
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import { databaseReadWithReadCredentials } from "../database-wrapper";
|
||||
import fs from 'fs';
|
||||
|
||||
import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
|
||||
|
||||
/* Definitions */
|
||||
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import { databaseReadWithReadCredentials } from "../database-wrapper";
|
||||
import fs from 'fs';
|
||||
|
||||
import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
|
||||
|
||||
/* Definitions */
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import crypto from "crypto";
|
||||
import crypto from 'crypto';
|
||||
|
||||
export const hash = (string) =>
|
||||
export const hash = (string: string) =>
|
||||
crypto.createHash("sha256").update(string).digest("hex").slice(0, 10);
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import fs from 'fs';
|
||||
|
||||
/* Definitions */
|
||||
let locationData = "../../input";
|
||||
/* Body */
|
||||
let rawdata = fs.readFileSync(
|
||||
`${locationData}/givewellopenphil-questions.json`
|
||||
`${locationData}/givewellopenphil-questions.json`,
|
||||
{ encoding: "utf-8" }
|
||||
);
|
||||
let data = JSON.parse(rawdata);
|
||||
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import fs from 'fs';
|
||||
|
||||
/* Definitions */
|
||||
let locationData = "../../data/";
|
||||
|
||||
/* Body */
|
||||
let rawdata = fs.readFileSync(
|
||||
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json"
|
||||
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-current/data/xrisk-questions-raw.json",
|
||||
{ encoding: "utf-8" }
|
||||
);
|
||||
let data = JSON.parse(rawdata);
|
||||
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import fs from 'fs';
|
||||
|
||||
/* Definitions */
|
||||
let locationData = "../../data/";
|
||||
|
||||
/* Body */
|
||||
let rawdata = fs.readFileSync(
|
||||
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json"
|
||||
"/home/nuno/Documents/core/software/fresh/js/metaforecasts/metaforecasts-mongo/src/input/xrisk-questions.json",
|
||||
{ encoding: "utf-8" }
|
||||
);
|
||||
let data = JSON.parse(rawdata);
|
||||
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import { databaseReadWithReadCredentials } from "../database-wrapper";
|
||||
import fs from 'fs';
|
||||
|
||||
import { databaseReadWithReadCredentials } from '../../database/database-wrapper';
|
||||
|
||||
/* Definitions */
|
||||
let locationData = "./data/";
|
||||
|
|
@ -1,11 +1,13 @@
|
|||
/* Imports */
|
||||
import fs from "fs";
|
||||
import fs from 'fs';
|
||||
|
||||
/* Definitions */
|
||||
let locationData = "./data/";
|
||||
|
||||
/* Body */
|
||||
let rawdata = fs.readFileSync("../data/merged-questions.json");
|
||||
let rawdata = fs.readFileSync("../data/merged-questions.json", {
|
||||
encoding: "utf-8",
|
||||
});
|
||||
let data = JSON.parse(rawdata);
|
||||
|
||||
let results = [];
|
||||
|
|
@ -13,5 +15,5 @@ for (let datum of data) {
|
|||
// do something
|
||||
}
|
||||
|
||||
let string = JSON.stringify(result, null, 2);
|
||||
let string = JSON.stringify(results, null, 2);
|
||||
fs.writeFileSync("../data/output.txt", string);
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import axios from 'axios';
|
||||
|
||||
let elicitEndpoint =
|
||||
"https://elicit.org/api/v1/binary-questions/csv?binaryQuestions.resolved=false&binaryQuestions.search=&binaryQuestions.sortBy=popularity&predictors=community";
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
/* Imports */
|
||||
|
||||
// React
|
||||
import { useRouter } from "next/router"; // https://nextjs.org/docs/api-reference/next/router
|
||||
import React, { useState } from "react";
|
||||
// Data
|
||||
import displayForecasts from "../web/display/displayForecasts";
|
||||
import { getDashboardForecastsByDashboardId } from "../web/worker/getDashboardForecasts";
|
||||
import { useRouter } from 'next/router'; // https://nextjs.org/docs/api-reference/next/router
|
||||
import { useState } from 'react';
|
||||
|
||||
import displayForecasts from '../web/display/displayForecasts';
|
||||
import { getDashboardForecastsByDashboardId } from '../web/worker/getDashboardForecasts';
|
||||
|
||||
/* get Props */
|
||||
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
/*
|
||||
import {MongoClient} from 'mongodb';
|
||||
// import pkg from 'mongodb';
|
||||
// const {MongoClient} = pkg;
|
||||
|
||||
async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
|
||||
const url = "mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database.
|
||||
|
||||
const client = new MongoClient(url, {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true,
|
||||
});
|
||||
|
||||
let documentContents
|
||||
try {
|
||||
await client.connect();
|
||||
// console.log(`Connected correctly to server to read ${documentName}`);
|
||||
const db = client.db(databaseName);
|
||||
|
||||
// Use the collection "data"
|
||||
const collection = db.collection(collectionName);
|
||||
|
||||
// Search options
|
||||
const query = { "name": documentName };
|
||||
const options = {
|
||||
// sort matched documents in descending order by rating
|
||||
sort: { rating: -1 },
|
||||
};
|
||||
|
||||
// Insert a single document, wait for promise so we can read it back
|
||||
// const p = await collection.insertOne(metaforecastDocument);
|
||||
const document = await collection.findOne(query, options);
|
||||
documentContents = document.contentsArray
|
||||
} catch (err) {
|
||||
console.log(err.stack);
|
||||
}
|
||||
finally {
|
||||
await client.close();
|
||||
}
|
||||
// console.log(documentContents.slice(0,10));
|
||||
return documentContents
|
||||
}
|
||||
|
||||
export async function getForecasts() {
|
||||
const mongodbDocument = await mongoRead("metaforecasts")
|
||||
// console.log(mongodbDocument)
|
||||
return mongodbDocument
|
||||
}
|
||||
|
||||
// getForecasts()
|
||||
|
||||
//---------------------------------------//
|
||||
|
||||
//- Previous methods: Graphql -//
|
||||
import { GraphQLClient } from "graphql-request";
|
||||
// import { request } from 'graphql-request'
|
||||
|
||||
const graphcms = new GraphQLClient(
|
||||
"https://api.baseql.com/airtable/graphql/apptDvDsHzSDEDARC"
|
||||
);
|
||||
|
||||
export async function getForecasts0() {
|
||||
return await graphcms.request(
|
||||
`
|
||||
query {
|
||||
metaforecasts{
|
||||
id
|
||||
title
|
||||
url
|
||||
platform
|
||||
binaryQuestion
|
||||
percentage
|
||||
forecasts
|
||||
description
|
||||
stars
|
||||
}
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
//- Previous methods: Github JSON -//
|
||||
import axios from "axios"
|
||||
|
||||
export async function getForecasts1() {
|
||||
const { data } = await axios.get(`https://raw.githubusercontent.com/QURIresearch/metaforecast/master/data/metaforecasts.json?ver=${Math.random().toFixed(10)}`) // this is, for now, a hack
|
||||
//console.log(data)
|
||||
return data
|
||||
}
|
||||
|
||||
*/
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/* Imports */
|
||||
import axios from "axios";
|
||||
import axios from 'axios';
|
||||
|
||||
/* Definitions */
|
||||
let urlEndPoint =
|
||||
|
|
@ -10,15 +10,15 @@ let urlEndPoint =
|
|||
export default function searchGuesstimate(query) {
|
||||
let response = axios({
|
||||
url: urlEndPoint,
|
||||
credentials: "omit",
|
||||
// credentials: "omit",
|
||||
headers: {
|
||||
// "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0",
|
||||
Accept: "application/json",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"content-type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
referrer:
|
||||
"https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
|
||||
// referrer:
|
||||
// "https://m629r9ugsg-dsn.algolia.net/1/indexes/Space_production/query?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.32.1&x-algolia-application-id=M629R9UGSG&x-algolia-api-key=4e893740a2bd467a96c8bfcf95b2809c",
|
||||
data: `{\"params\":\"query=${query.replace(
|
||||
/ /g,
|
||||
"%20"
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import algoliasearch from "algoliasearch";
|
||||
import algoliasearch from 'algoliasearch';
|
||||
|
||||
const client = algoliasearch("96UD3NTQ7L", "618dbd0092971388cfd43aac1ae5f1f5"); // Only search.
|
||||
const index = client.initIndex("metaforecast");
|
||||
|
|
@ -89,7 +89,7 @@ export default async function searchWithAlgolia({
|
|||
//facetFilters: buildFacetFilter({filterByPlatforms}),
|
||||
getRankingInfo: true,
|
||||
});
|
||||
let results = response.hits;
|
||||
let results: any[] = response.hits;
|
||||
console.log(
|
||||
"searchWithAlgolia.js/searchWithAlgolia/queryString",
|
||||
queryString
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
// import fetch from "fetch"
|
||||
import axios from "axios";
|
||||
import axios, { AxiosRequestConfig } from 'axios';
|
||||
|
||||
export async function uploadToImgur(dataURL, handleGettingImgurlImage) {
|
||||
let request = {
|
||||
let request: AxiosRequestConfig = {
|
||||
method: "post",
|
||||
url: "https://api.imgur.com/3/image",
|
||||
headers: {
|
||||
|
|
@ -12,7 +12,7 @@ export async function uploadToImgur(dataURL, handleGettingImgurlImage) {
|
|||
type: "base64",
|
||||
image: dataURL.split(",")[1],
|
||||
},
|
||||
redirect: "follow",
|
||||
// redirect: "follow",
|
||||
};
|
||||
let url;
|
||||
try {
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"allowJs": true,
|
||||
"target": "es6",
|
||||
"module": "CommonJS",
|
||||
"outDir": "out",
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user