quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added somewhat more complicated cookie functionality

Browse Source
This commit is contained in:
NunoSempere 2021-04-10 20:18:22 +02:00
parent abc4f61432
commit cf795eb90e
7 changed files with 78 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/input/privatekeys_example.json
View File

@ -1,5 +1,7 @@
{
"csetforetellcookie": "_cset_prod_fof_session=xxx; landing_url=https://www.cset-foretell.com/; randomization_seed=xxx; remember_user_token=xxx; ",
"goodjudmentopencookie": "__cfduid=xxx; _gj_prod_flyover_forecasts_session=xxx; randomization_seed=xxx; referring_url=https://www.gjopen.com/; remember_user_token=xxx;",
"hypermindcookie": "lumAuth=xxxx:yyyyyyyyyyyyyyyyy"
"csetforetell": "_cset_prod_fof_session=xxx; landing_url=https://www.cset-foretell.com/; randomization_seed=xxx; remember_user_token=xxx; ",
"goodjudmentopen": "__cfduid=xxx; _gj_prod_flyover_forecasts_session=xxx; randomization_seed=xxx; referring_url=https://www.gjopen.com/; remember_user_token=xxx;",
"hypermind": "lumAuth=xxxx:yyyyyyyyyyyyyyyyy",
"mongodb": "mongodb+srv://<username>:<password>@<mongodbclusterurl>/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true",
"netlify": "https://api.netlify.com/build_hooks/<netlifysecretnumber>?trigger_title=<some_title>"
}

32
src/platforms/csetforetell-fetch.js
View File

@ -1,6 +1,6 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
import {Tabletojson} from "tabletojson"
import toMarkdown from "../utils/toMarkdown.js"
import {calculateStars} from "../utils/stars.js"
@ -12,24 +12,6 @@ String.prototype.replaceAll = function replaceAll(search, replace) { return this
/* Support functions */
function getcookie(){
return process.env.CSETFORETELL_COOKIE
/*
try {
let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
let cookie = JSON.parse(rawcookie).csetforetellcookie
if(cookie == undefined){
throw new Error('No cookie for CSET-foretell!');
}
return cookie
} catch(error) {
console.log("Error: No cookies for CSET-foretell on src/privatekeys.json! See the README.md")
process.exit()
}*/
}
async function fetchPage(page, cookie){
let response = await axios({
@ -131,8 +113,7 @@ function sleep(ms) {
/* Body */
export async function csetforetell(){
let cookie = getcookie()
async function csetforetell_inner(cookie){
let i=1
let response = await fetchPage(i, cookie)
let results = []
@ -183,10 +164,17 @@ export async function csetforetell(){
}
// let string = JSON.stringify(results,null, 2)
// fs.writeFileSync('./data/csetforetell-questions.json', string);
console.log(results)
await upsert(results, "csetforetell-questions")
let end = Date.now()
let difference = end-init
console.log(`Took ${difference/1000} seconds, or ${difference/(1000*60)} minutes.`)
}
export async function csetforetell(){
let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell")
await applyIfCookieExists(csetforetell_inner, cookie)
}

30
src/platforms/goodjudmentopen-fetch.js
View File

@ -1,6 +1,7 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
import { Tabletojson } from "tabletojson"
import { calculateStars } from "../utils/stars.js"
import toMarkdown from "../utils/toMarkdown.js"
@ -12,24 +13,6 @@ let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecastin
/* Support functions */
function getcookie() {
return process.env.GOODJUDGMENTOPENCOOKIE
/*
try {
let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
let cookie = JSON.parse(rawcookie).goodjudmentopencookie
if (cookie == undefined) {
throw new Error('No cookie for Good Judgment Open!');
}
return cookie
} catch (error) {
console.log("Error: No cookies for Good Judgment Open on src/privatekeys.json! See the README.md")
process.exit()
}
*/
}
async function fetchPage(page, cookie) {
let response = await axios({
url: htmlEndPoint + page,
@ -134,9 +117,7 @@ function sleep(ms) {
/* Body */
export async function goodjudgmentopen() {
let cookie = getcookie()
async function goodjudgmentopen_inner(cookie) {
let i = 1
let response = await fetchPage(i, cookie)
let results = []
@ -166,7 +147,7 @@ export async function goodjudgmentopen() {
"platform": "Good Judgment Open",
...moreinfo
})
if(j % 10 == 0){
if(j % 30 == 0){
console.log(`Page #${i}`)
console.log(question)
}
@ -198,3 +179,8 @@ export async function goodjudgmentopen() {
let difference = end - init
console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`)
}
export async function goodjudmentopen(){
let cookie = process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudgmentopen")
await applyIfCookieExists(goodjudgmentopen_inner, cookie)
}

43
src/platforms/hypermind-fetch.js
View File

@ -1,6 +1,7 @@
/* Imports */
import fs from 'fs'
import axios from "axios"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
import toMarkdown from "../utils/toMarkdown.js"
import { calculateStars } from "../utils/stars.js"
import { upsert } from "../utils/mongo-wrapper.js"
@ -14,23 +15,6 @@ function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function getcookie() {
return process.env.HYPERMINDCOOKIE
/*
try {
let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
let cookie = JSON.parse(rawcookie).hypermindcookie
if (cookie == undefined) {
throw new Error('No cookie for Hypermind!');
}
return cookie
} catch (error) {
console.log("Error: No cookies for Hypermind on src/privatekeys.json! See the README.md")
process.exit()
}
*/
}
async function fetchHypermindData1(slug) {
let response = await axios("https://predict.hypermind.com/dash/jsx.json", {
"credentials": "omit",
@ -48,7 +32,7 @@ async function fetchHypermindData1(slug) {
return response
}
async function fetchHypermindData2() {
async function fetchHypermindData2(cookie) {
let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
"credentials": "include",
"headers": {
@ -56,7 +40,7 @@ async function fetchHypermindData2() {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8",
"Cookie": getcookie()
"Cookie": cookie
},
"referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
"data": `[["showcase","queryIFPs",{"query":{"showcaseOnly":true},"fmt":{"stats":true,"crowdFcstHist":true}}]]`,
@ -66,7 +50,7 @@ async function fetchHypermindData2() {
return response
}
async function fetchHypermindData3() {
async function fetchHypermindData3(cookie) {
let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
"credentials": "include",
"headers": {
@ -74,7 +58,7 @@ async function fetchHypermindData3() {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8",
"Cookie": getcookie()
"Cookie": cookie
},
"referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
"data": `[["showcase","getShowcase",{"showcase":"Covid19","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`,
@ -88,7 +72,7 @@ async function fetchHypermindData3() {
return response
}
async function fetchHypermindData4() {
async function fetchHypermindData4(cookie) {
let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
"credentials": "include",
"headers": {
@ -96,7 +80,7 @@ async function fetchHypermindData4() {
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/json; charset=UTF-8",
"Cookie": getcookie()
"Cookie": cookie
},
"referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
"data": `[["showcase","getShowcase",{"showcase":"AI2023","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`,
@ -111,7 +95,7 @@ async function fetchHypermindData4() {
}
/* Body */
export async function hypermind() {
async function hypermind_inner(cookie) {
let slugs = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]
let results1 = []
@ -156,7 +140,7 @@ export async function hypermind() {
console.log("GDP")
await sleep(1000 + Math.random() * 1000)
let results2 = await fetchHypermindData2()
let results2 = await fetchHypermindData2(cookie)
let results2processed = results2.map(res => {
//console.log(res.props.details)
let descriptionraw = res.props.details.split("<hr size=1>")[0]
@ -179,7 +163,7 @@ export async function hypermind() {
console.log("COVID-19 OpenPhil")
await sleep(1000 + Math.random() * 1000)
let results3 = await fetchHypermindData3()
let results3 = await fetchHypermindData3(cookie)
// console.log(results3)
let results3processed = results3.map(res => {
let descriptionraw = res.props.details.split("<hr size=1>")[0]
@ -200,7 +184,7 @@ export async function hypermind() {
console.log("AI in 2023")
await sleep(1000 + Math.random() * 1000)
let results4 = await fetchHypermindData4()
let results4 = await fetchHypermindData4(cookie)
// console.log(results4)
let results4processed = results2.map(res => {
let description = res.props.details
@ -233,3 +217,8 @@ export async function hypermind() {
}
//hypermind()
export async function hypermind() {
let cookie = process.env.HYPERMINDCOOKIE || getCookie("hypermind")
await applyIfCookieExists(hypermind_inner, cookie)
}

25
src/utils/getCookies.js Normal file
View File

@ -0,0 +1,25 @@
import fs from 'fs'
export function getCookie(property){
let answer = 0
try {
let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
let cookie = JSON.parse(rawcookie)
if (cookie[property]){
answer = cookie[property]
console.log(`Got cookie: ${answer.slice(0,5)}...`)
}
} catch (error) {
console.log(error)
}
return answer
}
export async function applyIfCookieExists(cookie, fun){
if(cookie){
await fun(cookie)
}else if(!cookie){
console.log(`Cannot proceed with ${fun.name} because cookie does not exist`)
throw new Error(`No cookie for ${fun.name}`)
}
}

6
src/utils/mongo-wrapper.js
View File

@ -1,8 +1,10 @@
import pkg from 'mongodb';
const {MongoClient} = pkg;
import {getCookie} from "./getCookies.js"
export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
const url = process.env.MONGODB_URL;
const url = process.env.MONGODB_URL || getCookie("mongodb");
const client = new MongoClient(url);
try {
@ -40,7 +42,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
}
export async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
const url = process.env.MONGODB_URL;
const url = process.env.MONGODB_URL || getCookie("mongodb");
const client = new MongoClient(url, {
useNewUrlParser: true,

15
src/utils/rebuildNetliftySiteWithNewData.js
View File

@ -1,9 +1,14 @@
import axios from "axios"
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
export async function rebuildNetlifySiteWithNewData_inner(cookie){
let payload = ({});
let response = await axios.post(cookie, payload);
let data = response.data;
console.log(data);
}
export async function rebuildNetlifySiteWithNewData(){
let rebuildNetlifyHookUrl = process.env.REBUIDNETLIFYHOOKURL;
let payload = ({});
let response = await axios.post(rebuildNetlifyHookUrl, payload);
let data = response.data;
console.log(data);
let cookie = process.env.REBUIDNETLIFYHOOKURL || getCookie("netlify");
await applyIfCookieExists(rebuildNetlifySiteWithNewData_inner, cookie)
}