Added somewhat more complicated cookie functionality

src/input/privatekeys_example.json

@@ -1,5 +1,7 @@
 {
-  "csetforetellcookie": "_cset_prod_fof_session=xxx; landing_url=https://www.cset-foretell.com/;  randomization_seed=xxx; remember_user_token=xxx; ",
+  "csetforetell": "_cset_prod_fof_session=xxx; landing_url=https://www.cset-foretell.com/;  randomization_seed=xxx; remember_user_token=xxx; ",
-  "goodjudmentopencookie": "__cfduid=xxx; _gj_prod_flyover_forecasts_session=xxx; randomization_seed=xxx; referring_url=https://www.gjopen.com/; remember_user_token=xxx;",
+  "goodjudmentopen": "__cfduid=xxx; _gj_prod_flyover_forecasts_session=xxx; randomization_seed=xxx; referring_url=https://www.gjopen.com/; remember_user_token=xxx;",
-  "hypermindcookie": "lumAuth=xxxx:yyyyyyyyyyyyyyyyy"
+  "hypermind": "lumAuth=xxxx:yyyyyyyyyyyyyyyyy",
+  "mongodb": "mongodb+srv://<username>:<password>@<mongodbclusterurl>/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true",
+  "netlify": "https://api.netlify.com/build_hooks/<netlifysecretnumber>?trigger_title=<some_title>"
 }

src/platforms/csetforetell-fetch.js

@@ -1,6 +1,6 @@
 /* Imports */
-import fs from 'fs'
 import axios from "axios"
+import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
 import {Tabletojson} from "tabletojson"
 import toMarkdown from "../utils/toMarkdown.js"
 import {calculateStars} from "../utils/stars.js"
@@ -12,24 +12,6 @@ String.prototype.replaceAll = function replaceAll(search, replace) { return this
 
 /* Support functions */
 
-function getcookie(){
-  return process.env.CSETFORETELL_COOKIE
-  /*
-  try {
-    let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
-    let cookie = JSON.parse(rawcookie).csetforetellcookie
-    if(cookie == undefined){
-      throw new Error('No cookie for CSET-foretell!');
-    }
-    
-    return cookie
-  } catch(error) {
-    console.log("Error: No cookies for CSET-foretell on src/privatekeys.json! See the README.md")
-    process.exit()
-  }*/
-}
-
-
 async function fetchPage(page, cookie){
   
   let response  = await axios({
@@ -131,8 +113,7 @@ function sleep(ms) {
 
 /* Body */
 
-export async function csetforetell(){
+async function csetforetell_inner(cookie){
-  let cookie = getcookie()
   let i=1
   let response = await fetchPage(i, cookie)
   let results = []
@@ -183,10 +164,17 @@ export async function csetforetell(){
   }
   // let string = JSON.stringify(results,null,  2)
   // fs.writeFileSync('./data/csetforetell-questions.json', string);
+  console.log(results)
   await upsert(results, "csetforetell-questions")
-
+  
   
   let end = Date.now()
   let difference = end-init
   console.log(`Took ${difference/1000} seconds, or ${difference/(1000*60)} minutes.`)
 }
+
+
+export async function csetforetell(){
+  let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell") 
+  await applyIfCookieExists(csetforetell_inner, cookie)
+}

src/platforms/goodjudmentopen-fetch.js

@@ -1,6 +1,7 @@
 /* Imports */
 import fs from 'fs'
 import axios from "axios"
+import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
 import { Tabletojson } from "tabletojson"
 import { calculateStars } from "../utils/stars.js"
 import toMarkdown from "../utils/toMarkdown.js"
@@ -12,24 +13,6 @@ let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecastin
 
 /* Support functions */
 
-function getcookie() {
-  return process.env.GOODJUDGMENTOPENCOOKIE
-  /*
-  try {
-    let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
-    let cookie = JSON.parse(rawcookie).goodjudmentopencookie
-    if (cookie == undefined) {
-      throw new Error('No cookie for Good Judgment Open!');
-    }
-
-    return cookie
-  } catch (error) {
-    console.log("Error: No cookies for Good Judgment Open on src/privatekeys.json! See the README.md")
-    process.exit()
-  }
-  */
-}
-
 async function fetchPage(page, cookie) {
   let response = await axios({
     url: htmlEndPoint + page,
@@ -134,9 +117,7 @@ function sleep(ms) {
 
 /* Body */
 
-export async function goodjudgmentopen() {
+async function goodjudgmentopen_inner(cookie) {
-  let cookie = getcookie()
-
   let i = 1
   let response = await fetchPage(i, cookie)
   let results = []
@@ -166,7 +147,7 @@ export async function goodjudgmentopen() {
             "platform": "Good Judgment Open",
             ...moreinfo
           })
-          if(j % 10 == 0){
+          if(j % 30 == 0){
             console.log(`Page #${i}`)
             console.log(question)
           }
@@ -198,3 +179,8 @@ export async function goodjudgmentopen() {
   let difference = end - init
   console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`)
 }
+
+export async function goodjudmentopen(){
+  let cookie = process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudgmentopen")
+  await applyIfCookieExists(goodjudgmentopen_inner, cookie)
+}

src/platforms/hypermind-fetch.js

@@ -1,6 +1,7 @@
 /* Imports */
 import fs from 'fs'
 import axios from "axios"
+import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
 import toMarkdown from "../utils/toMarkdown.js"
 import { calculateStars } from "../utils/stars.js"
 import { upsert } from "../utils/mongo-wrapper.js"
@@ -14,23 +15,6 @@ function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
-function getcookie() {
-  return process.env.HYPERMINDCOOKIE
-  /*
-  try {
-    let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
-    let cookie = JSON.parse(rawcookie).hypermindcookie
-    if (cookie == undefined) {
-      throw new Error('No cookie for Hypermind!');
-    }
-    return cookie
-  } catch (error) {
-    console.log("Error: No cookies for Hypermind on src/privatekeys.json! See the README.md")
-    process.exit()
-  }
-  */
-}
-
 async function fetchHypermindData1(slug) {
   let response = await axios("https://predict.hypermind.com/dash/jsx.json", {
     "credentials": "omit",
@@ -48,7 +32,7 @@ async function fetchHypermindData1(slug) {
   return response
 }
 
-async function fetchHypermindData2() {
+async function fetchHypermindData2(cookie) {
   let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
     "credentials": "include",
     "headers": {
@@ -56,7 +40,7 @@ async function fetchHypermindData2() {
       "Accept": "*/*",
       "Accept-Language": "en-US,en;q=0.5",
       "Content-Type": "application/json; charset=UTF-8",
-      "Cookie": getcookie()
+      "Cookie": cookie
     },
     "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
     "data": `[["showcase","queryIFPs",{"query":{"showcaseOnly":true},"fmt":{"stats":true,"crowdFcstHist":true}}]]`,
@@ -66,7 +50,7 @@ async function fetchHypermindData2() {
   return response
 }
 
-async function fetchHypermindData3() {
+async function fetchHypermindData3(cookie) {
   let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
     "credentials": "include",
     "headers": {
@@ -74,7 +58,7 @@ async function fetchHypermindData3() {
       "Accept": "*/*",
       "Accept-Language": "en-US,en;q=0.5",
       "Content-Type": "application/json; charset=UTF-8",
-      "Cookie": getcookie()
+      "Cookie": cookie
     },
     "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
     "data": `[["showcase","getShowcase",{"showcase":"Covid19","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`,
@@ -88,7 +72,7 @@ async function fetchHypermindData3() {
   return response
 }
 
-async function fetchHypermindData4() {
+async function fetchHypermindData4(cookie) {
   let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", {
     "credentials": "include",
     "headers": {
@@ -96,7 +80,7 @@ async function fetchHypermindData4() {
       "Accept": "*/*",
       "Accept-Language": "en-US,en;q=0.5",
       "Content-Type": "application/json; charset=UTF-8",
-      "Cookie": getcookie()
+      "Cookie": cookie
     },
     "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true",
     "data": `[["showcase","getShowcase",{"showcase":"AI2023","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`,
@@ -111,7 +95,7 @@ async function fetchHypermindData4() {
 }
 
 /* Body */
-export async function hypermind() {
+async function hypermind_inner(cookie) {
   let slugs = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"]
 
   let results1 = []
@@ -156,7 +140,7 @@ export async function hypermind() {
 
   console.log("GDP")
   await sleep(1000 + Math.random() * 1000)
-  let results2 = await fetchHypermindData2()
+  let results2 = await fetchHypermindData2(cookie)
   let results2processed = results2.map(res => {
     //console.log(res.props.details)
     let descriptionraw = res.props.details.split("<hr size=1>")[0]
@@ -179,7 +163,7 @@ export async function hypermind() {
 
   console.log("COVID-19 OpenPhil")
   await sleep(1000 + Math.random() * 1000)
-  let results3 = await fetchHypermindData3()
+  let results3 = await fetchHypermindData3(cookie)
   //  console.log(results3)
   let results3processed = results3.map(res => {
     let descriptionraw = res.props.details.split("<hr size=1>")[0]
@@ -200,7 +184,7 @@ export async function hypermind() {
 
   console.log("AI in 2023")
   await sleep(1000 + Math.random() * 1000)
-  let results4 = await fetchHypermindData4()
+  let results4 = await fetchHypermindData4(cookie)
   // console.log(results4)
   let results4processed = results2.map(res => {
     let description = res.props.details
@@ -233,3 +217,8 @@ export async function hypermind() {
 
 }
 //hypermind()
+
+export async function hypermind() {
+  let cookie = process.env.HYPERMINDCOOKIE || getCookie("hypermind") 
+  await applyIfCookieExists(hypermind_inner, cookie)
+}

src/utils/getCookies.js

@@ -0,0 +1,25 @@
+import fs from 'fs'
+
+export function getCookie(property){
+    let answer = 0
+    try {
+        let rawcookie = fs.readFileSync("./src/input/privatekeys.json")
+        let cookie = JSON.parse(rawcookie)
+        if (cookie[property]){
+            answer = cookie[property]
+            console.log(`Got cookie: ${answer.slice(0,5)}...`)
+        }
+    } catch (error) {
+        console.log(error)
+    }
+    return answer
+}
+
+export async function applyIfCookieExists(cookie, fun){
+    if(cookie){
+        await fun(cookie)
+    }else if(!cookie){
+        console.log(`Cannot proceed with ${fun.name} because cookie does not exist`)
+        throw new Error(`No cookie for ${fun.name}`)
+    }
+}

src/utils/mongo-wrapper.js

@@ -1,8 +1,10 @@
 import pkg from 'mongodb';
 const {MongoClient} = pkg;
+import {getCookie} from "./getCookies.js"
+
 
 export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
-    const url = process.env.MONGODB_URL;
+    const url = process.env.MONGODB_URL || getCookie("mongodb");
     const client = new MongoClient(url);
 
     try {
@@ -40,7 +42,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
 }
 
 export async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){
-  const url = process.env.MONGODB_URL;
+  const url = process.env.MONGODB_URL || getCookie("mongodb");
 
   const client = new MongoClient(url, {
     useNewUrlParser: true,

src/utils/rebuildNetliftySiteWithNewData.js

@@ -1,9 +1,14 @@
 import axios from "axios"
+import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
+
+export async function rebuildNetlifySiteWithNewData_inner(cookie){
+    let payload = ({});
+    let response = await axios.post(cookie, payload);
+    let data = response.data;
+    console.log(data);
+}
 
 export async function rebuildNetlifySiteWithNewData(){
-    let rebuildNetlifyHookUrl = process.env.REBUIDNETLIFYHOOKURL;
+    let cookie = process.env.REBUIDNETLIFYHOOKURL || getCookie("netlify");
-    let payload = ({});
+    await applyIfCookieExists(rebuildNetlifySiteWithNewData_inner, cookie)
-    let response = await axios.post(rebuildNetlifyHookUrl, payload);
-    let data = response.data;
-    console.log(data);   
 }