From cf795eb90e465031f0dae2159f18cc7eebb7756b Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Sat, 10 Apr 2021 20:18:22 +0200 Subject: [PATCH] Added somewhat more complicated cookie functionality --- src/input/privatekeys_example.json | 8 ++-- src/platforms/csetforetell-fetch.js | 32 +++++---------- src/platforms/goodjudmentopen-fetch.js | 30 ++++---------- src/platforms/hypermind-fetch.js | 43 ++++++++------------- src/utils/getCookies.js | 25 ++++++++++++ src/utils/mongo-wrapper.js | 6 ++- src/utils/rebuildNetliftySiteWithNewData.js | 15 ++++--- 7 files changed, 78 insertions(+), 81 deletions(-) create mode 100644 src/utils/getCookies.js diff --git a/src/input/privatekeys_example.json b/src/input/privatekeys_example.json index fd64671..a9d73fb 100644 --- a/src/input/privatekeys_example.json +++ b/src/input/privatekeys_example.json @@ -1,5 +1,7 @@ { - "csetforetellcookie": "_cset_prod_fof_session=xxx; landing_url=https://www.cset-foretell.com/; randomization_seed=xxx; remember_user_token=xxx; ", - "goodjudmentopencookie": "__cfduid=xxx; _gj_prod_flyover_forecasts_session=xxx; randomization_seed=xxx; referring_url=https://www.gjopen.com/; remember_user_token=xxx;", - "hypermindcookie": "lumAuth=xxxx:yyyyyyyyyyyyyyyyy" + "csetforetell": "_cset_prod_fof_session=xxx; landing_url=https://www.cset-foretell.com/; randomization_seed=xxx; remember_user_token=xxx; ", + "goodjudmentopen": "__cfduid=xxx; _gj_prod_flyover_forecasts_session=xxx; randomization_seed=xxx; referring_url=https://www.gjopen.com/; remember_user_token=xxx;", + "hypermind": "lumAuth=xxxx:yyyyyyyyyyyyyyyyy", + "mongodb": "mongodb+srv://:@/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true", + "netlify": "https://api.netlify.com/build_hooks/?trigger_title=" } diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js index 72d02ac..e7d7f5f 100644 --- a/src/platforms/csetforetell-fetch.js +++ b/src/platforms/csetforetell-fetch.js @@ -1,6 +1,6 @@ /* Imports */ -import fs from 'fs' import axios from "axios" +import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import {Tabletojson} from "tabletojson" import toMarkdown from "../utils/toMarkdown.js" import {calculateStars} from "../utils/stars.js" @@ -12,24 +12,6 @@ String.prototype.replaceAll = function replaceAll(search, replace) { return this /* Support functions */ -function getcookie(){ - return process.env.CSETFORETELL_COOKIE - /* - try { - let rawcookie = fs.readFileSync("./src/input/privatekeys.json") - let cookie = JSON.parse(rawcookie).csetforetellcookie - if(cookie == undefined){ - throw new Error('No cookie for CSET-foretell!'); - } - - return cookie - } catch(error) { - console.log("Error: No cookies for CSET-foretell on src/privatekeys.json! See the README.md") - process.exit() - }*/ -} - - async function fetchPage(page, cookie){ let response = await axios({ @@ -131,8 +113,7 @@ function sleep(ms) { /* Body */ -export async function csetforetell(){ - let cookie = getcookie() +async function csetforetell_inner(cookie){ let i=1 let response = await fetchPage(i, cookie) let results = [] @@ -183,10 +164,17 @@ export async function csetforetell(){ } // let string = JSON.stringify(results,null, 2) // fs.writeFileSync('./data/csetforetell-questions.json', string); + console.log(results) await upsert(results, "csetforetell-questions") - + let end = Date.now() let difference = end-init console.log(`Took ${difference/1000} seconds, or ${difference/(1000*60)} minutes.`) } + + +export async function csetforetell(){ + let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell") + await applyIfCookieExists(csetforetell_inner, cookie) +} diff --git a/src/platforms/goodjudmentopen-fetch.js b/src/platforms/goodjudmentopen-fetch.js index 2847509..0c874c0 100644 --- a/src/platforms/goodjudmentopen-fetch.js +++ b/src/platforms/goodjudmentopen-fetch.js @@ -1,6 +1,7 @@ /* Imports */ import fs from 'fs' import axios from "axios" +import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import { Tabletojson } from "tabletojson" import { calculateStars } from "../utils/stars.js" import toMarkdown from "../utils/toMarkdown.js" @@ -12,24 +13,6 @@ let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecastin /* Support functions */ -function getcookie() { - return process.env.GOODJUDGMENTOPENCOOKIE - /* - try { - let rawcookie = fs.readFileSync("./src/input/privatekeys.json") - let cookie = JSON.parse(rawcookie).goodjudmentopencookie - if (cookie == undefined) { - throw new Error('No cookie for Good Judgment Open!'); - } - - return cookie - } catch (error) { - console.log("Error: No cookies for Good Judgment Open on src/privatekeys.json! See the README.md") - process.exit() - } - */ -} - async function fetchPage(page, cookie) { let response = await axios({ url: htmlEndPoint + page, @@ -134,9 +117,7 @@ function sleep(ms) { /* Body */ -export async function goodjudgmentopen() { - let cookie = getcookie() - +async function goodjudgmentopen_inner(cookie) { let i = 1 let response = await fetchPage(i, cookie) let results = [] @@ -166,7 +147,7 @@ export async function goodjudgmentopen() { "platform": "Good Judgment Open", ...moreinfo }) - if(j % 10 == 0){ + if(j % 30 == 0){ console.log(`Page #${i}`) console.log(question) } @@ -198,3 +179,8 @@ export async function goodjudgmentopen() { let difference = end - init console.log(`Took ${difference / 1000} seconds, or ${difference / (1000 * 60)} minutes.`) } + +export async function goodjudmentopen(){ + let cookie = process.env.GOODJUDGMENTOPENCOOKIE || getCookie("goodjudgmentopen") + await applyIfCookieExists(goodjudgmentopen_inner, cookie) +} diff --git a/src/platforms/hypermind-fetch.js b/src/platforms/hypermind-fetch.js index 5eff68a..4f9dc8a 100644 --- a/src/platforms/hypermind-fetch.js +++ b/src/platforms/hypermind-fetch.js @@ -1,6 +1,7 @@ /* Imports */ import fs from 'fs' import axios from "axios" +import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" import toMarkdown from "../utils/toMarkdown.js" import { calculateStars } from "../utils/stars.js" import { upsert } from "../utils/mongo-wrapper.js" @@ -14,23 +15,6 @@ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } -function getcookie() { - return process.env.HYPERMINDCOOKIE - /* - try { - let rawcookie = fs.readFileSync("./src/input/privatekeys.json") - let cookie = JSON.parse(rawcookie).hypermindcookie - if (cookie == undefined) { - throw new Error('No cookie for Hypermind!'); - } - return cookie - } catch (error) { - console.log("Error: No cookies for Hypermind on src/privatekeys.json! See the README.md") - process.exit() - } - */ -} - async function fetchHypermindData1(slug) { let response = await axios("https://predict.hypermind.com/dash/jsx.json", { "credentials": "omit", @@ -48,7 +32,7 @@ async function fetchHypermindData1(slug) { return response } -async function fetchHypermindData2() { +async function fetchHypermindData2(cookie) { let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { "credentials": "include", "headers": { @@ -56,7 +40,7 @@ async function fetchHypermindData2() { "Accept": "*/*", "Accept-Language": "en-US,en;q=0.5", "Content-Type": "application/json; charset=UTF-8", - "Cookie": getcookie() + "Cookie": cookie }, "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", "data": `[["showcase","queryIFPs",{"query":{"showcaseOnly":true},"fmt":{"stats":true,"crowdFcstHist":true}}]]`, @@ -66,7 +50,7 @@ async function fetchHypermindData2() { return response } -async function fetchHypermindData3() { +async function fetchHypermindData3(cookie) { let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { "credentials": "include", "headers": { @@ -74,7 +58,7 @@ async function fetchHypermindData3() { "Accept": "*/*", "Accept-Language": "en-US,en;q=0.5", "Content-Type": "application/json; charset=UTF-8", - "Cookie": getcookie() + "Cookie": cookie }, "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", "data": `[["showcase","getShowcase",{"showcase":"Covid19","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`, @@ -88,7 +72,7 @@ async function fetchHypermindData3() { return response } -async function fetchHypermindData4() { +async function fetchHypermindData4(cookie) { let response = await axios("https://prod.hypermind.com/ngdp-jsx/jsx.json", { "credentials": "include", "headers": { @@ -96,7 +80,7 @@ async function fetchHypermindData4() { "Accept": "*/*", "Accept-Language": "en-US,en;q=0.5", "Content-Type": "application/json; charset=UTF-8", - "Cookie": getcookie() + "Cookie": cookie }, "referrer": "https://prod.hypermind.com/ngdp/en/showcase/showcase.html?inFrame=true", "data": `[["showcase","getShowcase",{"showcase":"AI2023","fmt":{"fcsterCnt":true,"crowdFcst":true,"crowdFcstHist":true}}]]`, @@ -111,7 +95,7 @@ async function fetchHypermindData4() { } /* Body */ -export async function hypermind() { +async function hypermind_inner(cookie) { let slugs = ["USA", "FRA", "AFR", "INT", "COV", "POL", "ECO"] let results1 = [] @@ -156,7 +140,7 @@ export async function hypermind() { console.log("GDP") await sleep(1000 + Math.random() * 1000) - let results2 = await fetchHypermindData2() + let results2 = await fetchHypermindData2(cookie) let results2processed = results2.map(res => { //console.log(res.props.details) let descriptionraw = res.props.details.split("
")[0] @@ -179,7 +163,7 @@ export async function hypermind() { console.log("COVID-19 OpenPhil") await sleep(1000 + Math.random() * 1000) - let results3 = await fetchHypermindData3() + let results3 = await fetchHypermindData3(cookie) // console.log(results3) let results3processed = results3.map(res => { let descriptionraw = res.props.details.split("
")[0] @@ -200,7 +184,7 @@ export async function hypermind() { console.log("AI in 2023") await sleep(1000 + Math.random() * 1000) - let results4 = await fetchHypermindData4() + let results4 = await fetchHypermindData4(cookie) // console.log(results4) let results4processed = results2.map(res => { let description = res.props.details @@ -233,3 +217,8 @@ export async function hypermind() { } //hypermind() + +export async function hypermind() { + let cookie = process.env.HYPERMINDCOOKIE || getCookie("hypermind") + await applyIfCookieExists(hypermind_inner, cookie) +} \ No newline at end of file diff --git a/src/utils/getCookies.js b/src/utils/getCookies.js new file mode 100644 index 0000000..00e3e1c --- /dev/null +++ b/src/utils/getCookies.js @@ -0,0 +1,25 @@ +import fs from 'fs' + +export function getCookie(property){ + let answer = 0 + try { + let rawcookie = fs.readFileSync("./src/input/privatekeys.json") + let cookie = JSON.parse(rawcookie) + if (cookie[property]){ + answer = cookie[property] + console.log(`Got cookie: ${answer.slice(0,5)}...`) + } + } catch (error) { + console.log(error) + } + return answer +} + +export async function applyIfCookieExists(cookie, fun){ + if(cookie){ + await fun(cookie) + }else if(!cookie){ + console.log(`Cannot proceed with ${fun.name} because cookie does not exist`) + throw new Error(`No cookie for ${fun.name}`) + } +} \ No newline at end of file diff --git a/src/utils/mongo-wrapper.js b/src/utils/mongo-wrapper.js index b8bf38a..95dcfcf 100644 --- a/src/utils/mongo-wrapper.js +++ b/src/utils/mongo-wrapper.js @@ -1,8 +1,10 @@ import pkg from 'mongodb'; const {MongoClient} = pkg; +import {getCookie} from "./getCookies.js" + export async function upsert (contents, documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){ - const url = process.env.MONGODB_URL; + const url = process.env.MONGODB_URL || getCookie("mongodb"); const client = new MongoClient(url); try { @@ -40,7 +42,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca } export async function mongoRead (documentName, collectionName="metaforecastCollection", databaseName="metaforecastDatabase"){ - const url = process.env.MONGODB_URL; + const url = process.env.MONGODB_URL || getCookie("mongodb"); const client = new MongoClient(url, { useNewUrlParser: true, diff --git a/src/utils/rebuildNetliftySiteWithNewData.js b/src/utils/rebuildNetliftySiteWithNewData.js index d6a8941..4d93c4a 100644 --- a/src/utils/rebuildNetliftySiteWithNewData.js +++ b/src/utils/rebuildNetliftySiteWithNewData.js @@ -1,9 +1,14 @@ import axios from "axios" +import {getCookie, applyIfCookieExists} from "../utils/getCookies.js" + +export async function rebuildNetlifySiteWithNewData_inner(cookie){ + let payload = ({}); + let response = await axios.post(cookie, payload); + let data = response.data; + console.log(data); +} export async function rebuildNetlifySiteWithNewData(){ - let rebuildNetlifyHookUrl = process.env.REBUIDNETLIFYHOOKURL; - let payload = ({}); - let response = await axios.post(rebuildNetlifyHookUrl, payload); - let data = response.data; - console.log(data); + let cookie = process.env.REBUIDNETLIFYHOOKURL || getCookie("netlify"); + await applyIfCookieExists(rebuildNetlifySiteWithNewData_inner, cookie) } \ No newline at end of file