From f3592cde40f8c6795b8edc3ef9538bed0001f1d3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 24 Mar 2022 01:07:55 +0300 Subject: [PATCH] remove getSecret, always use env variables --- src/backend/database/mongo-wrapper.js | 7 +++---- src/backend/database/pg-wrapper.js | 7 ++----- .../flow/rebuildNetliftySiteWithNewData.js | 6 +++--- src/backend/platforms/betfair-fetch.js | 4 +--- .../platforms/deprecated/csetforetell-fetch.js | 8 ++++---- .../platforms/deprecated/hypermind-fetch.js | 10 ++++------ src/backend/platforms/example-fetch.js | 5 +---- src/backend/platforms/goodjudmentopen-fetch.js | 8 +++----- src/backend/platforms/infer-fetch.js | 8 ++++---- src/backend/platforms/wildeford-fetch.js | 10 ++++------ src/backend/utils/algolia.js | 4 +--- src/backend/utils/getSecrets.js | 17 ----------------- 12 files changed, 30 insertions(+), 64 deletions(-) diff --git a/src/backend/database/mongo-wrapper.js b/src/backend/database/mongo-wrapper.js index 122ee75..c622f38 100644 --- a/src/backend/database/mongo-wrapper.js +++ b/src/backend/database/mongo-wrapper.js @@ -1,5 +1,4 @@ import pkg from "mongodb"; -import { getSecret } from "../utils/getSecrets.js"; import { roughSizeOfObject } from "../utils/roughSize.js"; const { MongoClient } = pkg; @@ -9,7 +8,7 @@ export async function mongoUpsert( collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase" ) { - const url = process.env.MONGODB_URL || getSecret("mongodb"); + const url = process.env.MONGODB_URL; const client = new MongoClient(url); try { await client.connect(); @@ -60,7 +59,7 @@ export async function mongoRead( collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase" ) { - const url = process.env.MONGODB_URL || getSecret("mongodb"); + const url = process.env.MONGODB_URL; const client = new MongoClient(url, { useNewUrlParser: true, @@ -142,7 +141,7 @@ export async function mongoGetAllElements( databaseName = "metaforecastDatabase", collectionName = "metaforecastCollection" ) { - const url = process.env.MONGODB_URL || getSecret("mongodb"); + const url = process.env.MONGODB_URL; const client = new MongoClient(url, { useNewUrlParser: true, useUnifiedTopology: true, diff --git a/src/backend/database/pg-wrapper.js b/src/backend/database/pg-wrapper.js index 506f21c..3fd5f92 100644 --- a/src/backend/database/pg-wrapper.js +++ b/src/backend/database/pg-wrapper.js @@ -1,6 +1,5 @@ import pkg from "pg"; import { platformNames } from "../platforms/all/platformNames.js"; -import { getSecret } from "../utils/getSecrets.js"; import { hash } from "../utils/hash.js"; import { roughSizeOfObject } from "../utils/roughSize.js"; const { Pool } = pkg; @@ -35,9 +34,7 @@ const tableWhiteList = [ ]; /* Postgres database connection code */ -const databaseURL = - process.env.DIGITALOCEAN_POSTGRES || getSecret("digitalocean-postgres"); -// process.env.DATABASE_URL || getSecret("heroku-postgres") +const databaseURL = process.env.DIGITALOCEAN_POSTGRES; export const readWritePool = new Pool({ connectionString: databaseURL, ssl: process.env.POSTGRES_NO_SSL @@ -49,7 +46,7 @@ export const readWritePool = new Pool({ const readOnlyDatabaseURL = "postgresql://public_read_only_user:gOcihnLhqRIQUQYt@postgres-red-do-user-10290909-0.b.db.ondigitalocean.com:25060/metaforecastpg?sslmode=require" || - getSecret("digitalocean-postgres-public"); + process.env.DIGITALOCEAN_POSTGRES_PUBLIC; const readOnlyPool = new Pool({ // never used connectionString: readOnlyDatabaseURL, diff --git a/src/backend/flow/rebuildNetliftySiteWithNewData.js b/src/backend/flow/rebuildNetliftySiteWithNewData.js index 96583da..c8ee81f 100644 --- a/src/backend/flow/rebuildNetliftySiteWithNewData.js +++ b/src/backend/flow/rebuildNetliftySiteWithNewData.js @@ -1,7 +1,7 @@ import axios from "axios"; -import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js"; +import { applyIfSecretExists } from "../utils/getSecrets.js"; -export async function rebuildNetlifySiteWithNewData_inner(cookie) { +async function rebuildNetlifySiteWithNewData_inner(cookie) { let payload = {}; let response = await axios.post(cookie, payload); let data = response.data; @@ -9,6 +9,6 @@ export async function rebuildNetlifySiteWithNewData_inner(cookie) { } export async function rebuildNetlifySiteWithNewData() { - let cookie = process.env.REBUIDNETLIFYHOOKURL || getSecret("netlify"); + let cookie = process.env.REBUIDNETLIFYHOOKURL; await applyIfSecretExists(cookie, rebuildNetlifySiteWithNewData_inner); } diff --git a/src/backend/platforms/betfair-fetch.js b/src/backend/platforms/betfair-fetch.js index 15cd0da..bf61237 100644 --- a/src/backend/platforms/betfair-fetch.js +++ b/src/backend/platforms/betfair-fetch.js @@ -1,9 +1,8 @@ /* Imports */ -import fs from "fs"; import axios from "axios"; import https from "https"; -import { calculateStars } from "../utils/stars.js"; import { databaseUpsert } from "../database/database-wrapper.js"; +import { calculateStars } from "../utils/stars.js"; /* Definitions */ let endpoint = process.env.SECRET_BETFAIR_ENDPOINT; @@ -142,7 +141,6 @@ export async function betfair() { let results = await processPredictions(data); // somehow needed // console.log(results.map(result => ({title: result.title, description: result.description}))) // let string = JSON.stringify(results, null, 2) - // fs.writeFileSync('polyprediction-questions.json', string); await databaseUpsert({ contents: results, group: "betfair" }); console.log("Done"); } diff --git a/src/backend/platforms/deprecated/csetforetell-fetch.js b/src/backend/platforms/deprecated/csetforetell-fetch.js index d65d9c8..7f1ea07 100644 --- a/src/backend/platforms/deprecated/csetforetell-fetch.js +++ b/src/backend/platforms/deprecated/csetforetell-fetch.js @@ -1,10 +1,10 @@ /* Imports */ import axios from "axios"; -import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js"; import { Tabletojson } from "tabletojson"; -import toMarkdown from "../utils/toMarkdown.js"; -import { calculateStars } from "../utils/stars.js"; import { databaseUpsert } from "../utils/database-wrapper.js"; +import { applyIfSecretExists } from "../utils/getSecrets.js"; +import { calculateStars } from "../utils/stars.js"; +import toMarkdown from "../utils/toMarkdown.js"; /* Definitions */ let htmlEndPoint = "https://www.cset-foretell.com/questions?page="; @@ -278,6 +278,6 @@ async function csetforetell_inner(cookie) { } export async function csetforetell() { - let cookie = process.env.CSETFORETELL_COOKIE || getSecret("csetforetell"); + let cookie = process.env.CSETFORETELL_COOKIE; await applyIfSecretExists(cookie, csetforetell_inner); } diff --git a/src/backend/platforms/deprecated/hypermind-fetch.js b/src/backend/platforms/deprecated/hypermind-fetch.js index 1ca543a..ad4364c 100644 --- a/src/backend/platforms/deprecated/hypermind-fetch.js +++ b/src/backend/platforms/deprecated/hypermind-fetch.js @@ -1,12 +1,10 @@ /* Imports */ -import fs from "fs"; import axios from "axios"; import https from "https"; -import fetch from "isomorphic-fetch"; -import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js"; -import toMarkdown from "../utils/toMarkdown.js"; -import { calculateStars } from "../utils/stars.js"; import { databaseUpsert } from "../utils/database-wrapper.js"; +import { applyIfSecretExists } from "../utils/getSecrets.js"; +import { calculateStars } from "../utils/stars.js"; +import toMarkdown from "../utils/toMarkdown.js"; /* Definitions */ let hypermindEnpoint1 = "https://predict.hypermind.com/dash/jsx.json"; @@ -200,6 +198,6 @@ async function hypermind_inner(cookie) { //hypermind() export async function hypermind() { - let cookie = process.env.HYPERMINDCOOKIE || getSecret("hypermind"); + let cookie = process.env.HYPERMINDCOOKIE; await applyIfSecretExists(cookie, hypermind_inner); } diff --git a/src/backend/platforms/example-fetch.js b/src/backend/platforms/example-fetch.js index 6d7ddcf..19bf4bc 100644 --- a/src/backend/platforms/example-fetch.js +++ b/src/backend/platforms/example-fetch.js @@ -1,9 +1,7 @@ /* Imports */ -import fs from "fs"; import axios from "axios"; -import toMarkdown from "../utils/toMarkdown.js"; -import { calculateStars } from "../utils/stars.js"; import { databaseUpsert } from "../database/database-wrapper.js"; +import { calculateStars } from "../utils/stars.js"; /* Definitions */ let endpoint = "https://example.com/"; @@ -63,7 +61,6 @@ export async function example() { let results = await processPredictions(data); // somehow needed // console.log(results) // let string = JSON.stringify(results, null, 2) - // fs.writeFileSync('polyprediction-questions.json', string); await databaseUpsert({ contents: results, group: "example" }); console.log("Done"); } diff --git a/src/backend/platforms/goodjudmentopen-fetch.js b/src/backend/platforms/goodjudmentopen-fetch.js index cb5aa64..881c338 100644 --- a/src/backend/platforms/goodjudmentopen-fetch.js +++ b/src/backend/platforms/goodjudmentopen-fetch.js @@ -1,11 +1,10 @@ /* Imports */ -import fs from "fs"; import axios from "axios"; -import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js"; import { Tabletojson } from "tabletojson"; +import { databaseUpsert } from "../database/database-wrapper.js"; +import { applyIfSecretExists } from "../utils/getSecrets.js"; import { calculateStars } from "../utils/stars.js"; import toMarkdown from "../utils/toMarkdown.js"; -import { databaseUpsert } from "../database/database-wrapper.js"; /* Definitions */ let htmlEndPoint = "https://www.gjopen.com/questions?page="; @@ -236,7 +235,6 @@ async function goodjudgmentopen_inner(cookie) { } export async function goodjudgmentopen() { - let cookie = - process.env.GOODJUDGMENTOPENCOOKIE || getSecret("goodjudmentopen"); + let cookie = process.env.GOODJUDGMENTOPENCOOKIE; await applyIfSecretExists(cookie, goodjudgmentopen_inner); } diff --git a/src/backend/platforms/infer-fetch.js b/src/backend/platforms/infer-fetch.js index f2f7e80..2ea30c8 100644 --- a/src/backend/platforms/infer-fetch.js +++ b/src/backend/platforms/infer-fetch.js @@ -1,10 +1,10 @@ /* Imports */ import axios from "axios"; -import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js"; import { Tabletojson } from "tabletojson"; -import toMarkdown from "../utils/toMarkdown.js"; -import { calculateStars } from "../utils/stars.js"; import { databaseUpsert } from "../database/database-wrapper.js"; +import { applyIfSecretExists } from "../utils/getSecrets.js"; +import { calculateStars } from "../utils/stars.js"; +import toMarkdown from "../utils/toMarkdown.js"; /* Definitions */ let htmlEndPoint = "https://www.infer-pub.com/questions"; @@ -282,6 +282,6 @@ async function infer_inner(cookie) { } export async function infer() { - let cookie = process.env.INFER_COOKIE || getSecret("infer"); + let cookie = process.env.INFER_COOKIE; await applyIfSecretExists(cookie, infer_inner); } diff --git a/src/backend/platforms/wildeford-fetch.js b/src/backend/platforms/wildeford-fetch.js index b1752d3..6e2225d 100644 --- a/src/backend/platforms/wildeford-fetch.js +++ b/src/backend/platforms/wildeford-fetch.js @@ -1,12 +1,10 @@ /* Imports */ -import fs from "fs"; // import axios from "axios" import { GoogleSpreadsheet } from "google-spreadsheet"; -import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js"; -import toMarkdown from "../utils/toMarkdown.js"; -import { calculateStars } from "../utils/stars.js"; -import { hash } from "../utils/hash.js"; import { databaseUpsert } from "../database/database-wrapper.js"; +import { applyIfSecretExists } from "../utils/getSecrets.js"; +import { hash } from "../utils/hash.js"; +import { calculateStars } from "../utils/stars.js"; /* Definitions */ const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL @@ -132,6 +130,6 @@ export async function wildeford_inner(google_api_key) { //example() export async function wildeford() { - const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getSecret("google-api"); // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey + const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner); } diff --git a/src/backend/utils/algolia.js b/src/backend/utils/algolia.js index 8e3fe35..df21bec 100644 --- a/src/backend/utils/algolia.js +++ b/src/backend/utils/algolia.js @@ -1,10 +1,8 @@ import algoliasearch from "algoliasearch"; -import fs from "fs"; -import { getSecret } from "./getSecrets.js"; import { databaseReadWithReadCredentials } from "../database/database-wrapper.js"; import { mergeEverythingInner } from "../flow/mergeEverything.js"; -let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia"); +let cookie = process.env.ALGOLIA_MASTER_API_KEY; const client = algoliasearch("96UD3NTQ7L", cookie); const index = client.initIndex("metaforecast"); diff --git a/src/backend/utils/getSecrets.js b/src/backend/utils/getSecrets.js index 82b799b..5ce5e1b 100644 --- a/src/backend/utils/getSecrets.js +++ b/src/backend/utils/getSecrets.js @@ -1,20 +1,3 @@ -import fs from "fs"; - -export function getSecret(property) { - let answer = 0; - try { - let rawcookie = fs.readFileSync("./input/secrets.json"); - let cookie = JSON.parse(rawcookie); - if (cookie[property]) { - answer = cookie[property]; - console.log(`Got cookie: ${answer.slice(0, 5)}...`); - } - } catch (error) { - console.log(error); - } - return answer; -} - export async function applyIfSecretExists(cookie, fun) { if (cookie) { await fun(cookie);