quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove getSecret, always use env variables

Browse Source
This commit is contained in:
Vyacheslav Matyukhin 2022-03-24 01:07:55 +03:00
parent fad762ab3a
commit f3592cde40
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
12 changed files with 30 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/backend/database/mongo-wrapper.js
View File

@ -1,5 +1,4 @@
import pkg from "mongodb";
import { getSecret } from "../utils/getSecrets.js";
import { roughSizeOfObject } from "../utils/roughSize.js";
const { MongoClient } = pkg;
@ -9,7 +8,7 @@ export async function mongoUpsert(
collectionName = "metaforecastCollection",
databaseName = "metaforecastDatabase"
) {
const url = process.env.MONGODB_URL || getSecret("mongodb");
const url = process.env.MONGODB_URL;
const client = new MongoClient(url);
try {
await client.connect();
@ -60,7 +59,7 @@ export async function mongoRead(
collectionName = "metaforecastCollection",
databaseName = "metaforecastDatabase"
) {
const url = process.env.MONGODB_URL || getSecret("mongodb");
const url = process.env.MONGODB_URL;
const client = new MongoClient(url, {
useNewUrlParser: true,
@ -142,7 +141,7 @@ export async function mongoGetAllElements(
databaseName = "metaforecastDatabase",
collectionName = "metaforecastCollection"
) {
const url = process.env.MONGODB_URL || getSecret("mongodb");
const url = process.env.MONGODB_URL;
const client = new MongoClient(url, {
useNewUrlParser: true,
useUnifiedTopology: true,

7
src/backend/database/pg-wrapper.js
View File

@ -1,6 +1,5 @@
import pkg from "pg";
import { platformNames } from "../platforms/all/platformNames.js";
import { getSecret } from "../utils/getSecrets.js";
import { hash } from "../utils/hash.js";
import { roughSizeOfObject } from "../utils/roughSize.js";
const { Pool } = pkg;
@ -35,9 +34,7 @@ const tableWhiteList = [
];
/* Postgres database connection code */
const databaseURL =
process.env.DIGITALOCEAN_POSTGRES || getSecret("digitalocean-postgres");
// process.env.DATABASE_URL || getSecret("heroku-postgres")
const databaseURL = process.env.DIGITALOCEAN_POSTGRES;
export const readWritePool = new Pool({
connectionString: databaseURL,
ssl: process.env.POSTGRES_NO_SSL
@ -49,7 +46,7 @@ export const readWritePool = new Pool({
const readOnlyDatabaseURL =
"postgresql://public_read_only_user:gOcihnLhqRIQUQYt@postgres-red-do-user-10290909-0.b.db.ondigitalocean.com:25060/metaforecastpg?sslmode=require" ||
getSecret("digitalocean-postgres-public");
process.env.DIGITALOCEAN_POSTGRES_PUBLIC;
const readOnlyPool = new Pool({
// never used
connectionString: readOnlyDatabaseURL,

6
src/backend/flow/rebuildNetliftySiteWithNewData.js
View File

@ -1,7 +1,7 @@
import axios from "axios";
import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js";
import { applyIfSecretExists } from "../utils/getSecrets.js";
export async function rebuildNetlifySiteWithNewData_inner(cookie) {
async function rebuildNetlifySiteWithNewData_inner(cookie) {
let payload = {};
let response = await axios.post(cookie, payload);
let data = response.data;
@ -9,6 +9,6 @@ export async function rebuildNetlifySiteWithNewData_inner(cookie) {
}
export async function rebuildNetlifySiteWithNewData() {
let cookie = process.env.REBUIDNETLIFYHOOKURL || getSecret("netlify");
let cookie = process.env.REBUIDNETLIFYHOOKURL;
await applyIfSecretExists(cookie, rebuildNetlifySiteWithNewData_inner);
}

4
src/backend/platforms/betfair-fetch.js
View File

@ -1,9 +1,8 @@
/* Imports */
import fs from "fs";
import axios from "axios";
import https from "https";
import { calculateStars } from "../utils/stars.js";
import { databaseUpsert } from "../database/database-wrapper.js";
import { calculateStars } from "../utils/stars.js";
/* Definitions */
let endpoint = process.env.SECRET_BETFAIR_ENDPOINT;
@ -142,7 +141,6 @@ export async function betfair() {
let results = await processPredictions(data); // somehow needed
// console.log(results.map(result => ({title: result.title, description: result.description})))
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string);
await databaseUpsert({ contents: results, group: "betfair" });
console.log("Done");
}

8
src/backend/platforms/deprecated/csetforetell-fetch.js
View File

@ -1,10 +1,10 @@
/* Imports */
import axios from "axios";
import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js";
import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { databaseUpsert } from "../utils/database-wrapper.js";
import { applyIfSecretExists } from "../utils/getSecrets.js";
import { calculateStars } from "../utils/stars.js";
import toMarkdown from "../utils/toMarkdown.js";
/* Definitions */
let htmlEndPoint = "https://www.cset-foretell.com/questions?page=";
@ -278,6 +278,6 @@ async function csetforetell_inner(cookie) {
}
export async function csetforetell() {
let cookie = process.env.CSETFORETELL_COOKIE || getSecret("csetforetell");
let cookie = process.env.CSETFORETELL_COOKIE;
await applyIfSecretExists(cookie, csetforetell_inner);
}

10
src/backend/platforms/deprecated/hypermind-fetch.js
View File

@ -1,12 +1,10 @@
/* Imports */
import fs from "fs";
import axios from "axios";
import https from "https";
import fetch from "isomorphic-fetch";
import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { databaseUpsert } from "../utils/database-wrapper.js";
import { applyIfSecretExists } from "../utils/getSecrets.js";
import { calculateStars } from "../utils/stars.js";
import toMarkdown from "../utils/toMarkdown.js";
/* Definitions */
let hypermindEnpoint1 = "https://predict.hypermind.com/dash/jsx.json";
@ -200,6 +198,6 @@ async function hypermind_inner(cookie) {
//hypermind()
export async function hypermind() {
let cookie = process.env.HYPERMINDCOOKIE || getSecret("hypermind");
let cookie = process.env.HYPERMINDCOOKIE;
await applyIfSecretExists(cookie, hypermind_inner);
}

5
src/backend/platforms/example-fetch.js
View File

@ -1,9 +1,7 @@
/* Imports */
import fs from "fs";
import axios from "axios";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { databaseUpsert } from "../database/database-wrapper.js";
import { calculateStars } from "../utils/stars.js";
/* Definitions */
let endpoint = "https://example.com/";
@ -63,7 +61,6 @@ export async function example() {
let results = await processPredictions(data); // somehow needed
// console.log(results)
// let string = JSON.stringify(results, null, 2)
// fs.writeFileSync('polyprediction-questions.json', string);
await databaseUpsert({ contents: results, group: "example" });
console.log("Done");
}

8
src/backend/platforms/goodjudmentopen-fetch.js
View File

@ -1,11 +1,10 @@
/* Imports */
import fs from "fs";
import axios from "axios";
import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js";
import { Tabletojson } from "tabletojson";
import { databaseUpsert } from "../database/database-wrapper.js";
import { applyIfSecretExists } from "../utils/getSecrets.js";
import { calculateStars } from "../utils/stars.js";
import toMarkdown from "../utils/toMarkdown.js";
import { databaseUpsert } from "../database/database-wrapper.js";
/* Definitions */
let htmlEndPoint = "https://www.gjopen.com/questions?page=";
@ -236,7 +235,6 @@ async function goodjudgmentopen_inner(cookie) {
}
export async function goodjudgmentopen() {
let cookie =
process.env.GOODJUDGMENTOPENCOOKIE || getSecret("goodjudmentopen");
let cookie = process.env.GOODJUDGMENTOPENCOOKIE;
await applyIfSecretExists(cookie, goodjudgmentopen_inner);
}

8
src/backend/platforms/infer-fetch.js
View File

@ -1,10 +1,10 @@
/* Imports */
import axios from "axios";
import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js";
import { Tabletojson } from "tabletojson";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { databaseUpsert } from "../database/database-wrapper.js";
import { applyIfSecretExists } from "../utils/getSecrets.js";
import { calculateStars } from "../utils/stars.js";
import toMarkdown from "../utils/toMarkdown.js";
/* Definitions */
let htmlEndPoint = "https://www.infer-pub.com/questions";
@ -282,6 +282,6 @@ async function infer_inner(cookie) {
}
export async function infer() {
let cookie = process.env.INFER_COOKIE || getSecret("infer");
let cookie = process.env.INFER_COOKIE;
await applyIfSecretExists(cookie, infer_inner);
}

10
src/backend/platforms/wildeford-fetch.js
View File

@ -1,12 +1,10 @@
/* Imports */
import fs from "fs";
// import axios from "axios"
import { GoogleSpreadsheet } from "google-spreadsheet";
import { getSecret, applyIfSecretExists } from "../utils/getSecrets.js";
import toMarkdown from "../utils/toMarkdown.js";
import { calculateStars } from "../utils/stars.js";
import { hash } from "../utils/hash.js";
import { databaseUpsert } from "../database/database-wrapper.js";
import { applyIfSecretExists } from "../utils/getSecrets.js";
import { hash } from "../utils/hash.js";
import { calculateStars } from "../utils/stars.js";
/* Definitions */
const SHEET_ID = "1xcgYF7Q0D95TPHLLSgwhWBHFrWZUGJn7yTyAhDR4vi0"; // spreadsheet key is the long id in the sheets URL
@ -132,6 +130,6 @@ export async function wildeford_inner(google_api_key) {
//example()
export async function wildeford() {
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY || getSecret("google-api"); // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY; // See: https://developers.google.com/sheets/api/guides/authorizing#APIKey
await applyIfSecretExists(GOOGLE_API_KEY, wildeford_inner);
}

4
src/backend/utils/algolia.js
View File

@ -1,10 +1,8 @@
import algoliasearch from "algoliasearch";
import fs from "fs";
import { getSecret } from "./getSecrets.js";
import { databaseReadWithReadCredentials } from "../database/database-wrapper.js";
import { mergeEverythingInner } from "../flow/mergeEverything.js";
let cookie = process.env.ALGOLIA_MASTER_API_KEY || getSecret("algolia");
let cookie = process.env.ALGOLIA_MASTER_API_KEY;
const client = algoliasearch("96UD3NTQ7L", cookie);
const index = client.initIndex("metaforecast");

17
src/backend/utils/getSecrets.js
View File

@ -1,20 +1,3 @@
import fs from "fs";
export function getSecret(property) {
let answer = 0;
try {
let rawcookie = fs.readFileSync("./input/secrets.json");
let cookie = JSON.parse(rawcookie);
if (cookie[property]) {
answer = cookie[property];
console.log(`Got cookie: ${answer.slice(0, 5)}...`);
}
} catch (error) {
console.log(error);
}
return answer;
}
export async function applyIfSecretExists(cookie, fun) {
if (cookie) {
await fun(cookie);