From a1c3f03d7d849a074f4eba4abe8884cf04851c2b Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Mon, 28 Mar 2022 02:09:34 +0300 Subject: [PATCH] feat: remove mongodb support also: - refactor and significantly simplify database-wrapper code - rename manualSendToMongo to manualSendToDb --- README.md | 2 +- docs/configuration.md | 3 +- env.example | 2 - package-lock.json | 191 +++--------------- package.json | 1 - src/backend/database/database-wrapper.ts | 158 +++------------ src/backend/database/mongo-wrapper.ts | 179 ---------------- src/backend/index.ts | 2 +- ...manualSendToMongo.ts => manualSendToDb.ts} | 1 - 9 files changed, 60 insertions(+), 479 deletions(-) delete mode 100644 src/backend/database/mongo-wrapper.ts rename src/backend/manual/{manualSendToMongo.ts => manualSendToDb.ts} (95%) diff --git a/README.md b/README.md index f17cf97..2d04538 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Overall, the services which we use are: - Algolia for search - Netlify for website deployment - DigitalOcean for background jobs, e.g. fetching new forecasts -- Postgres on DigitalOcean and Mongo for databases +- Postgres on DigitalOcean for database ## Various notes diff --git a/docs/configuration.md b/docs/configuration.md index 38a4bfc..8e58241 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,14 +13,13 @@ There's also a template configuration file in `../env.example`. - `DIGITALOCEAN_POSTGRES`, of the form `postgres://username:password@domain.com:port/configvars`. (Disregard `DIGITALOCEAN_` prefix, you can use any endpoint you like). - `DIGITALOCEAN_POSTGRES_PUBLIC` -- `MONGODB_URL`, a string in the format `"mongodb+srv://:@/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"` (No longer really needed, superseded by Postgres). - `ALGOLIA_MASTER_API_KEY`, a string of 32 hexidecimal characters, like `19b6c2234e50c98d30668659a39e3127` (not an actual key). ## Platform cookies and keys Most of these are just session cookies, necessary to query INFER (previously CSET Foretell), Good Judgment Open and Hypermind (Hypermind is now deprecated). You can get these cookies by creating an account in said platforms and then making and inspecting a request (e.g., by making a prediction, or browsing questions). -Note that not all of these cookies are needed to use all parts of the source code. For instance, to download Polymarket data, one could just interface with the Polymarket code. In particular, the code in this repository contains code to with the mongo database using read permissions, which are freely available. +Note that not all of these cookies are needed to use all parts of the source code. For instance, to download Polymarket data, one could just interface with the Polymarket code. In particular, the code in this repository contains code to connect with the postgres database using read permissions, which are freely available. - `GOODJUDGEMENTOPENCOOKIE` - `INFER_COOKIE` diff --git a/env.example b/env.example index bf54efe..9a2eda2 100644 --- a/env.example +++ b/env.example @@ -6,8 +6,6 @@ # DIGITALOCEAN_POSTGRES=postgresql://...@localhost:5432/...?schema=public # POSTGRES_NO_SSL=1 -# MONGODB_URL=mongodb://localhost:27017 - # NEXT_PUBLIC_SITE_URL=http://localhost:3000 # DEBUG_MODE=off diff --git a/package-lock.json b/package-lock.json index 3761b35..c8029f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,6 @@ "https": "^1.0.0", "isomorphic-fetch": "^3.0.0", "json2csv": "^5.0.5", - "mongodb": "^3.6.6", "multiselect-react-dropdown": "^2.0.17", "next": "12", "nprogress": "^0.2.0", @@ -2307,15 +2306,6 @@ "node": ">=1.0.0" } }, - "node_modules/bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -2373,14 +2363,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/bson": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", - "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==", - "engines": { - "node": ">=0.6.19" - } - }, "node_modules/btoa": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", @@ -2841,7 +2823,8 @@ "node_modules/core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true }, "node_modules/cosmiconfig": { "version": "7.0.1", @@ -3214,6 +3197,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", + "dev": true, "engines": { "node": ">=0.10" } @@ -4304,7 +4288,8 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true }, "node_modules/ini": { "version": "1.3.8", @@ -4595,7 +4580,8 @@ "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true }, "node_modules/isexe": { "version": "2.0.0", @@ -5193,12 +5179,6 @@ "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==", "license": "MIT" }, - "node_modules/memory-pager": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true - }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -5897,44 +5877,6 @@ "micro-memoize": "^4.0.9" } }, - "node_modules/mongodb": { - "version": "3.6.6", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz", - "integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==", - "dependencies": { - "bl": "^2.2.1", - "bson": "^1.1.4", - "denque": "^1.4.1", - "optional-require": "^1.0.2", - "safe-buffer": "^5.1.2" - }, - "engines": { - "node": ">=4" - }, - "optionalDependencies": { - "saslprep": "^1.0.0" - }, - "peerDependenciesMeta": { - "aws4": { - "optional": true - }, - "bson-ext": { - "optional": true - }, - "kerberos": { - "optional": true - }, - "mongodb-client-encryption": { - "optional": true - }, - "mongodb-extjson": { - "optional": true - }, - "snappy": { - "optional": true - } - } - }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -32281,14 +32223,6 @@ "node": ">=8" } }, - "node_modules/optional-require": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz", - "integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg==", - "engines": { - "node": ">=4" - } - }, "node_modules/outdent": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz", @@ -33181,7 +33115,8 @@ "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true }, "node_modules/prop-types": { "version": "15.8.1", @@ -33497,6 +33432,7 @@ "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -33510,7 +33446,8 @@ "node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true }, "node_modules/readdirp": { "version": "3.6.0", @@ -33752,18 +33689,6 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, - "node_modules/saslprep": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", - "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", - "optional": true, - "dependencies": { - "sparse-bitfield": "^3.0.3" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/scheduler": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz", @@ -34011,15 +33936,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/sparse-bitfield": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", - "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", - "optional": true, - "dependencies": { - "memory-pager": "^1.0.2" - } - }, "node_modules/split-on-first": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", @@ -34095,6 +34011,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -34102,7 +34019,8 @@ "node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true }, "node_modules/string-width": { "version": "1.0.2", @@ -36532,15 +36450,6 @@ "psl": "^1.1.7" } }, - "bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -36576,11 +36485,6 @@ "picocolors": "^1.0.0" } }, - "bson": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", - "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" - }, "btoa": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", @@ -36892,7 +36796,8 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true }, "cosmiconfig": { "version": "7.0.1", @@ -37140,7 +37045,8 @@ "denque": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", - "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", + "dev": true }, "dequal": { "version": "2.0.2", @@ -37892,7 +37798,8 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true }, "ini": { "version": "1.3.8", @@ -38085,7 +37992,8 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true }, "isexe": { "version": "2.0.0", @@ -38527,12 +38435,6 @@ "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==" }, - "memory-pager": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true - }, "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -38922,19 +38824,6 @@ "micro-memoize": "^4.0.9" } }, - "mongodb": { - "version": "3.6.6", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz", - "integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==", - "requires": { - "bl": "^2.2.1", - "bson": "^1.1.4", - "denque": "^1.4.1", - "optional-require": "^1.0.2", - "safe-buffer": "^5.1.2", - "saslprep": "^1.0.0" - } - }, "mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -58949,11 +58838,6 @@ "is-wsl": "^2.1.1" } }, - "optional-require": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz", - "integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg==" - }, "outdent": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz", @@ -59500,7 +59384,8 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true }, "prop-types": { "version": "15.8.1", @@ -59716,6 +59601,7 @@ "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -59729,7 +59615,8 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true } } }, @@ -59876,15 +59763,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "saslprep": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", - "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", - "optional": true, - "requires": { - "sparse-bitfield": "^3.0.3" - } - }, "scheduler": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz", @@ -60040,15 +59918,6 @@ "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" }, - "sparse-bitfield": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", - "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", - "optional": true, - "requires": { - "memory-pager": "^1.0.2" - } - }, "split-on-first": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", @@ -60107,6 +59976,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, "requires": { "safe-buffer": "~5.1.0" }, @@ -60114,7 +59984,8 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true } } }, diff --git a/package.json b/package.json index f113e7b..a49ef52 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "https": "^1.0.0", "isomorphic-fetch": "^3.0.0", "json2csv": "^5.0.5", - "mongodb": "^3.6.6", "multiselect-react-dropdown": "^2.0.17", "next": "12", "nprogress": "^0.2.0", diff --git a/src/backend/database/database-wrapper.ts b/src/backend/database/database-wrapper.ts index c3a7b4c..943c149 100644 --- a/src/backend/database/database-wrapper.ts +++ b/src/backend/database/database-wrapper.ts @@ -1,161 +1,55 @@ -import { mongoRead, mongoReadWithReadCredentials, mongoUpsert } from './mongo-wrapper'; import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper'; +const dateUpToYear = () => new Date().toISOString().slice(0, 4); +const dateUpToMonth = () => + new Date().toISOString().slice(0, 7).replace("-", "_"); + export async function databaseUpsert({ contents, group }) { // No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear. // (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){ - let mongoDocName; switch (group) { case "combined": - mongoDocName = "metaforecasts"; - await mongoUpsert( - contents, - mongoDocName, - "metaforecastCollection", - "metaforecastDatabase" - ); await pgUpsert({ contents, schema: "latest", tableName: "combined" }); break; case "history": - let currentDate = new Date(); - let dateUpToYear = currentDate.toISOString().slice(0, 4); - let dateUpToMonth = currentDate - .toISOString() - .slice(0, 7) - .replace("-", "_"); - mongoDocName = `metaforecast_history_${dateUpToMonth}`; - await mongoUpsert( - contents, - mongoDocName, - "metaforecastHistory", - "metaforecastDatabase" - ); await pgUpsert({ contents, schema: "history", - tableName: `h${dateUpToYear}`, + tableName: `h${dateUpToYear()}`, }); await pgUpsert({ contents, schema: "history", - tableName: `h${dateUpToMonth}`, + tableName: `h${dateUpToMonth()}`, }); break; default: - mongoDocName = `${group}-questions`; - await mongoUpsert( - contents, - mongoDocName, - "metaforecastCollection", - "metaforecastDatabase" - ); await pgUpsert({ contents, schema: "latest", tableName: group }); } } -// databaseUpsert(contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase") + +const readWithReader = async ( + group: string, + reader: (opts: { schema: string; tableName: string }) => Promise +) => { + const schema = group === "history" ? "history" : "latest"; + const tableName = group === "history" ? `h${dateUpToMonth()}` : group; + const response = await reader({ + schema, + tableName, + }); + + console.log("Postgres: "); + console.log(response.slice(0, 2)); + console.log(""); + + return response; +}; export async function databaseRead({ group }) { - let response, mongoDocName, responseMongo, responsePg; - let currentDate = new Date(); - let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); // e.g., 2022_02 - - let displayPossibleResponses = (response1, response2) => { - console.log("Possible responses:"); - console.log("Mongo: "); - console.log(response1.slice(0, 2)); - console.log("Postgres: "); - console.log(response2.slice(0, 2)); - console.log(""); - }; - - switch (group) { - case "combined": - mongoDocName = "metaforecasts"; - responseMongo = await mongoRead( - mongoDocName, - "metaforecastCollection", - "metaforecastDatabase" - ); - responsePg = await pgRead({ schema: "latest", tableName: "combined" }); - displayPossibleResponses(responseMongo, responsePg); - break; - case "history": - mongoDocName = `metaforecast_history_${dateUpToMonth}`; - responseMongo = await mongoRead( - mongoDocName, - "metaforecastHistory", - "metaforecastDatabase" - ); - responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. - break; - default: - mongoDocName = `${group}-questions`; - responseMongo = mongoRead( - mongoDocName, - "metaforecastCollection", - "metaforecastDatabase" - ); - responsePg = await pgRead({ schema: "latest", tableName: group }); - } - - response = responsePg; // responseMongo; - return response; + return await readWithReader(group, pgRead); } -// databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase") export async function databaseReadWithReadCredentials({ group }) { - let response, mongoDocName, responseMongo, responsePg; - let currentDate = new Date(); - let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); // e.g., 2022_02 - - let displayPossibleResponses = (response1, response2) => { - console.log("Possible responses:"); - console.log("Mongo: "); - console.log(response1.slice(0, 2)); - console.log("Postgres: "); - console.log(response2.slice(0, 2)); - console.log(""); - }; - - switch (group) { - case "combined": - mongoDocName = "metaforecasts"; - responseMongo = await mongoReadWithReadCredentials( - mongoDocName, - "metaforecastCollection", - "metaforecastDatabase" - ); - responsePg = await pgReadWithReadCredentials({ - schema: "latest", - tableName: "combined", - }); - displayPossibleResponses(responseMongo, responsePg); - break; - case "history": - mongoDocName = `metaforecast_history_${dateUpToMonth}`; - responseMongo = await mongoReadWithReadCredentials( - mongoDocName, - "metaforecastHistory", - "metaforecastDatabase" - ); - responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month. - break; - default: - mongoDocName = `${group}-questions`; - responseMongo = mongoReadWithReadCredentials( - mongoDocName, - "metaforecastCollection", - "metaforecastDatabase" - ); - responsePg = await pgReadWithReadCredentials({ - schema: "latest", - tableName: group, - }); - displayPossibleResponses(responseMongo, responsePg); - } - - response = responsePg; // responseMongo; - return response; + return await readWithReader(group, pgReadWithReadCredentials); } -//= ; -// databaseReadWithReadCredentials(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase") diff --git a/src/backend/database/mongo-wrapper.ts b/src/backend/database/mongo-wrapper.ts deleted file mode 100644 index f922407..0000000 --- a/src/backend/database/mongo-wrapper.ts +++ /dev/null @@ -1,179 +0,0 @@ -import pkg from 'mongodb'; - -import { roughSizeOfObject } from '../utils/roughSize'; - -const { MongoClient } = pkg; - -export async function mongoUpsert( - contents, - documentName, - collectionName = "metaforecastCollection", - databaseName = "metaforecastDatabase" -) { - const url = process.env.MONGODB_URL; - const client = new MongoClient(url); - try { - await client.connect(); - console.log("Connected correctly to server"); - const db = client.db(databaseName); - - // Use the collection "data" - const collection = db.collection(collectionName); - - // Construct a document - let document = { - name: documentName, - timestamp: new Date().toISOString(), - contentsArray: contents, - }; - - // Create a filter - const filter = { name: documentName }; - - // Insert a single document, wait for promise so we can read it back - // const p = await collection.insertOne(metaforecastDocument); - await collection.replaceOne(filter, document, { upsert: true }); - console.log( - `Pushed document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject( - document - )} MB` - ); - - // Find one document - const myDocument = await collection.findOne(filter); - // Print to the console - console.log( - `Received document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject( - contents - )} MB` - ); - console.log("Sample: "); - console.log(JSON.stringify(myDocument.contentsArray.slice(0, 1), null, 4)); - } catch (err) { - console.log(err.stack); - } finally { - await client.close(); - } -} - -export async function mongoRead( - documentName, - collectionName = "metaforecastCollection", - databaseName = "metaforecastDatabase" -) { - const url = process.env.MONGODB_URL; - - const client = new MongoClient(url, { - useNewUrlParser: true, - useUnifiedTopology: true, - }); - - let documentContents = []; - try { - await client.connect(); - console.log(`Connected correctly to server to read ${documentName}`); - const db = client.db(databaseName); - - // Use the collection "data" - const collection = db.collection(collectionName); - - // Search options - const query = { name: documentName }; - const options = { - // sort matched documents in descending order by rating - sort: { rating: -1 }, - }; - - // Insert a single document, wait for promise so we can read it back - // const p = await collection.insertOne(metaforecastDocument); - const document = await collection.findOne(query, options); - documentContents = document.contentsArray; - } catch (err) { - console.log(err.stack); - } finally { - await client.close(); - } - console.log(documentContents.slice(0, 1)); - return documentContents; -} - -export async function mongoReadWithReadCredentials( - documentName, - collectionName = "metaforecastCollection", - databaseName = "metaforecastDatabase" -) { - const url = - "mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database. - - const client = new MongoClient(url, { - useNewUrlParser: true, - useUnifiedTopology: true, - }); - - let documentContents; - try { - await client.connect(); - // console.log(`Connected correctly to server to read ${documentName}`); - const db = client.db(databaseName); - - // Use the collection "data" - const collection = db.collection(collectionName); - - // Search options - const query = { name: documentName }; - const options = { - // sort matched documents in descending order by rating - sort: { rating: -1 }, - }; - - // Insert a single document, wait for promise so we can read it back - // const p = await collection.insertOne(metaforecastDocument); - const document = await collection.findOne(query, options); - documentContents = document.contentsArray; - } catch (err) { - console.log(err.stack); - } finally { - await client.close(); - } - // console.log(documentContents.slice(0,1)); - return documentContents; -} - -export async function mongoGetAllElements( - databaseName = "metaforecastDatabase", - collectionName = "metaforecastCollection" -) { - const url = process.env.MONGODB_URL; - const client = new MongoClient(url, { - useNewUrlParser: true, - useUnifiedTopology: true, - }); - - try { - await client.connect(); - console.log(`Connected correctly to server`); - const db = client.db(databaseName); - - // Use the collection "data" - const collection = db.collection(collectionName); - - // Search options - const query = {}; - const options = {}; - - // Insert a single document, wait for promise so we can read it back - // const p = await collection.insertOne(metaforecastDocument); - const documents = await collection.find().toArray(); - let documentNames = documents.map((document) => ({ - name: document.name, - roughSizeMBs: roughSizeOfObject(document), - })); - console.log(documentNames); - } catch (error) { - console.log(error); - } finally { - await client.close(); - } -} -//mongoGetAllElements() -//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory") diff --git a/src/backend/index.ts b/src/backend/index.ts index c6ded7e..0c1f686 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -30,7 +30,7 @@ let generateWhatToDoMessage = () => { (fun, i) => `[${i}]: Download predictions from ${fun.name}` ); let otherMessages = [ - "Merge jsons/tables into one big json/table (and push the result to a mongodb/pg database)", + "Merge tables into one big table (and push the result to a pg database)", `Rebuild algolia database ("index")`, `Update history`, `Rebuild netlify site with new data`, diff --git a/src/backend/manual/manualSendToMongo.ts b/src/backend/manual/manualSendToDb.ts similarity index 95% rename from src/backend/manual/manualSendToMongo.ts rename to src/backend/manual/manualSendToDb.ts index 19ddcfd..ff3ca2a 100644 --- a/src/backend/manual/manualSendToMongo.ts +++ b/src/backend/manual/manualSendToDb.ts @@ -6,7 +6,6 @@ import { databaseUpsert } from '../database/database-wrapper'; let pushManualFiles = ["givewellopenphil"]; // ["estimize", "givewellopenphil", "xrisk"] let suffixFiles = "-questions.json"; -let suffixMongo = "-questions"; let main = async () => { for (let file of pushManualFiles) {