Merge pull request #26 from QURIresearch/no-mongo
feat: Remove mongodb support
This commit is contained in:
GitHub
commit
7c0ad75bea
|
|
@ -71,8 +71,8 @@ Overall, the services which we use are:
|
|||
|
||||
- Algolia for search
|
||||
- Netlify for website deployment
|
||||
- Herokiu for background jobs, e.g. fetching new forecasts
|
||||
- Postgres on DigitalOcean and Mongo for databases
|
||||
- Heroku for background jobs, e.g. fetching new forecasts
|
||||
- Postgres on DigitalOcean for database
|
||||
|
||||
## Various notes
|
||||
|
||||
|
|
|
|||
|
|
@ -13,14 +13,13 @@ There's also a template configuration file in `../env.example`.
|
|||
|
||||
- `DIGITALOCEAN_POSTGRES`, of the form `postgres://username:password@domain.com:port/configvars`. (Disregard `DIGITALOCEAN_` prefix, you can use any endpoint you like).
|
||||
- `DIGITALOCEAN_POSTGRES_PUBLIC`
|
||||
- `MONGODB_URL`, a string in the format `"mongodb+srv://<username>:<password>@<mongodburl>/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"` (No longer really needed, superseded by Postgres).
|
||||
- `ALGOLIA_MASTER_API_KEY`, a string of 32 hexidecimal characters, like `19b6c2234e50c98d30668659a39e3127` (not an actual key).
|
||||
|
||||
## Platform cookies and keys
|
||||
|
||||
Most of these are just session cookies, necessary to query INFER (previously CSET Foretell), Good Judgment Open and Hypermind (Hypermind is now deprecated). You can get these cookies by creating an account in said platforms and then making and inspecting a request (e.g., by making a prediction, or browsing questions).
|
||||
|
||||
Note that not all of these cookies are needed to use all parts of the source code. For instance, to download Polymarket data, one could just interface with the Polymarket code. In particular, the code in this repository contains code to with the mongo database using read permissions, which are freely available.
|
||||
Note that not all of these cookies are needed to use all parts of the source code. For instance, to download Polymarket data, one could just interface with the Polymarket code. In particular, the code in this repository contains code to connect with the postgres database using read permissions, which are freely available.
|
||||
|
||||
- `GOODJUDGEMENTOPENCOOKIE`
|
||||
- `INFER_COOKIE`
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@
|
|||
# DIGITALOCEAN_POSTGRES=postgresql://...@localhost:5432/...?schema=public
|
||||
# POSTGRES_NO_SSL=1
|
||||
|
||||
# MONGODB_URL=mongodb://localhost:27017
|
||||
|
||||
# NEXT_PUBLIC_SITE_URL=http://localhost:3000
|
||||
|
||||
# DEBUG_MODE=off
|
||||
|
|
|
|||
191
package-lock.json
generated
191
package-lock.json
generated
|
|
@ -31,7 +31,6 @@
|
|||
"https": "^1.0.0",
|
||||
"isomorphic-fetch": "^3.0.0",
|
||||
"json2csv": "^5.0.5",
|
||||
"mongodb": "^3.6.6",
|
||||
"multiselect-react-dropdown": "^2.0.17",
|
||||
"next": "12",
|
||||
"next-plausible": "^3.1.6",
|
||||
|
|
@ -2308,15 +2307,6 @@
|
|||
"node": ">=1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bl": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
|
||||
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
|
||||
"dependencies": {
|
||||
"readable-stream": "^2.3.5",
|
||||
"safe-buffer": "^5.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
|
|
@ -2374,14 +2364,6 @@
|
|||
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
|
||||
}
|
||||
},
|
||||
"node_modules/bson": {
|
||||
"version": "1.1.6",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
|
||||
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==",
|
||||
"engines": {
|
||||
"node": ">=0.6.19"
|
||||
}
|
||||
},
|
||||
"node_modules/btoa": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
|
||||
|
|
@ -2842,7 +2824,8 @@
|
|||
"node_modules/core-util-is": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/cosmiconfig": {
|
||||
"version": "7.0.1",
|
||||
|
|
@ -3215,6 +3198,7 @@
|
|||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
|
||||
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
|
|
@ -4305,7 +4289,8 @@
|
|||
"node_modules/inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/ini": {
|
||||
"version": "1.3.8",
|
||||
|
|
@ -4596,7 +4581,8 @@
|
|||
"node_modules/isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
|
|
@ -5194,12 +5180,6 @@
|
|||
"integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/memory-pager": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
|
||||
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/merge-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
|
||||
|
|
@ -5898,44 +5878,6 @@
|
|||
"micro-memoize": "^4.0.9"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb": {
|
||||
"version": "3.6.6",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz",
|
||||
"integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==",
|
||||
"dependencies": {
|
||||
"bl": "^2.2.1",
|
||||
"bson": "^1.1.4",
|
||||
"denque": "^1.4.1",
|
||||
"optional-require": "^1.0.2",
|
||||
"safe-buffer": "^5.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"saslprep": "^1.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"aws4": {
|
||||
"optional": true
|
||||
},
|
||||
"bson-ext": {
|
||||
"optional": true
|
||||
},
|
||||
"kerberos": {
|
||||
"optional": true
|
||||
},
|
||||
"mongodb-client-encryption": {
|
||||
"optional": true
|
||||
},
|
||||
"mongodb-extjson": {
|
||||
"optional": true
|
||||
},
|
||||
"snappy": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/mri": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
|
||||
|
|
@ -32295,14 +32237,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/optional-require": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz",
|
||||
"integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg==",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/outdent": {
|
||||
"version": "0.8.0",
|
||||
"resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz",
|
||||
|
|
@ -33195,7 +33129,8 @@
|
|||
"node_modules/process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/prop-types": {
|
||||
"version": "15.8.1",
|
||||
|
|
@ -33511,6 +33446,7 @@
|
|||
"version": "2.3.7",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
|
||||
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
|
|
@ -33524,7 +33460,8 @@
|
|||
"node_modules/readable-stream/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/readdirp": {
|
||||
"version": "3.6.0",
|
||||
|
|
@ -33766,18 +33703,6 @@
|
|||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/saslprep": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
|
||||
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"sparse-bitfield": "^3.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/scheduler": {
|
||||
"version": "0.20.2",
|
||||
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz",
|
||||
|
|
@ -34025,15 +33950,6 @@
|
|||
"url": "https://github.com/sponsors/wooorm"
|
||||
}
|
||||
},
|
||||
"node_modules/sparse-bitfield": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
|
||||
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"memory-pager": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/split-on-first": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
|
||||
|
|
@ -34109,6 +34025,7 @@
|
|||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
|
|
@ -34116,7 +34033,8 @@
|
|||
"node_modules/string_decoder/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/string-width": {
|
||||
"version": "1.0.2",
|
||||
|
|
@ -36546,15 +36464,6 @@
|
|||
"psl": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"bl": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
|
||||
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
|
||||
"requires": {
|
||||
"readable-stream": "^2.3.5",
|
||||
"safe-buffer": "^5.1.1"
|
||||
}
|
||||
},
|
||||
"boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
|
|
@ -36590,11 +36499,6 @@
|
|||
"picocolors": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"bson": {
|
||||
"version": "1.1.6",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
|
||||
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg=="
|
||||
},
|
||||
"btoa": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
|
||||
|
|
@ -36906,7 +36810,8 @@
|
|||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
|
||||
"dev": true
|
||||
},
|
||||
"cosmiconfig": {
|
||||
"version": "7.0.1",
|
||||
|
|
@ -37154,7 +37059,8 @@
|
|||
"denque": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
|
||||
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ=="
|
||||
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==",
|
||||
"dev": true
|
||||
},
|
||||
"dequal": {
|
||||
"version": "2.0.2",
|
||||
|
|
@ -37906,7 +37812,8 @@
|
|||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
|
||||
"dev": true
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.3.8",
|
||||
|
|
@ -38099,7 +38006,8 @@
|
|||
"isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
|
||||
"dev": true
|
||||
},
|
||||
"isexe": {
|
||||
"version": "2.0.0",
|
||||
|
|
@ -38541,12 +38449,6 @@
|
|||
"resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
|
||||
"integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q=="
|
||||
},
|
||||
"memory-pager": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
|
||||
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
|
||||
"optional": true
|
||||
},
|
||||
"merge-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
|
||||
|
|
@ -38936,19 +38838,6 @@
|
|||
"micro-memoize": "^4.0.9"
|
||||
}
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "3.6.6",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz",
|
||||
"integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==",
|
||||
"requires": {
|
||||
"bl": "^2.2.1",
|
||||
"bson": "^1.1.4",
|
||||
"denque": "^1.4.1",
|
||||
"optional-require": "^1.0.2",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"saslprep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"mri": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
|
||||
|
|
@ -58969,11 +58858,6 @@
|
|||
"is-wsl": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"optional-require": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz",
|
||||
"integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg=="
|
||||
},
|
||||
"outdent": {
|
||||
"version": "0.8.0",
|
||||
"resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz",
|
||||
|
|
@ -59520,7 +59404,8 @@
|
|||
"process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
|
||||
"dev": true
|
||||
},
|
||||
"prop-types": {
|
||||
"version": "15.8.1",
|
||||
|
|
@ -59736,6 +59621,7 @@
|
|||
"version": "2.3.7",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
|
||||
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
|
|
@ -59749,7 +59635,8 @@
|
|||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -59896,15 +59783,6 @@
|
|||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"saslprep": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
|
||||
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"sparse-bitfield": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"scheduler": {
|
||||
"version": "0.20.2",
|
||||
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz",
|
||||
|
|
@ -60060,15 +59938,6 @@
|
|||
"resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz",
|
||||
"integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw=="
|
||||
},
|
||||
"sparse-bitfield": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
|
||||
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"memory-pager": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"split-on-first": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
|
||||
|
|
@ -60127,6 +59996,7 @@
|
|||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
},
|
||||
|
|
@ -60134,7 +60004,8 @@
|
|||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@
|
|||
"https": "^1.0.0",
|
||||
"isomorphic-fetch": "^3.0.0",
|
||||
"json2csv": "^5.0.5",
|
||||
"mongodb": "^3.6.6",
|
||||
"multiselect-react-dropdown": "^2.0.17",
|
||||
"next": "12",
|
||||
"next-plausible": "^3.1.6",
|
||||
|
|
|
|||
|
|
@ -1,161 +1,55 @@
|
|||
import { mongoRead, mongoReadWithReadCredentials, mongoUpsert } from './mongo-wrapper';
|
||||
import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper';
|
||||
|
||||
const dateUpToYear = () => new Date().toISOString().slice(0, 4);
|
||||
const dateUpToMonth = () =>
|
||||
new Date().toISOString().slice(0, 7).replace("-", "_");
|
||||
|
||||
export async function databaseUpsert({ contents, group }) {
|
||||
// No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
|
||||
// (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){
|
||||
let mongoDocName;
|
||||
switch (group) {
|
||||
case "combined":
|
||||
mongoDocName = "metaforecasts";
|
||||
await mongoUpsert(
|
||||
contents,
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
await pgUpsert({ contents, schema: "latest", tableName: "combined" });
|
||||
break;
|
||||
case "history":
|
||||
let currentDate = new Date();
|
||||
let dateUpToYear = currentDate.toISOString().slice(0, 4);
|
||||
let dateUpToMonth = currentDate
|
||||
.toISOString()
|
||||
.slice(0, 7)
|
||||
.replace("-", "_");
|
||||
mongoDocName = `metaforecast_history_${dateUpToMonth}`;
|
||||
await mongoUpsert(
|
||||
contents,
|
||||
mongoDocName,
|
||||
"metaforecastHistory",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
await pgUpsert({
|
||||
contents,
|
||||
schema: "history",
|
||||
tableName: `h${dateUpToYear}`,
|
||||
tableName: `h${dateUpToYear()}`,
|
||||
});
|
||||
await pgUpsert({
|
||||
contents,
|
||||
schema: "history",
|
||||
tableName: `h${dateUpToMonth}`,
|
||||
tableName: `h${dateUpToMonth()}`,
|
||||
});
|
||||
break;
|
||||
default:
|
||||
mongoDocName = `${group}-questions`;
|
||||
await mongoUpsert(
|
||||
contents,
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
await pgUpsert({ contents, schema: "latest", tableName: group });
|
||||
}
|
||||
}
|
||||
// databaseUpsert(contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
|
||||
|
||||
const readWithReader = async (
|
||||
group: string,
|
||||
reader: (opts: { schema: string; tableName: string }) => Promise<any>
|
||||
) => {
|
||||
const schema = group === "history" ? "history" : "latest";
|
||||
const tableName = group === "history" ? `h${dateUpToMonth()}` : group;
|
||||
const response = await reader({
|
||||
schema,
|
||||
tableName,
|
||||
});
|
||||
|
||||
console.log("Postgres: ");
|
||||
console.log(response.slice(0, 2));
|
||||
console.log("");
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
export async function databaseRead({ group }) {
|
||||
let response, mongoDocName, responseMongo, responsePg;
|
||||
let currentDate = new Date();
|
||||
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); // e.g., 2022_02
|
||||
|
||||
let displayPossibleResponses = (response1, response2) => {
|
||||
console.log("Possible responses:");
|
||||
console.log("Mongo: ");
|
||||
console.log(response1.slice(0, 2));
|
||||
console.log("Postgres: ");
|
||||
console.log(response2.slice(0, 2));
|
||||
console.log("");
|
||||
};
|
||||
|
||||
switch (group) {
|
||||
case "combined":
|
||||
mongoDocName = "metaforecasts";
|
||||
responseMongo = await mongoRead(
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
responsePg = await pgRead({ schema: "latest", tableName: "combined" });
|
||||
displayPossibleResponses(responseMongo, responsePg);
|
||||
break;
|
||||
case "history":
|
||||
mongoDocName = `metaforecast_history_${dateUpToMonth}`;
|
||||
responseMongo = await mongoRead(
|
||||
mongoDocName,
|
||||
"metaforecastHistory",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
|
||||
break;
|
||||
default:
|
||||
mongoDocName = `${group}-questions`;
|
||||
responseMongo = mongoRead(
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
responsePg = await pgRead({ schema: "latest", tableName: group });
|
||||
}
|
||||
|
||||
response = responsePg; // responseMongo;
|
||||
return response;
|
||||
return await readWithReader(group, pgRead);
|
||||
}
|
||||
// databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
|
||||
|
||||
export async function databaseReadWithReadCredentials({ group }) {
|
||||
let response, mongoDocName, responseMongo, responsePg;
|
||||
let currentDate = new Date();
|
||||
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); // e.g., 2022_02
|
||||
|
||||
let displayPossibleResponses = (response1, response2) => {
|
||||
console.log("Possible responses:");
|
||||
console.log("Mongo: ");
|
||||
console.log(response1.slice(0, 2));
|
||||
console.log("Postgres: ");
|
||||
console.log(response2.slice(0, 2));
|
||||
console.log("");
|
||||
};
|
||||
|
||||
switch (group) {
|
||||
case "combined":
|
||||
mongoDocName = "metaforecasts";
|
||||
responseMongo = await mongoReadWithReadCredentials(
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
responsePg = await pgReadWithReadCredentials({
|
||||
schema: "latest",
|
||||
tableName: "combined",
|
||||
});
|
||||
displayPossibleResponses(responseMongo, responsePg);
|
||||
break;
|
||||
case "history":
|
||||
mongoDocName = `metaforecast_history_${dateUpToMonth}`;
|
||||
responseMongo = await mongoReadWithReadCredentials(
|
||||
mongoDocName,
|
||||
"metaforecastHistory",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
|
||||
break;
|
||||
default:
|
||||
mongoDocName = `${group}-questions`;
|
||||
responseMongo = mongoReadWithReadCredentials(
|
||||
mongoDocName,
|
||||
"metaforecastCollection",
|
||||
"metaforecastDatabase"
|
||||
);
|
||||
responsePg = await pgReadWithReadCredentials({
|
||||
schema: "latest",
|
||||
tableName: group,
|
||||
});
|
||||
displayPossibleResponses(responseMongo, responsePg);
|
||||
}
|
||||
|
||||
response = responsePg; // responseMongo;
|
||||
return response;
|
||||
return await readWithReader(group, pgReadWithReadCredentials);
|
||||
}
|
||||
//= ;
|
||||
// databaseReadWithReadCredentials(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
import pkg from 'mongodb';
|
||||
|
||||
import { roughSizeOfObject } from '../utils/roughSize';
|
||||
|
||||
const { MongoClient } = pkg;
|
||||
|
||||
export async function mongoUpsert(
|
||||
contents,
|
||||
documentName,
|
||||
collectionName = "metaforecastCollection",
|
||||
databaseName = "metaforecastDatabase"
|
||||
) {
|
||||
const url = process.env.MONGODB_URL;
|
||||
const client = new MongoClient(url);
|
||||
try {
|
||||
await client.connect();
|
||||
console.log("Connected correctly to server");
|
||||
const db = client.db(databaseName);
|
||||
|
||||
// Use the collection "data"
|
||||
const collection = db.collection(collectionName);
|
||||
|
||||
// Construct a document
|
||||
let document = {
|
||||
name: documentName,
|
||||
timestamp: new Date().toISOString(),
|
||||
contentsArray: contents,
|
||||
};
|
||||
|
||||
// Create a filter
|
||||
const filter = { name: documentName };
|
||||
|
||||
// Insert a single document, wait for promise so we can read it back
|
||||
// const p = await collection.insertOne(metaforecastDocument);
|
||||
await collection.replaceOne(filter, document, { upsert: true });
|
||||
console.log(
|
||||
`Pushed document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(
|
||||
document
|
||||
)} MB`
|
||||
);
|
||||
|
||||
// Find one document
|
||||
const myDocument = await collection.findOne(filter);
|
||||
// Print to the console
|
||||
console.log(
|
||||
`Received document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(
|
||||
contents
|
||||
)} MB`
|
||||
);
|
||||
console.log("Sample: ");
|
||||
console.log(JSON.stringify(myDocument.contentsArray.slice(0, 1), null, 4));
|
||||
} catch (err) {
|
||||
console.log(err.stack);
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function mongoRead(
|
||||
documentName,
|
||||
collectionName = "metaforecastCollection",
|
||||
databaseName = "metaforecastDatabase"
|
||||
) {
|
||||
const url = process.env.MONGODB_URL;
|
||||
|
||||
const client = new MongoClient(url, {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true,
|
||||
});
|
||||
|
||||
let documentContents = [];
|
||||
try {
|
||||
await client.connect();
|
||||
console.log(`Connected correctly to server to read ${documentName}`);
|
||||
const db = client.db(databaseName);
|
||||
|
||||
// Use the collection "data"
|
||||
const collection = db.collection(collectionName);
|
||||
|
||||
// Search options
|
||||
const query = { name: documentName };
|
||||
const options = {
|
||||
// sort matched documents in descending order by rating
|
||||
sort: { rating: -1 },
|
||||
};
|
||||
|
||||
// Insert a single document, wait for promise so we can read it back
|
||||
// const p = await collection.insertOne(metaforecastDocument);
|
||||
const document = await collection.findOne(query, options);
|
||||
documentContents = document.contentsArray;
|
||||
} catch (err) {
|
||||
console.log(err.stack);
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
console.log(documentContents.slice(0, 1));
|
||||
return documentContents;
|
||||
}
|
||||
|
||||
export async function mongoReadWithReadCredentials(
|
||||
documentName,
|
||||
collectionName = "metaforecastCollection",
|
||||
databaseName = "metaforecastDatabase"
|
||||
) {
|
||||
const url =
|
||||
"mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database.
|
||||
|
||||
const client = new MongoClient(url, {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true,
|
||||
});
|
||||
|
||||
let documentContents;
|
||||
try {
|
||||
await client.connect();
|
||||
// console.log(`Connected correctly to server to read ${documentName}`);
|
||||
const db = client.db(databaseName);
|
||||
|
||||
// Use the collection "data"
|
||||
const collection = db.collection(collectionName);
|
||||
|
||||
// Search options
|
||||
const query = { name: documentName };
|
||||
const options = {
|
||||
// sort matched documents in descending order by rating
|
||||
sort: { rating: -1 },
|
||||
};
|
||||
|
||||
// Insert a single document, wait for promise so we can read it back
|
||||
// const p = await collection.insertOne(metaforecastDocument);
|
||||
const document = await collection.findOne(query, options);
|
||||
documentContents = document.contentsArray;
|
||||
} catch (err) {
|
||||
console.log(err.stack);
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
// console.log(documentContents.slice(0,1));
|
||||
return documentContents;
|
||||
}
|
||||
|
||||
export async function mongoGetAllElements(
|
||||
databaseName = "metaforecastDatabase",
|
||||
collectionName = "metaforecastCollection"
|
||||
) {
|
||||
const url = process.env.MONGODB_URL;
|
||||
const client = new MongoClient(url, {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true,
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
console.log(`Connected correctly to server`);
|
||||
const db = client.db(databaseName);
|
||||
|
||||
// Use the collection "data"
|
||||
const collection = db.collection(collectionName);
|
||||
|
||||
// Search options
|
||||
const query = {};
|
||||
const options = {};
|
||||
|
||||
// Insert a single document, wait for promise so we can read it back
|
||||
// const p = await collection.insertOne(metaforecastDocument);
|
||||
const documents = await collection.find().toArray();
|
||||
let documentNames = documents.map((document) => ({
|
||||
name: document.name,
|
||||
roughSizeMBs: roughSizeOfObject(document),
|
||||
}));
|
||||
console.log(documentNames);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
}
|
||||
//mongoGetAllElements()
|
||||
//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory")
|
||||
|
|
@ -30,7 +30,7 @@ let generateWhatToDoMessage = () => {
|
|||
(fun, i) => `[${i}]: Download predictions from ${fun.name}`
|
||||
);
|
||||
let otherMessages = [
|
||||
"Merge jsons/tables into one big json/table (and push the result to a mongodb/pg database)",
|
||||
"Merge tables into one big table (and push the result to a pg database)",
|
||||
`Rebuild algolia database ("index")`,
|
||||
`Update history`,
|
||||
`Rebuild netlify site with new data`,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import { databaseUpsert } from '../database/database-wrapper';
|
|||
|
||||
let pushManualFiles = ["givewellopenphil"]; // ["estimize", "givewellopenphil", "xrisk"]
|
||||
let suffixFiles = "-questions.json";
|
||||
let suffixMongo = "-questions";
|
||||
|
||||
let main = async () => {
|
||||
for (let file of pushManualFiles) {
|
||||
Loading…
Reference in New Issue
Block a user