quantified.uncertainty/metaforecast
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: remove mongodb support

Browse Source 
also:
- refactor and significantly simplify database-wrapper code
- rename manualSendToMongo to manualSendToDb
This commit is contained in:
Vyacheslav Matyukhin 2022-03-28 02:09:34 +03:00
parent edeefdb3f5
commit a1c3f03d7d
No known key found for this signature in database
GPG Key ID: 3D2A774C5489F96C
9 changed files with 60 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -63,7 +63,7 @@ Overall, the services which we use are:
- Algolia for search - Algolia for search
- Netlify for website deployment - Netlify for website deployment
- DigitalOcean for background jobs, e.g. fetching new forecasts - DigitalOcean for background jobs, e.g. fetching new forecasts
- Postgres on DigitalOcean and Mongo for databases - Postgres on DigitalOcean for database
## Various notes ## Various notes

3
docs/configuration.md
View File

@ -13,14 +13,13 @@ There's also a template configuration file in `../env.example`.
- `DIGITALOCEAN_POSTGRES`, of the form `postgres://username:password@domain.com:port/configvars`. (Disregard `DIGITALOCEAN_` prefix, you can use any endpoint you like). - `DIGITALOCEAN_POSTGRES`, of the form `postgres://username:password@domain.com:port/configvars`. (Disregard `DIGITALOCEAN_` prefix, you can use any endpoint you like).
- `DIGITALOCEAN_POSTGRES_PUBLIC` - `DIGITALOCEAN_POSTGRES_PUBLIC`
- `MONGODB_URL`, a string in the format `"mongodb+srv://<username>:<password>@<mongodburl>/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"` (No longer really needed, superseded by Postgres).
- `ALGOLIA_MASTER_API_KEY`, a string of 32 hexidecimal characters, like `19b6c2234e50c98d30668659a39e3127` (not an actual key). - `ALGOLIA_MASTER_API_KEY`, a string of 32 hexidecimal characters, like `19b6c2234e50c98d30668659a39e3127` (not an actual key).
## Platform cookies and keys ## Platform cookies and keys
Most of these are just session cookies, necessary to query INFER (previously CSET Foretell), Good Judgment Open and Hypermind (Hypermind is now deprecated). You can get these cookies by creating an account in said platforms and then making and inspecting a request (e.g., by making a prediction, or browsing questions). Most of these are just session cookies, necessary to query INFER (previously CSET Foretell), Good Judgment Open and Hypermind (Hypermind is now deprecated). You can get these cookies by creating an account in said platforms and then making and inspecting a request (e.g., by making a prediction, or browsing questions).
Note that not all of these cookies are needed to use all parts of the source code. For instance, to download Polymarket data, one could just interface with the Polymarket code. In particular, the code in this repository contains code to with the mongo database using read permissions, which are freely available. Note that not all of these cookies are needed to use all parts of the source code. For instance, to download Polymarket data, one could just interface with the Polymarket code. In particular, the code in this repository contains code to connect with the postgres database using read permissions, which are freely available.
- `GOODJUDGEMENTOPENCOOKIE` - `GOODJUDGEMENTOPENCOOKIE`
- `INFER_COOKIE` - `INFER_COOKIE`

2
env.example
View File

@ -6,8 +6,6 @@
# DIGITALOCEAN_POSTGRES=postgresql://...@localhost:5432/...?schema=public # DIGITALOCEAN_POSTGRES=postgresql://...@localhost:5432/...?schema=public
# POSTGRES_NO_SSL=1 # POSTGRES_NO_SSL=1
# MONGODB_URL=mongodb://localhost:27017
# NEXT_PUBLIC_SITE_URL=http://localhost:3000 # NEXT_PUBLIC_SITE_URL=http://localhost:3000
# DEBUG_MODE=off # DEBUG_MODE=off

191
package-lock.json generated
View File

@ -31,7 +31,6 @@
"https": "^1.0.0", "https": "^1.0.0",
"isomorphic-fetch": "^3.0.0", "isomorphic-fetch": "^3.0.0",
"json2csv": "^5.0.5", "json2csv": "^5.0.5",
"mongodb": "^3.6.6",
"multiselect-react-dropdown": "^2.0.17", "multiselect-react-dropdown": "^2.0.17",
"next": "12", "next": "12",
"nprogress": "^0.2.0", "nprogress": "^0.2.0",
@ -2307,15 +2306,6 @@
"node": ">=1.0.0" "node": ">=1.0.0"
} }
}, },
"node_modules/bl": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
"dependencies": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"node_modules/boolbase": { "node_modules/boolbase": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@ -2373,14 +2363,6 @@
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
} }
}, },
"node_modules/bson": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==",
"engines": {
"node": ">=0.6.19"
}
},
"node_modules/btoa": { "node_modules/btoa": {
"version": "1.2.1", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
@ -2841,7 +2823,8 @@
"node_modules/core-util-is": { "node_modules/core-util-is": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
}, },
"node_modules/cosmiconfig": { "node_modules/cosmiconfig": {
"version": "7.0.1", "version": "7.0.1",
@ -3214,6 +3197,7 @@
"version": "1.5.0", "version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==",
"dev": true,
"engines": { "engines": {
"node": ">=0.10" "node": ">=0.10"
} }
@ -4304,7 +4288,8 @@
"node_modules/inherits": { "node_modules/inherits": {
"version": "2.0.4", "version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"dev": true
}, },
"node_modules/ini": { "node_modules/ini": {
"version": "1.3.8", "version": "1.3.8",
@ -4595,7 +4580,8 @@
"node_modules/isarray": { "node_modules/isarray": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
}, },
"node_modules/isexe": { "node_modules/isexe": {
"version": "2.0.0", "version": "2.0.0",
@ -5193,12 +5179,6 @@
"integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==", "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/memory-pager": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
"optional": true
},
"node_modules/merge-stream": { "node_modules/merge-stream": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@ -5897,44 +5877,6 @@
"micro-memoize": "^4.0.9" "micro-memoize": "^4.0.9"
} }
}, },
"node_modules/mongodb": {
"version": "3.6.6",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz",
"integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==",
"dependencies": {
"bl": "^2.2.1",
"bson": "^1.1.4",
"denque": "^1.4.1",
"optional-require": "^1.0.2",
"safe-buffer": "^5.1.2"
},
"engines": {
"node": ">=4"
},
"optionalDependencies": {
"saslprep": "^1.0.0"
},
"peerDependenciesMeta": {
"aws4": {
"optional": true
},
"bson-ext": {
"optional": true
},
"kerberos": {
"optional": true
},
"mongodb-client-encryption": {
"optional": true
},
"mongodb-extjson": {
"optional": true
},
"snappy": {
"optional": true
}
}
},
"node_modules/mri": { "node_modules/mri": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@ -32281,14 +32223,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/optional-require": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz",
"integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg==",
"engines": {
"node": ">=4"
}
},
"node_modules/outdent": { "node_modules/outdent": {
"version": "0.8.0", "version": "0.8.0",
"resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz", "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz",
@ -33181,7 +33115,8 @@
"node_modules/process-nextick-args": { "node_modules/process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
}, },
"node_modules/prop-types": { "node_modules/prop-types": {
"version": "15.8.1", "version": "15.8.1",
@ -33497,6 +33432,7 @@
"version": "2.3.7", "version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dev": true,
"dependencies": { "dependencies": {
"core-util-is": "~1.0.0", "core-util-is": "~1.0.0",
"inherits": "~2.0.3", "inherits": "~2.0.3",
@ -33510,7 +33446,8 @@
"node_modules/readable-stream/node_modules/safe-buffer": { "node_modules/readable-stream/node_modules/safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
}, },
"node_modules/readdirp": { "node_modules/readdirp": {
"version": "3.6.0", "version": "3.6.0",
@ -33752,18 +33689,6 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/saslprep": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
"optional": true,
"dependencies": {
"sparse-bitfield": "^3.0.3"
},
"engines": {
"node": ">=6"
}
},
"node_modules/scheduler": { "node_modules/scheduler": {
"version": "0.20.2", "version": "0.20.2",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz",
@ -34011,15 +33936,6 @@
"url": "https://github.com/sponsors/wooorm" "url": "https://github.com/sponsors/wooorm"
} }
}, },
"node_modules/sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
"optional": true,
"dependencies": {
"memory-pager": "^1.0.2"
}
},
"node_modules/split-on-first": { "node_modules/split-on-first": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
@ -34095,6 +34011,7 @@
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"dependencies": { "dependencies": {
"safe-buffer": "~5.1.0" "safe-buffer": "~5.1.0"
} }
@ -34102,7 +34019,8 @@
"node_modules/string_decoder/node_modules/safe-buffer": { "node_modules/string_decoder/node_modules/safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
}, },
"node_modules/string-width": { "node_modules/string-width": {
"version": "1.0.2", "version": "1.0.2",
@ -36532,15 +36450,6 @@
"psl": "^1.1.7" "psl": "^1.1.7"
} }
}, },
"bl": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"boolbase": { "boolbase": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@ -36576,11 +36485,6 @@
"picocolors": "^1.0.0" "picocolors": "^1.0.0"
} }
}, },
"bson": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz",
"integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg=="
},
"btoa": { "btoa": {
"version": "1.2.1", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
@ -36892,7 +36796,8 @@
"core-util-is": { "core-util-is": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
}, },
"cosmiconfig": { "cosmiconfig": {
"version": "7.0.1", "version": "7.0.1",
@ -37140,7 +37045,8 @@
"denque": { "denque": {
"version": "1.5.0", "version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==",
"dev": true
}, },
"dequal": { "dequal": {
"version": "2.0.2", "version": "2.0.2",
@ -37892,7 +37798,8 @@
"inherits": { "inherits": {
"version": "2.0.4", "version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"dev": true
}, },
"ini": { "ini": {
"version": "1.3.8", "version": "1.3.8",
@ -38085,7 +37992,8 @@
"isarray": { "isarray": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
}, },
"isexe": { "isexe": {
"version": "2.0.0", "version": "2.0.0",
@ -38527,12 +38435,6 @@
"resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
"integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==" "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q=="
}, },
"memory-pager": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
"optional": true
},
"merge-stream": { "merge-stream": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@ -38922,19 +38824,6 @@
"micro-memoize": "^4.0.9" "micro-memoize": "^4.0.9"
} }
}, },
"mongodb": {
"version": "3.6.6",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz",
"integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==",
"requires": {
"bl": "^2.2.1",
"bson": "^1.1.4",
"denque": "^1.4.1",
"optional-require": "^1.0.2",
"safe-buffer": "^5.1.2",
"saslprep": "^1.0.0"
}
},
"mri": { "mri": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@ -58949,11 +58838,6 @@
"is-wsl": "^2.1.1" "is-wsl": "^2.1.1"
} }
}, },
"optional-require": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz",
"integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg=="
},
"outdent": { "outdent": {
"version": "0.8.0", "version": "0.8.0",
"resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz", "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz",
@ -59500,7 +59384,8 @@
"process-nextick-args": { "process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
}, },
"prop-types": { "prop-types": {
"version": "15.8.1", "version": "15.8.1",
@ -59716,6 +59601,7 @@
"version": "2.3.7", "version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dev": true,
"requires": { "requires": {
"core-util-is": "~1.0.0", "core-util-is": "~1.0.0",
"inherits": "~2.0.3", "inherits": "~2.0.3",
@ -59729,7 +59615,8 @@
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
} }
} }
}, },
@ -59876,15 +59763,6 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
}, },
"saslprep": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
"optional": true,
"requires": {
"sparse-bitfield": "^3.0.3"
}
},
"scheduler": { "scheduler": {
"version": "0.20.2", "version": "0.20.2",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz",
@ -60040,15 +59918,6 @@
"resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz",
"integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw=="
}, },
"sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
"optional": true,
"requires": {
"memory-pager": "^1.0.2"
}
},
"split-on-first": { "split-on-first": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
@ -60107,6 +59976,7 @@
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"requires": { "requires": {
"safe-buffer": "~5.1.0" "safe-buffer": "~5.1.0"
}, },
@ -60114,7 +59984,8 @@
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
} }
} }
}, },

1
package.json
View File

@ -49,7 +49,6 @@
"https": "^1.0.0", "https": "^1.0.0",
"isomorphic-fetch": "^3.0.0", "isomorphic-fetch": "^3.0.0",
"json2csv": "^5.0.5", "json2csv": "^5.0.5",
"mongodb": "^3.6.6",
"multiselect-react-dropdown": "^2.0.17", "multiselect-react-dropdown": "^2.0.17",
"next": "12", "next": "12",
"nprogress": "^0.2.0", "nprogress": "^0.2.0",

158
src/backend/database/database-wrapper.ts
View File

@ -1,161 +1,55 @@
import { mongoRead, mongoReadWithReadCredentials, mongoUpsert } from './mongo-wrapper';
import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper'; import { pgRead, pgReadWithReadCredentials, pgUpsert } from './pg-wrapper';
const dateUpToYear = () => new Date().toISOString().slice(0, 4);
const dateUpToMonth = () =>
new Date().toISOString().slice(0, 7).replace("-", "_");
export async function databaseUpsert({ contents, group }) { export async function databaseUpsert({ contents, group }) {
// No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear. // No, this should be more rational, ({contents, group, schema})? Or should this be managed by this layer? Unclear.
// (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){ // (contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase"){
let mongoDocName;
switch (group) { switch (group) {
case "combined": case "combined":
mongoDocName = "metaforecasts";
await mongoUpsert(
contents,
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"
);
await pgUpsert({ contents, schema: "latest", tableName: "combined" }); await pgUpsert({ contents, schema: "latest", tableName: "combined" });
break; break;
case "history": case "history":
let currentDate = new Date();
let dateUpToYear = currentDate.toISOString().slice(0, 4);
let dateUpToMonth = currentDate
.toISOString()
.slice(0, 7)
.replace("-", "_");
mongoDocName = `metaforecast_history_${dateUpToMonth}`;
await mongoUpsert(
contents,
mongoDocName,
"metaforecastHistory",
"metaforecastDatabase"
);
await pgUpsert({ await pgUpsert({
contents, contents,
schema: "history", schema: "history",
tableName: `h${dateUpToYear}`, tableName: `h${dateUpToYear()}`,
}); });
await pgUpsert({ await pgUpsert({
contents, contents,
schema: "history", schema: "history",
tableName: `h${dateUpToMonth}`, tableName: `h${dateUpToMonth()}`,
}); });
break; break;
default: default:
mongoDocName = `${group}-questions`;
await mongoUpsert(
contents,
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"
);
await pgUpsert({ contents, schema: "latest", tableName: group }); await pgUpsert({ contents, schema: "latest", tableName: group });
} }
} }
// databaseUpsert(contents, documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
const readWithReader = async (
group: string,
reader: (opts: { schema: string; tableName: string }) => Promise<any>
) => {
const schema = group === "history" ? "history" : "latest";
const tableName = group === "history" ? `h${dateUpToMonth()}` : group;
const response = await reader({
schema,
tableName,
});
console.log("Postgres: ");
console.log(response.slice(0, 2));
console.log("");
return response;
};
export async function databaseRead({ group }) { export async function databaseRead({ group }) {
let response, mongoDocName, responseMongo, responsePg; return await readWithReader(group, pgRead);
let currentDate = new Date();
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); // e.g., 2022_02
let displayPossibleResponses = (response1, response2) => {
console.log("Possible responses:");
console.log("Mongo: ");
console.log(response1.slice(0, 2));
console.log("Postgres: ");
console.log(response2.slice(0, 2));
console.log("");
};
switch (group) {
case "combined":
mongoDocName = "metaforecasts";
responseMongo = await mongoRead(
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"
);
responsePg = await pgRead({ schema: "latest", tableName: "combined" });
displayPossibleResponses(responseMongo, responsePg);
break;
case "history":
mongoDocName = `metaforecast_history_${dateUpToMonth}`;
responseMongo = await mongoRead(
mongoDocName,
"metaforecastHistory",
"metaforecastDatabase"
);
responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
break;
default:
mongoDocName = `${group}-questions`;
responseMongo = mongoRead(
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"
);
responsePg = await pgRead({ schema: "latest", tableName: group });
} }
response = responsePg; // responseMongo;
return response;
}
// databaseRead(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")
export async function databaseReadWithReadCredentials({ group }) { export async function databaseReadWithReadCredentials({ group }) {
let response, mongoDocName, responseMongo, responsePg; return await readWithReader(group, pgReadWithReadCredentials);
let currentDate = new Date();
let dateUpToMonth = currentDate.toISOString().slice(0, 7).replace("-", "_"); // e.g., 2022_02
let displayPossibleResponses = (response1, response2) => {
console.log("Possible responses:");
console.log("Mongo: ");
console.log(response1.slice(0, 2));
console.log("Postgres: ");
console.log(response2.slice(0, 2));
console.log("");
};
switch (group) {
case "combined":
mongoDocName = "metaforecasts";
responseMongo = await mongoReadWithReadCredentials(
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"
);
responsePg = await pgReadWithReadCredentials({
schema: "latest",
tableName: "combined",
});
displayPossibleResponses(responseMongo, responsePg);
break;
case "history":
mongoDocName = `metaforecast_history_${dateUpToMonth}`;
responseMongo = await mongoReadWithReadCredentials(
mongoDocName,
"metaforecastHistory",
"metaforecastDatabase"
);
responsePg = responseMongo; // await pgReadWithReadCredentials({ schema: "history", tableName: "combined" }) // fix, make dependent on month.
break;
default:
mongoDocName = `${group}-questions`;
responseMongo = mongoReadWithReadCredentials(
mongoDocName,
"metaforecastCollection",
"metaforecastDatabase"
);
responsePg = await pgReadWithReadCredentials({
schema: "latest",
tableName: group,
});
displayPossibleResponses(responseMongo, responsePg);
} }
response = responsePg; // responseMongo;
return response;
}
//= ;
// databaseReadWithReadCredentials(documentName, collectionName = "metaforecastCollection", databaseName = "metaforecastDatabase")

179
src/backend/database/mongo-wrapper.ts
View File

@ -1,179 +0,0 @@
import pkg from 'mongodb';
import { roughSizeOfObject } from '../utils/roughSize';
const { MongoClient } = pkg;
export async function mongoUpsert(
contents,
documentName,
collectionName = "metaforecastCollection",
databaseName = "metaforecastDatabase"
) {
const url = process.env.MONGODB_URL;
const client = new MongoClient(url);
try {
await client.connect();
console.log("Connected correctly to server");
const db = client.db(databaseName);
// Use the collection "data"
const collection = db.collection(collectionName);
// Construct a document
let document = {
name: documentName,
timestamp: new Date().toISOString(),
contentsArray: contents,
};
// Create a filter
const filter = { name: documentName };
// Insert a single document, wait for promise so we can read it back
// const p = await collection.insertOne(metaforecastDocument);
await collection.replaceOne(filter, document, { upsert: true });
console.log(
`Pushed document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(
document
)} MB`
);
// Find one document
const myDocument = await collection.findOne(filter);
// Print to the console
console.log(
`Received document ${documentName} in collection ${collectionName} in database ${databaseName} with approximate size ${roughSizeOfObject(
contents
)} MB`
);
console.log("Sample: ");
console.log(JSON.stringify(myDocument.contentsArray.slice(0, 1), null, 4));
} catch (err) {
console.log(err.stack);
} finally {
await client.close();
}
}
export async function mongoRead(
documentName,
collectionName = "metaforecastCollection",
databaseName = "metaforecastDatabase"
) {
const url = process.env.MONGODB_URL;
const client = new MongoClient(url, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
let documentContents = [];
try {
await client.connect();
console.log(`Connected correctly to server to read ${documentName}`);
const db = client.db(databaseName);
// Use the collection "data"
const collection = db.collection(collectionName);
// Search options
const query = { name: documentName };
const options = {
// sort matched documents in descending order by rating
sort: { rating: -1 },
};
// Insert a single document, wait for promise so we can read it back
// const p = await collection.insertOne(metaforecastDocument);
const document = await collection.findOne(query, options);
documentContents = document.contentsArray;
} catch (err) {
console.log(err.stack);
} finally {
await client.close();
}
console.log(documentContents.slice(0, 1));
return documentContents;
}
export async function mongoReadWithReadCredentials(
documentName,
collectionName = "metaforecastCollection",
databaseName = "metaforecastDatabase"
) {
const url =
"mongodb+srv://metaforecast-frontend:hJr5c9kDhbutBtF1@metaforecastdatabaseclu.wgk8a.mongodb.net/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"; // This user only has read permissions, so I'm not excessively worried, and would even be pleased, if someone read this and decided to do something cool with the database.
const client = new MongoClient(url, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
let documentContents;
try {
await client.connect();
// console.log(`Connected correctly to server to read ${documentName}`);
const db = client.db(databaseName);
// Use the collection "data"
const collection = db.collection(collectionName);
// Search options
const query = { name: documentName };
const options = {
// sort matched documents in descending order by rating
sort: { rating: -1 },
};
// Insert a single document, wait for promise so we can read it back
// const p = await collection.insertOne(metaforecastDocument);
const document = await collection.findOne(query, options);
documentContents = document.contentsArray;
} catch (err) {
console.log(err.stack);
} finally {
await client.close();
}
// console.log(documentContents.slice(0,1));
return documentContents;
}
export async function mongoGetAllElements(
databaseName = "metaforecastDatabase",
collectionName = "metaforecastCollection"
) {
const url = process.env.MONGODB_URL;
const client = new MongoClient(url, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
try {
await client.connect();
console.log(`Connected correctly to server`);
const db = client.db(databaseName);
// Use the collection "data"
const collection = db.collection(collectionName);
// Search options
const query = {};
const options = {};
// Insert a single document, wait for promise so we can read it back
// const p = await collection.insertOne(metaforecastDocument);
const documents = await collection.find().toArray();
let documentNames = documents.map((document) => ({
name: document.name,
roughSizeMBs: roughSizeOfObject(document),
}));
console.log(documentNames);
} catch (error) {
console.log(error);
} finally {
await client.close();
}
}
//mongoGetAllElements()
//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory")

2
src/backend/index.ts
View File

@ -30,7 +30,7 @@ let generateWhatToDoMessage = () => {
(fun, i) => `[${i}]: Download predictions from ${fun.name}` (fun, i) => `[${i}]: Download predictions from ${fun.name}`
); );
let otherMessages = [ let otherMessages = [
"Merge jsons/tables into one big json/table (and push the result to a mongodb/pg database)", "Merge tables into one big table (and push the result to a pg database)",
`Rebuild algolia database ("index")`, `Rebuild algolia database ("index")`,
`Update history`, `Update history`,
`Rebuild netlify site with new data`, `Rebuild netlify site with new data`,

1
src/backend/manual/manualSendToMongo.ts → src/backend/manual/manualSendToDb.ts
View File

@ -6,7 +6,6 @@ import { databaseUpsert } from '../database/database-wrapper';
let pushManualFiles = ["givewellopenphil"]; // ["estimize", "givewellopenphil", "xrisk"] let pushManualFiles = ["givewellopenphil"]; // ["estimize", "givewellopenphil", "xrisk"]
let suffixFiles = "-questions.json"; let suffixFiles = "-questions.json";
let suffixMongo = "-questions";
let main = async () => { let main = async () => {
for (let file of pushManualFiles) { for (let file of pushManualFiles) {