feat: Added Algolia to search

Also some light cleanup necessary to Algolia, e.g., numforecasts is now guaranteed to be a Number() (or a NaN, in case something goes very wrong)

.gitignore

@@ -1,7 +1,14 @@
+## Node modules
 node_modules/
 
+## Security
 **/betfaircertificates/
 **/privatekeys.json
 
+## Personal notes
 possiblenewsources.md
-NotesHeroku.md
+NotesHeroku.md
+AlgoliaNotes.md
+
+## Build artifacts
+done.txt

node_modules/.package-lock.json

@@ -4,6 +4,121 @@
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
+    "node_modules/@algolia/cache-browser-local-storage": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz",
+      "integrity": "sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==",
+      "dependencies": {
+        "@algolia/cache-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/cache-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.10.3.tgz",
+      "integrity": "sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw=="
+    },
+    "node_modules/@algolia/cache-in-memory": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz",
+      "integrity": "sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==",
+      "dependencies": {
+        "@algolia/cache-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-account": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.10.3.tgz",
+      "integrity": "sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-analytics": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.10.3.tgz",
+      "integrity": "sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.10.3.tgz",
+      "integrity": "sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==",
+      "dependencies": {
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-personalization": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.10.3.tgz",
+      "integrity": "sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-search": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.10.3.tgz",
+      "integrity": "sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/logger-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.10.3.tgz",
+      "integrity": "sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg=="
+    },
+    "node_modules/@algolia/logger-console": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.10.3.tgz",
+      "integrity": "sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==",
+      "dependencies": {
+        "@algolia/logger-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/requester-browser-xhr": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz",
+      "integrity": "sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==",
+      "dependencies": {
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/requester-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.10.3.tgz",
+      "integrity": "sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ=="
+    },
+    "node_modules/@algolia/requester-node-http": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz",
+      "integrity": "sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==",
+      "dependencies": {
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/transporter": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.10.3.tgz",
+      "integrity": "sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==",
+      "dependencies": {
+        "@algolia/cache-common": "4.10.3",
+        "@algolia/logger-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
     "node_modules/@sindresorhus/is": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
@@ -60,6 +175,27 @@
         "@types/node": "*"
       }
     },
+    "node_modules/algoliasearch": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
+      "integrity": "sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==",
+      "dependencies": {
+        "@algolia/cache-browser-local-storage": "4.10.3",
+        "@algolia/cache-common": "4.10.3",
+        "@algolia/cache-in-memory": "4.10.3",
+        "@algolia/client-account": "4.10.3",
+        "@algolia/client-analytics": "4.10.3",
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-personalization": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/logger-common": "4.10.3",
+        "@algolia/logger-console": "4.10.3",
+        "@algolia/requester-browser-xhr": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/requester-node-http": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
     "node_modules/axios": {
       "version": "0.21.1",
       "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",

package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.3.0",
       "license": "MIT",
       "dependencies": {
+        "algoliasearch": "^4.10.3",
         "axios": "^0.21.1",
         "fs": "^0.0.1-security",
         "https": "^1.0.0",
@@ -21,6 +22,121 @@
         "textversionjs": "^1.1.3"
       }
     },
+    "node_modules/@algolia/cache-browser-local-storage": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz",
+      "integrity": "sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==",
+      "dependencies": {
+        "@algolia/cache-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/cache-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.10.3.tgz",
+      "integrity": "sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw=="
+    },
+    "node_modules/@algolia/cache-in-memory": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz",
+      "integrity": "sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==",
+      "dependencies": {
+        "@algolia/cache-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-account": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.10.3.tgz",
+      "integrity": "sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-analytics": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.10.3.tgz",
+      "integrity": "sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.10.3.tgz",
+      "integrity": "sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==",
+      "dependencies": {
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-personalization": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.10.3.tgz",
+      "integrity": "sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/client-search": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.10.3.tgz",
+      "integrity": "sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==",
+      "dependencies": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/logger-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.10.3.tgz",
+      "integrity": "sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg=="
+    },
+    "node_modules/@algolia/logger-console": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.10.3.tgz",
+      "integrity": "sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==",
+      "dependencies": {
+        "@algolia/logger-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/requester-browser-xhr": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz",
+      "integrity": "sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==",
+      "dependencies": {
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/requester-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.10.3.tgz",
+      "integrity": "sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ=="
+    },
+    "node_modules/@algolia/requester-node-http": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz",
+      "integrity": "sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==",
+      "dependencies": {
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
+    "node_modules/@algolia/transporter": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.10.3.tgz",
+      "integrity": "sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==",
+      "dependencies": {
+        "@algolia/cache-common": "4.10.3",
+        "@algolia/logger-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
     "node_modules/@sindresorhus/is": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
@@ -77,6 +193,27 @@
         "@types/node": "*"
       }
     },
+    "node_modules/algoliasearch": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
+      "integrity": "sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==",
+      "dependencies": {
+        "@algolia/cache-browser-local-storage": "4.10.3",
+        "@algolia/cache-common": "4.10.3",
+        "@algolia/cache-in-memory": "4.10.3",
+        "@algolia/client-account": "4.10.3",
+        "@algolia/client-analytics": "4.10.3",
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-personalization": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/logger-common": "4.10.3",
+        "@algolia/logger-console": "4.10.3",
+        "@algolia/requester-browser-xhr": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/requester-node-http": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
     "node_modules/axios": {
       "version": "0.21.1",
       "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
@@ -815,6 +952,121 @@
     }
   },
   "dependencies": {
+    "@algolia/cache-browser-local-storage": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz",
+      "integrity": "sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==",
+      "requires": {
+        "@algolia/cache-common": "4.10.3"
+      }
+    },
+    "@algolia/cache-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.10.3.tgz",
+      "integrity": "sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw=="
+    },
+    "@algolia/cache-in-memory": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz",
+      "integrity": "sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==",
+      "requires": {
+        "@algolia/cache-common": "4.10.3"
+      }
+    },
+    "@algolia/client-account": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.10.3.tgz",
+      "integrity": "sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==",
+      "requires": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "@algolia/client-analytics": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.10.3.tgz",
+      "integrity": "sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==",
+      "requires": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "@algolia/client-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.10.3.tgz",
+      "integrity": "sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==",
+      "requires": {
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "@algolia/client-personalization": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.10.3.tgz",
+      "integrity": "sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==",
+      "requires": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "@algolia/client-search": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.10.3.tgz",
+      "integrity": "sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==",
+      "requires": {
+        "@algolia/client-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
+    "@algolia/logger-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.10.3.tgz",
+      "integrity": "sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg=="
+    },
+    "@algolia/logger-console": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.10.3.tgz",
+      "integrity": "sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==",
+      "requires": {
+        "@algolia/logger-common": "4.10.3"
+      }
+    },
+    "@algolia/requester-browser-xhr": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz",
+      "integrity": "sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==",
+      "requires": {
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
+    "@algolia/requester-common": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.10.3.tgz",
+      "integrity": "sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ=="
+    },
+    "@algolia/requester-node-http": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz",
+      "integrity": "sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==",
+      "requires": {
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
+    "@algolia/transporter": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.10.3.tgz",
+      "integrity": "sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==",
+      "requires": {
+        "@algolia/cache-common": "4.10.3",
+        "@algolia/logger-common": "4.10.3",
+        "@algolia/requester-common": "4.10.3"
+      }
+    },
     "@sindresorhus/is": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
@@ -865,6 +1117,27 @@
         "@types/node": "*"
       }
     },
+    "algoliasearch": {
+      "version": "4.10.3",
+      "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
+      "integrity": "sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==",
+      "requires": {
+        "@algolia/cache-browser-local-storage": "4.10.3",
+        "@algolia/cache-common": "4.10.3",
+        "@algolia/cache-in-memory": "4.10.3",
+        "@algolia/client-account": "4.10.3",
+        "@algolia/client-analytics": "4.10.3",
+        "@algolia/client-common": "4.10.3",
+        "@algolia/client-personalization": "4.10.3",
+        "@algolia/client-search": "4.10.3",
+        "@algolia/logger-common": "4.10.3",
+        "@algolia/logger-console": "4.10.3",
+        "@algolia/requester-browser-xhr": "4.10.3",
+        "@algolia/requester-common": "4.10.3",
+        "@algolia/requester-node-http": "4.10.3",
+        "@algolia/transporter": "4.10.3"
+      }
+    },
     "axios": {
       "version": "0.21.1",
       "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",

package.json

@@ -23,6 +23,7 @@
   },
   "homepage": "https://github.com/QURIresearch/metaforecasts#readme",
   "dependencies": {
+    "algoliasearch": "^4.10.3",
     "axios": "^0.21.1",
     "fs": "^0.0.1-security",
     "https": "^1.0.0",

src/index.js

@@ -24,19 +24,21 @@ import {smarkets} from "./platforms/smarkets-fetch.js"
 import {williamhill} from "./platforms/williamhill-fetch.js"
 import {mergeEverything} from "./utils/mergeEverything.js"
 import {updateHistory} from "./utils/history/updateHistory.js"
+import {rebuildAlgoliaDatabase} from "./utils/algolia.js"
 import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
 import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
 
 /* Support functions */
-let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything]
+let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
 let functionNames =  functions.map(fun => fun.name)
 
 let whattodoMessage = functionNames
-    .slice(0,functionNames.length-4)
+    .slice(0,functionNames.length-5)
     .map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
     .join('\n') +
-  `\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` + 
+  `\n[${functionNames.length-5}]: Merge jsons them into one big json (and push it to mongodb database)` + 
-  `\n[${functionNames.length-3}]: Update history` + 
+  `\n[${functionNames.length-4}]: Update history` + 
+  `\n[${functionNames.length-3}]: Rebuild algolia database ("index")` + 
   `\n[${functionNames.length-2}]: Rebuild netlify site with new data` + 
   // `\n[${functionNames.length-1}]: Add to history` + 
   `\n[${functionNames.length-1}]: All of the above` + 

src/input/metaforecast-algolia-configuration.json

@@ -0,0 +1,61 @@
+{
+  "settings": {
+    "minWordSizefor1Typo": 4,
+    "minWordSizefor2Typos": 8,
+    "hitsPerPage": 300,
+    "maxValuesPerFacet": 100,
+    "attributesToIndex": [
+      "unordered(title)",
+      "unordered(description)",
+      "unordered(optionsstringforsearch)",
+      "unordered(platform)"
+    ],
+    "numericAttributesToIndex": null,
+    "attributesToRetrieve": null,
+    "ignorePlurals": true,
+    "removeStopWords": true,
+    "unretrievableAttributes": null,
+    "optionalWords": null,
+    "queryLanguages": [
+      "en"
+    ],
+    "attributesForFaceting": null,
+    "attributesToSnippet": null,
+    "attributesToHighlight": [],
+    "paginationLimitedTo": 1000,
+    "attributeForDistinct": null,
+    "exactOnSingleWordQuery": "attribute",
+    "ranking": [
+      "typo",
+      "geo",
+      "words",
+      "filters",
+      "proximity",
+      "attribute",
+      "exact",
+      "custom"
+    ],
+    "customRanking": [
+      "desc(title)",
+      "desc(description)",
+      "desc(optionsstringforsearch)",
+      "desc(platform)"
+    ],
+    "separatorsToIndex": "",
+    "removeWordsIfNoResults": "allOptional",
+    "queryType": "prefixLast",
+    "highlightPreTag": "<em>",
+    "highlightPostTag": "</em>",
+    "snippetEllipsisText": "",
+    "alternativesAsExact": [
+      "ignorePlurals",
+      "singleWordSynonym",
+      "multiWordsSynonym"
+    ],
+    "indexLanguages": [
+      "en"
+    ]
+  },
+  "rules": [],
+  "synonyms": []
+}

src/platforms/astralcodexten-fetch.js

@@ -71,7 +71,7 @@ export async function astralcodexten(){
             "type": "PROBABILITY"
           }
         ]
-      }
+      } 
       let result = {
           "title": question.name.split(". ")[1],
           "url": `https://www.foretold.io/c/${community}/m/${question.id}`,
@@ -80,7 +80,7 @@ export async function astralcodexten(){
           "options": options,
           "timestamp": new Date().toISOString(),
           "qualityindicators": {
-            "numforecasts": (question.measurementCount +1) / 2,
+            "numforecasts": Number((question.measurementCount +1) / 2),
             "stars": calculateStars("AstralCodexTen", ({  }))
             }
           /*liquidity: liquidity.toFixed(2),

src/platforms/coupcast-fetch.js

@@ -8,7 +8,7 @@ import {calculateStars} from "../utils/stars.js"
 import {upsert} from "../utils/mongo-wrapper.js"
 
 /* Definitions */
-let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_05.csv"
+let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv"
 var datenow = new Date();
 var currentmonth = datenow.getMonth() + 1; 
 
@@ -64,7 +64,7 @@ async function processArray(countryArray) {
         "title": `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
         "url": url,
         "platform": "CoupCast",
-        "description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years.  ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
+        "description": `The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years.  ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years.`,
         "options": [
           {
             "name": "Yes",
@@ -102,7 +102,7 @@ async function processArray(countryArray) {
         "title": `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
         "url": url,
         "platform": "CoupCast",
-        "description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
+        "description": `The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
         "options": [
           {
             "name": "Yes",

src/platforms/csetforetell-fetch-old.js

@@ -1,201 +0,0 @@
-/* Imports */
-import axios from "axios"
-import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
-import {Tabletojson} from "tabletojson"
-import toMarkdown from "../utils/toMarkdown.js"
-import {calculateStars} from "../utils/stars.js"
-import {upsert} from "../utils/mongo-wrapper.js"
-
-/* Definitions */
-let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
-String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
-
-/* Support functions */
-
-async function fetchPage(page, cookie){
-  if(page==1){
-    cookie=cookie.split(";")[0] // Interesting that it otherwise doesn't work :(
-  }
-  let response  = await axios({
-    url: htmlEndPoint+page,
-    method: 'GET',
-    headers: ({ 
-    'Content-Type': 'text/html',
-    'Cookie': cookie
-    }),
-  })
-  .then(res => res.data)
-  // console.log(response)
-  return response
-}
-
-async function fetchStats(questionUrl, cookie){
-  let response  = await axios({
-    url: questionUrl+"/stats",
-    method: 'GET',
-    headers: ({ 
-    'Content-Type': 'text/html',
-    'Cookie': cookie,
-    'Referer': questionUrl,
-    }),
-  })
-  .then(res => res.data)
-  //console.log(response)
-  
-  // Is binary?
-  let isbinary = response.includes("binary?":true")
-  // console.log(`is binary? ${isbinary}`)
-  let options = []
-  if(isbinary){
-    // Crowd percentage
-    let htmlElements = response.split("\n")
-    // console.log(htmlElements)
-    let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
-    let crowdpercentage = h3Element.split(">")[1].split("<")[0]
-    let probability = Number(crowdpercentage.replace("%", ""))/100
-    options.push(({
-      name: "Yes",
-      probability: probability,
-      type: "PROBABILITY"
-    }), ({
-      name: "No",
-      probability: +(1-probability).toFixed(2), // avoids floating point shenanigans
-      type: "PROBABILITY"
-    }))
-  }else{
-    let optionsBody = response.split("tbody")[3] // Previously [1], but they added a new table.
-    // console.log(optionsBody)
-    let optionsHtmlElement = "<table" + optionsBody + "table>"
-    let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
-    let firstTable = tablesAsJson[0]
-    options = firstTable.map(element => ({
-      name: element['0'],
-      probability: Number(element['1'].replace("%",""))/100,
-      type: "PROBABILITY"
-    }))
-  }
-  // Description  
-  let descriptionraw = response.split(`<meta name="description" content="`)[1]
-  let descriptionprocessed1 = descriptionraw.split(`">`)[0]
-  let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
-  let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
-  // console.log(descriptionprocessed3)
-  let descriptionprocessed4=descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
-  let descriptionprocessed5=  descriptionprocessed4.replaceAll("\n\n", "\n")  
-  let descriptionprocessed6=descriptionprocessed5.replaceAll("&quot;", `"`)
-  let descriptionprocessed7=descriptionprocessed6.replaceAll("&#39;", "'")
-  let descriptionprocessed8=toMarkdown(descriptionprocessed7)
-  let description = descriptionprocessed8
-
-  // Number of forecasts
-  let numforecasts = response.split("prediction_sets_count&quot;:")[1].split(",")[0]
-  // console.log(numforecasts)
-
-  // Number of predictors
-  let numforecasters = response.split("predictors_count&quot;:")[1].split(",")[0]
-  // console.log(numpredictors)
-  
-  let result = {
-    "description": description, 
-    "options": options,
-    "timestamp": new Date().toISOString(),
-    "qualityindicators": {
-      "numforecasts": numforecasts,
-      "num_forecasters": numforecasters,
-      "stars": calculateStars("CSET-foretell", {numforecasts})
-    }
-  }
-  
-  return result
-}
-
-function isEnd(html){
-  return html.includes("No questions match your filter")
-}
-
-function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-/* Body */
-
-async function csetforetell_inner(cookie){
-  let i=0
-  let response = await fetchPage(i, cookie)
-  let results = []
-  let init = Date.now()
-  // console.log("Downloading... This might take a couple of minutes. Results will be shown.")
-  while(!isEnd(response)){
-    
-    let htmlLines = response.split("\n")
-    let h4elements = htmlLines.filter(str => str.includes("<h5><a href=") || str.includes("<h4><a href=")) 
-
-    if(process.env.DEBUG_MODE == "on"){
-      console.log(response)
-    }
-
-    //console.log("")
-    //console.log("")
-    //console.log(h4elements)
-    
-    for(let h4element of h4elements){
-      let h4elementSplit = h4element.split('"><span>')
-      let url = h4elementSplit[0].split('<a href="')[1]
-      //console.log(url)
-      let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "")
-      await sleep(1000 + Math.random()*1000) // don't be as noticeable
-      try{
-        let moreinfo = await fetchStats(url, cookie)
-        if(moreinfo.isbinary){
-          if(!moreinfo.crowdpercentage){ // then request again.
-            moreinfo = await fetchStats(url, cookie)
-          }
-        }
-        let question = ({
-            "title": title,
-            "url": url,
-            "platform": "CSET-foretell",
-            ...moreinfo
-          })
-          if(i % 30 == 0){
-            console.log(`Page #${i}`)
-            console.log(question)
-          }
-          results.push(question)
-          if(process.env.DEBUG_MODE == "on"){
-            console.log(question)
-          }
-
-      } catch(error){
-        console.log(error)
-        console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
-      }
-      i=i+1
-    }
-    
-    // console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
-    await sleep(5000 + Math.random()*1000) // don't be as noticeable
-    
-    try{
-      response = await fetchPage(i, cookie)
-    }catch(error){
-      console.log(error)
-      console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
-    }
-  }
-  // let string = JSON.stringify(results,null,  2)
-  // fs.writeFileSync('./data/csetforetell-questions.json', string);
-  // console.log(results)
-  await upsert(results, "csetforetell-questions")
-  
-  
-  let end = Date.now()
-  let difference = end-init
-  console.log(`Took ${difference/1000} seconds, or ${difference/(1000*60)} minutes.`)
-}
-
-
-export async function csetforetell(){
-  let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell") 
-  await applyIfCookieExists(cookie, csetforetell_inner)
-}

src/platforms/csetforetell-fetch.js

@@ -122,8 +122,8 @@ async function fetchStats(questionUrl, cookie){
     "options": options,
     "timestamp": new Date().toISOString(),
     "qualityindicators": {
-      "numforecasts": numforecasts,
+      "numforecasts":Number(numforecasts),
-      "num_forecasters": numforecasters,
+      "numforecasters":Number(numforecasters),
       "stars": calculateStars("CSET-foretell", {numforecasts})
     }
   }

src/platforms/elicit-fetch.js

@@ -73,8 +73,8 @@ async function processArray(arrayQuestions) {
         ],
         "timestamp": new Date().toISOString(),
         "qualityindicators": {
-          "numforecasts": numforecasts,
+          "numforecasts": Number(numforecasts),
-          "numforecasters": numforecasters,
+          "numforecasters": Number(numforecasters),
           "stars": calculateStars("Elicit", ({}))
         }
       })

src/platforms/fantasyscotus-fetch.js

@@ -87,7 +87,7 @@ async function processData(data) {
               ],
         "timestamp": new Date().toISOString(),
         "qualityindicators": {
-          "numforecasts": predictionData.numForecasts,
+          "numforecasts": Number(predictionData.numForecasts),
           "stars": calculateStars("FantasySCOTUS", ({}))
         }
       })

src/platforms/foretold-fetch.js

@@ -80,7 +80,7 @@ export async function foretold(){
           "options": options,
           "timestamp": new Date().toISOString(),
           "qualityindicators": {
-            "numforecasts": question.measurementCount / 2,
+            "numforecasts": Math.floor(Number(question.measurementCount) / 2),
             "stars": calculateStars("Foretold", ({  }))
             }
           /*liquidity: liquidity.toFixed(2),

src/platforms/goodjudmentopen-fetch.js

@@ -99,8 +99,8 @@ async function fetchStats(questionUrl, cookie) {
     "options": options,
     "timestamp": new Date().toISOString(),
     "qualityindicators": {
-      "numforecasts": numforecasts,
+      "numforecasts": Number(numforecasts),
-      "numforecasters": numforecasters,
+      "numforecasters": Number(numforecasters),
       "stars": calculateStars("Good Judgment Open", ({ numforecasts, minProbability, maxProbability }))
     }
   }

src/platforms/kalshi-fetch.js

@@ -22,6 +22,9 @@ async function fetchAllMarkets() { // for info which the polymarket graphql API
 
 
 async function processMarkets(markets) {
+  let dateNow = new Date().toISOString()
+  // console.log(markets)
+  markets = markets.filter(market => market.close_date > dateNow)
   let results = await markets.map(market => {
     let probability = market.last_price/100
     let options = [
@@ -54,6 +57,12 @@ async function processMarkets(markets) {
     })
     return result
   })
+  //console.log(results.length)
+  console.log(results.map(result => result.title))
+  console.log(results.map(result => result.title).length)
+  console.log([...new Set(results.map(result => result.title))])
+  console.log([...new Set(results.map(result => result.title))].length)
+  // console.log([...new Set(results.map(result => result.title))].length)
   return results //resultsProcessed
 }
 

src/platforms/metaculus-fetch.js

@@ -111,7 +111,7 @@ export async function metaculus() {
           "options": options,
           "timestamp": new Date().toISOString(),
           "qualityindicators": {
-            "numforecasts": result.number_of_predictions,
+            "numforecasts": Number(result.number_of_predictions),
             "resolution_data": {
               "publish_time": result.publish_time,
               "resolution": result.resolution,

src/utils/algolia.js

@@ -0,0 +1,23 @@
+import algoliasearch from 'algoliasearch';
+import fs from "fs"
+
+import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
+
+const client = algoliasearch('96UD3NTQ7L', process.env.ALGOLIA_MASTER_API_KEY); // delete this when committing
+const index = client.initIndex('metaforecast');
+
+export async function rebuildAlgoliaDatabase(){
+  let records = await mongoReadWithReadCredentials("metaforecasts")
+  // let string = JSON.stringify(json, null, 2)
+  // fs.writeFileSync('metaforecasts.json', string);
+  records = records.map(record => ({...record, has_numforecasts: record.has_numforecasts ? true : false}) )
+  // this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
+
+  await index.clearObjects()
+  index.saveObjects(records, { autoGenerateObjectIDIfNotExist: true }).then(() => 
+    console.log("algolia search: done")
+  ).catch(error => {
+    console.log("algolia search: error", error)
+  })
+}
+// main()

src/utils/doEverything.js

@@ -20,6 +20,7 @@ import {smarkets} from "../platforms/smarkets-fetch.js"
 import {williamhill} from "../platforms/williamhill-fetch.js"
 import {mergeEverything} from "./mergeEverything.js"
 import {updateHistory} from "./history/updateHistory.js"
+import {rebuildAlgoliaDatabase} from "./algolia.js"
 import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js"
 
 /* Do everything */
@@ -39,7 +40,7 @@ export async function tryCatchTryAgain (fun) {
     }
 }
 export async function doEverything(){
-    let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData] 
+    let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus,  foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData] 
     // Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
 
     console.log("")

src/utils/manualDownloadFromMongo.js

@@ -7,7 +7,7 @@ let main = async () => {
   let string = JSON.stringify(json, null, 2)
   fs.writeFileSync('metaforecasts.json', string);
 }
-// main()
+main()
 
 let extractQualityIndicators = async () => {
   let json = await mongoReadWithReadCredentials("metaforecasts")
@@ -18,4 +18,4 @@ let extractQualityIndicators = async () => {
   // let string = JSON.stringify(json, null, 2)
   // fs.writeFileSync('metaforecasts.json', string);
 }
-extractQualityIndicators()
+// extractQualityIndicators()