feat: Added Algolia to search
Also some light cleanup necessary to Algolia, e.g., numforecasts is now guaranteed to be a Number() (or a NaN, in case something goes very wrong)
This commit is contained in:
parent
b1d8c3cba9
commit
e2adcf95f4
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -1,7 +1,14 @@
|
|||
## Node modules
|
||||
node_modules/
|
||||
|
||||
## Security
|
||||
**/betfaircertificates/
|
||||
**/privatekeys.json
|
||||
|
||||
## Personal notes
|
||||
possiblenewsources.md
|
||||
NotesHeroku.md
|
||||
NotesHeroku.md
|
||||
AlgoliaNotes.md
|
||||
|
||||
## Build artifacts
|
||||
done.txt
|
||||
31540
metaforecasts.json
31540
metaforecasts.json
File diff suppressed because one or more lines are too long
136
node_modules/.package-lock.json
generated
vendored
136
node_modules/.package-lock.json
generated
vendored
|
|
@ -4,6 +4,121 @@
|
|||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"node_modules/@algolia/cache-browser-local-storage": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz",
|
||||
"integrity": "sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/cache-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.10.3.tgz",
|
||||
"integrity": "sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw=="
|
||||
},
|
||||
"node_modules/@algolia/cache-in-memory": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz",
|
||||
"integrity": "sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-account": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.10.3.tgz",
|
||||
"integrity": "sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-analytics": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.10.3.tgz",
|
||||
"integrity": "sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.10.3.tgz",
|
||||
"integrity": "sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==",
|
||||
"dependencies": {
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-personalization": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.10.3.tgz",
|
||||
"integrity": "sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-search": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.10.3.tgz",
|
||||
"integrity": "sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/logger-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.10.3.tgz",
|
||||
"integrity": "sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg=="
|
||||
},
|
||||
"node_modules/@algolia/logger-console": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.10.3.tgz",
|
||||
"integrity": "sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==",
|
||||
"dependencies": {
|
||||
"@algolia/logger-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/requester-browser-xhr": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz",
|
||||
"integrity": "sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==",
|
||||
"dependencies": {
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/requester-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.10.3.tgz",
|
||||
"integrity": "sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ=="
|
||||
},
|
||||
"node_modules/@algolia/requester-node-http": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz",
|
||||
"integrity": "sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==",
|
||||
"dependencies": {
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/transporter": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.10.3.tgz",
|
||||
"integrity": "sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-common": "4.10.3",
|
||||
"@algolia/logger-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@sindresorhus/is": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
|
||||
|
|
@ -60,6 +175,27 @@
|
|||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/algoliasearch": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
|
||||
"integrity": "sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-browser-local-storage": "4.10.3",
|
||||
"@algolia/cache-common": "4.10.3",
|
||||
"@algolia/cache-in-memory": "4.10.3",
|
||||
"@algolia/client-account": "4.10.3",
|
||||
"@algolia/client-analytics": "4.10.3",
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-personalization": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/logger-common": "4.10.3",
|
||||
"@algolia/logger-console": "4.10.3",
|
||||
"@algolia/requester-browser-xhr": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/requester-node-http": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "0.21.1",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
|
||||
|
|
|
|||
273
package-lock.json
generated
273
package-lock.json
generated
|
|
@ -9,6 +9,7 @@
|
|||
"version": "0.3.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"algoliasearch": "^4.10.3",
|
||||
"axios": "^0.21.1",
|
||||
"fs": "^0.0.1-security",
|
||||
"https": "^1.0.0",
|
||||
|
|
@ -21,6 +22,121 @@
|
|||
"textversionjs": "^1.1.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/cache-browser-local-storage": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz",
|
||||
"integrity": "sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/cache-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.10.3.tgz",
|
||||
"integrity": "sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw=="
|
||||
},
|
||||
"node_modules/@algolia/cache-in-memory": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz",
|
||||
"integrity": "sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-account": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.10.3.tgz",
|
||||
"integrity": "sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-analytics": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.10.3.tgz",
|
||||
"integrity": "sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.10.3.tgz",
|
||||
"integrity": "sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==",
|
||||
"dependencies": {
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-personalization": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.10.3.tgz",
|
||||
"integrity": "sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/client-search": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.10.3.tgz",
|
||||
"integrity": "sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==",
|
||||
"dependencies": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/logger-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.10.3.tgz",
|
||||
"integrity": "sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg=="
|
||||
},
|
||||
"node_modules/@algolia/logger-console": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.10.3.tgz",
|
||||
"integrity": "sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==",
|
||||
"dependencies": {
|
||||
"@algolia/logger-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/requester-browser-xhr": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz",
|
||||
"integrity": "sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==",
|
||||
"dependencies": {
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/requester-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.10.3.tgz",
|
||||
"integrity": "sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ=="
|
||||
},
|
||||
"node_modules/@algolia/requester-node-http": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz",
|
||||
"integrity": "sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==",
|
||||
"dependencies": {
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@algolia/transporter": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.10.3.tgz",
|
||||
"integrity": "sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-common": "4.10.3",
|
||||
"@algolia/logger-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@sindresorhus/is": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
|
||||
|
|
@ -77,6 +193,27 @@
|
|||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/algoliasearch": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
|
||||
"integrity": "sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==",
|
||||
"dependencies": {
|
||||
"@algolia/cache-browser-local-storage": "4.10.3",
|
||||
"@algolia/cache-common": "4.10.3",
|
||||
"@algolia/cache-in-memory": "4.10.3",
|
||||
"@algolia/client-account": "4.10.3",
|
||||
"@algolia/client-analytics": "4.10.3",
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-personalization": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/logger-common": "4.10.3",
|
||||
"@algolia/logger-console": "4.10.3",
|
||||
"@algolia/requester-browser-xhr": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/requester-node-http": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "0.21.1",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
|
||||
|
|
@ -815,6 +952,121 @@
|
|||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@algolia/cache-browser-local-storage": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz",
|
||||
"integrity": "sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==",
|
||||
"requires": {
|
||||
"@algolia/cache-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/cache-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.10.3.tgz",
|
||||
"integrity": "sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw=="
|
||||
},
|
||||
"@algolia/cache-in-memory": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz",
|
||||
"integrity": "sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==",
|
||||
"requires": {
|
||||
"@algolia/cache-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/client-account": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.10.3.tgz",
|
||||
"integrity": "sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/client-analytics": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.10.3.tgz",
|
||||
"integrity": "sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/client-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.10.3.tgz",
|
||||
"integrity": "sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==",
|
||||
"requires": {
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/client-personalization": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.10.3.tgz",
|
||||
"integrity": "sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/client-search": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.10.3.tgz",
|
||||
"integrity": "sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/logger-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.10.3.tgz",
|
||||
"integrity": "sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg=="
|
||||
},
|
||||
"@algolia/logger-console": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.10.3.tgz",
|
||||
"integrity": "sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==",
|
||||
"requires": {
|
||||
"@algolia/logger-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/requester-browser-xhr": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz",
|
||||
"integrity": "sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==",
|
||||
"requires": {
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/requester-common": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.10.3.tgz",
|
||||
"integrity": "sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ=="
|
||||
},
|
||||
"@algolia/requester-node-http": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz",
|
||||
"integrity": "sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==",
|
||||
"requires": {
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@algolia/transporter": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.10.3.tgz",
|
||||
"integrity": "sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==",
|
||||
"requires": {
|
||||
"@algolia/cache-common": "4.10.3",
|
||||
"@algolia/logger-common": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3"
|
||||
}
|
||||
},
|
||||
"@sindresorhus/is": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz",
|
||||
|
|
@ -865,6 +1117,27 @@
|
|||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"algoliasearch": {
|
||||
"version": "4.10.3",
|
||||
"resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.10.3.tgz",
|
||||
"integrity": "sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==",
|
||||
"requires": {
|
||||
"@algolia/cache-browser-local-storage": "4.10.3",
|
||||
"@algolia/cache-common": "4.10.3",
|
||||
"@algolia/cache-in-memory": "4.10.3",
|
||||
"@algolia/client-account": "4.10.3",
|
||||
"@algolia/client-analytics": "4.10.3",
|
||||
"@algolia/client-common": "4.10.3",
|
||||
"@algolia/client-personalization": "4.10.3",
|
||||
"@algolia/client-search": "4.10.3",
|
||||
"@algolia/logger-common": "4.10.3",
|
||||
"@algolia/logger-console": "4.10.3",
|
||||
"@algolia/requester-browser-xhr": "4.10.3",
|
||||
"@algolia/requester-common": "4.10.3",
|
||||
"@algolia/requester-node-http": "4.10.3",
|
||||
"@algolia/transporter": "4.10.3"
|
||||
}
|
||||
},
|
||||
"axios": {
|
||||
"version": "0.21.1",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
},
|
||||
"homepage": "https://github.com/QURIresearch/metaforecasts#readme",
|
||||
"dependencies": {
|
||||
"algoliasearch": "^4.10.3",
|
||||
"axios": "^0.21.1",
|
||||
"fs": "^0.0.1-security",
|
||||
"https": "^1.0.0",
|
||||
|
|
|
|||
10
src/index.js
10
src/index.js
|
|
@ -24,19 +24,21 @@ import {smarkets} from "./platforms/smarkets-fetch.js"
|
|||
import {williamhill} from "./platforms/williamhill-fetch.js"
|
||||
import {mergeEverything} from "./utils/mergeEverything.js"
|
||||
import {updateHistory} from "./utils/history/updateHistory.js"
|
||||
import {rebuildAlgoliaDatabase} from "./utils/algolia.js"
|
||||
import {rebuildNetlifySiteWithNewData} from "./utils/rebuildNetliftySiteWithNewData.js"
|
||||
import {doEverything, tryCatchTryAgain} from "./utils/doEverything.js"
|
||||
|
||||
/* Support functions */
|
||||
let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData, doEverything]
|
||||
let functions = [astralcodexten, betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, kalshi, ladbrokes, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData, doEverything]
|
||||
let functionNames = functions.map(fun => fun.name)
|
||||
|
||||
let whattodoMessage = functionNames
|
||||
.slice(0,functionNames.length-4)
|
||||
.slice(0,functionNames.length-5)
|
||||
.map((functionName,i) => `[${i}]: Download predictions from ${functionName}`)
|
||||
.join('\n') +
|
||||
`\n[${functionNames.length-4}]: Merge jsons them into one big json (and push it to mongodb database)` +
|
||||
`\n[${functionNames.length-3}]: Update history` +
|
||||
`\n[${functionNames.length-5}]: Merge jsons them into one big json (and push it to mongodb database)` +
|
||||
`\n[${functionNames.length-4}]: Update history` +
|
||||
`\n[${functionNames.length-3}]: Rebuild algolia database ("index")` +
|
||||
`\n[${functionNames.length-2}]: Rebuild netlify site with new data` +
|
||||
// `\n[${functionNames.length-1}]: Add to history` +
|
||||
`\n[${functionNames.length-1}]: All of the above` +
|
||||
|
|
|
|||
61
src/input/metaforecast-algolia-configuration.json
Normal file
61
src/input/metaforecast-algolia-configuration.json
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
{
|
||||
"settings": {
|
||||
"minWordSizefor1Typo": 4,
|
||||
"minWordSizefor2Typos": 8,
|
||||
"hitsPerPage": 300,
|
||||
"maxValuesPerFacet": 100,
|
||||
"attributesToIndex": [
|
||||
"unordered(title)",
|
||||
"unordered(description)",
|
||||
"unordered(optionsstringforsearch)",
|
||||
"unordered(platform)"
|
||||
],
|
||||
"numericAttributesToIndex": null,
|
||||
"attributesToRetrieve": null,
|
||||
"ignorePlurals": true,
|
||||
"removeStopWords": true,
|
||||
"unretrievableAttributes": null,
|
||||
"optionalWords": null,
|
||||
"queryLanguages": [
|
||||
"en"
|
||||
],
|
||||
"attributesForFaceting": null,
|
||||
"attributesToSnippet": null,
|
||||
"attributesToHighlight": [],
|
||||
"paginationLimitedTo": 1000,
|
||||
"attributeForDistinct": null,
|
||||
"exactOnSingleWordQuery": "attribute",
|
||||
"ranking": [
|
||||
"typo",
|
||||
"geo",
|
||||
"words",
|
||||
"filters",
|
||||
"proximity",
|
||||
"attribute",
|
||||
"exact",
|
||||
"custom"
|
||||
],
|
||||
"customRanking": [
|
||||
"desc(title)",
|
||||
"desc(description)",
|
||||
"desc(optionsstringforsearch)",
|
||||
"desc(platform)"
|
||||
],
|
||||
"separatorsToIndex": "",
|
||||
"removeWordsIfNoResults": "allOptional",
|
||||
"queryType": "prefixLast",
|
||||
"highlightPreTag": "<em>",
|
||||
"highlightPostTag": "</em>",
|
||||
"snippetEllipsisText": "",
|
||||
"alternativesAsExact": [
|
||||
"ignorePlurals",
|
||||
"singleWordSynonym",
|
||||
"multiWordsSynonym"
|
||||
],
|
||||
"indexLanguages": [
|
||||
"en"
|
||||
]
|
||||
},
|
||||
"rules": [],
|
||||
"synonyms": []
|
||||
}
|
||||
|
|
@ -71,7 +71,7 @@ export async function astralcodexten(){
|
|||
"type": "PROBABILITY"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
let result = {
|
||||
"title": question.name.split(". ")[1],
|
||||
"url": `https://www.foretold.io/c/${community}/m/${question.id}`,
|
||||
|
|
@ -80,7 +80,7 @@ export async function astralcodexten(){
|
|||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": (question.measurementCount +1) / 2,
|
||||
"numforecasts": Number((question.measurementCount +1) / 2),
|
||||
"stars": calculateStars("AstralCodexTen", ({ }))
|
||||
}
|
||||
/*liquidity: liquidity.toFixed(2),
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import {calculateStars} from "../utils/stars.js"
|
|||
import {upsert} from "../utils/mongo-wrapper.js"
|
||||
|
||||
/* Definitions */
|
||||
let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_05.csv"
|
||||
let coupCastEndpoint = "https://www.oneearthfuture.org/sites/all/themes/stability/stability_sub/data/dashboard_2021_code_06.csv"
|
||||
var datenow = new Date();
|
||||
var currentmonth = datenow.getMonth() + 1;
|
||||
|
||||
|
|
@ -64,7 +64,7 @@ async function processArray(countryArray) {
|
|||
"title": `Will there be a coup in ${country.country_name} in the next month (as of ${country.month}/${country.year})?`,
|
||||
"url": url,
|
||||
"platform": "CoupCast",
|
||||
"description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
|
||||
"description": `The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years.`,
|
||||
"options": [
|
||||
{
|
||||
"name": "Yes",
|
||||
|
|
@ -102,7 +102,7 @@ async function processArray(countryArray) {
|
|||
"title": `Will there be a coup in ${country.country_name} in the next year (as of ${country.month}/${country.year})?`,
|
||||
"url": url,
|
||||
"platform": "CoupCast",
|
||||
"description": `. The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
|
||||
"description": `The current leader of ${country.country_name} is ${country.leader_name}, who has been in power for ${Number(country.leader_years).toFixed(1)} years. ${country.country_name} has a ${(country.regime_type).toLowerCase()} regime type which has lasted for ${country.regime_years} years`,
|
||||
"options": [
|
||||
{
|
||||
"name": "Yes",
|
||||
|
|
|
|||
|
|
@ -1,201 +0,0 @@
|
|||
/* Imports */
|
||||
import axios from "axios"
|
||||
import {getCookie, applyIfCookieExists} from "../utils/getCookies.js"
|
||||
import {Tabletojson} from "tabletojson"
|
||||
import toMarkdown from "../utils/toMarkdown.js"
|
||||
import {calculateStars} from "../utils/stars.js"
|
||||
import {upsert} from "../utils/mongo-wrapper.js"
|
||||
|
||||
/* Definitions */
|
||||
let htmlEndPoint = 'https://www.cset-foretell.com/questions?page='
|
||||
String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); }
|
||||
|
||||
/* Support functions */
|
||||
|
||||
async function fetchPage(page, cookie){
|
||||
if(page==1){
|
||||
cookie=cookie.split(";")[0] // Interesting that it otherwise doesn't work :(
|
||||
}
|
||||
let response = await axios({
|
||||
url: htmlEndPoint+page,
|
||||
method: 'GET',
|
||||
headers: ({
|
||||
'Content-Type': 'text/html',
|
||||
'Cookie': cookie
|
||||
}),
|
||||
})
|
||||
.then(res => res.data)
|
||||
// console.log(response)
|
||||
return response
|
||||
}
|
||||
|
||||
async function fetchStats(questionUrl, cookie){
|
||||
let response = await axios({
|
||||
url: questionUrl+"/stats",
|
||||
method: 'GET',
|
||||
headers: ({
|
||||
'Content-Type': 'text/html',
|
||||
'Cookie': cookie,
|
||||
'Referer': questionUrl,
|
||||
}),
|
||||
})
|
||||
.then(res => res.data)
|
||||
//console.log(response)
|
||||
|
||||
// Is binary?
|
||||
let isbinary = response.includes("binary?":true")
|
||||
// console.log(`is binary? ${isbinary}`)
|
||||
let options = []
|
||||
if(isbinary){
|
||||
// Crowd percentage
|
||||
let htmlElements = response.split("\n")
|
||||
// console.log(htmlElements)
|
||||
let h3Element = htmlElements.filter(str => str.includes("<h3>"))[0]
|
||||
let crowdpercentage = h3Element.split(">")[1].split("<")[0]
|
||||
let probability = Number(crowdpercentage.replace("%", ""))/100
|
||||
options.push(({
|
||||
name: "Yes",
|
||||
probability: probability,
|
||||
type: "PROBABILITY"
|
||||
}), ({
|
||||
name: "No",
|
||||
probability: +(1-probability).toFixed(2), // avoids floating point shenanigans
|
||||
type: "PROBABILITY"
|
||||
}))
|
||||
}else{
|
||||
let optionsBody = response.split("tbody")[3] // Previously [1], but they added a new table.
|
||||
// console.log(optionsBody)
|
||||
let optionsHtmlElement = "<table" + optionsBody + "table>"
|
||||
let tablesAsJson = Tabletojson.convert(optionsHtmlElement)
|
||||
let firstTable = tablesAsJson[0]
|
||||
options = firstTable.map(element => ({
|
||||
name: element['0'],
|
||||
probability: Number(element['1'].replace("%",""))/100,
|
||||
type: "PROBABILITY"
|
||||
}))
|
||||
}
|
||||
// Description
|
||||
let descriptionraw = response.split(`<meta name="description" content="`)[1]
|
||||
let descriptionprocessed1 = descriptionraw.split(`">`)[0]
|
||||
let descriptionprocessed2 = descriptionprocessed1.replace(">", "")
|
||||
let descriptionprocessed3 = descriptionprocessed2.replace("To suggest a change or clarification to this question, please select Request Clarification from the green gear-shaped dropdown button to the right of the question.", ``)
|
||||
// console.log(descriptionprocessed3)
|
||||
let descriptionprocessed4=descriptionprocessed3.replaceAll("\r\n\r\n", "\n")
|
||||
let descriptionprocessed5= descriptionprocessed4.replaceAll("\n\n", "\n")
|
||||
let descriptionprocessed6=descriptionprocessed5.replaceAll(""", `"`)
|
||||
let descriptionprocessed7=descriptionprocessed6.replaceAll("'", "'")
|
||||
let descriptionprocessed8=toMarkdown(descriptionprocessed7)
|
||||
let description = descriptionprocessed8
|
||||
|
||||
// Number of forecasts
|
||||
let numforecasts = response.split("prediction_sets_count":")[1].split(",")[0]
|
||||
// console.log(numforecasts)
|
||||
|
||||
// Number of predictors
|
||||
let numforecasters = response.split("predictors_count":")[1].split(",")[0]
|
||||
// console.log(numpredictors)
|
||||
|
||||
let result = {
|
||||
"description": description,
|
||||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": numforecasts,
|
||||
"num_forecasters": numforecasters,
|
||||
"stars": calculateStars("CSET-foretell", {numforecasts})
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function isEnd(html){
|
||||
return html.includes("No questions match your filter")
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/* Body */
|
||||
|
||||
async function csetforetell_inner(cookie){
|
||||
let i=0
|
||||
let response = await fetchPage(i, cookie)
|
||||
let results = []
|
||||
let init = Date.now()
|
||||
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
|
||||
while(!isEnd(response)){
|
||||
|
||||
let htmlLines = response.split("\n")
|
||||
let h4elements = htmlLines.filter(str => str.includes("<h5><a href=") || str.includes("<h4><a href="))
|
||||
|
||||
if(process.env.DEBUG_MODE == "on"){
|
||||
console.log(response)
|
||||
}
|
||||
|
||||
//console.log("")
|
||||
//console.log("")
|
||||
//console.log(h4elements)
|
||||
|
||||
for(let h4element of h4elements){
|
||||
let h4elementSplit = h4element.split('"><span>')
|
||||
let url = h4elementSplit[0].split('<a href="')[1]
|
||||
//console.log(url)
|
||||
let title = h4elementSplit[1].replace('</span></a></h4>', "").replace('</span></a></h5>', "")
|
||||
await sleep(1000 + Math.random()*1000) // don't be as noticeable
|
||||
try{
|
||||
let moreinfo = await fetchStats(url, cookie)
|
||||
if(moreinfo.isbinary){
|
||||
if(!moreinfo.crowdpercentage){ // then request again.
|
||||
moreinfo = await fetchStats(url, cookie)
|
||||
}
|
||||
}
|
||||
let question = ({
|
||||
"title": title,
|
||||
"url": url,
|
||||
"platform": "CSET-foretell",
|
||||
...moreinfo
|
||||
})
|
||||
if(i % 30 == 0){
|
||||
console.log(`Page #${i}`)
|
||||
console.log(question)
|
||||
}
|
||||
results.push(question)
|
||||
if(process.env.DEBUG_MODE == "on"){
|
||||
console.log(question)
|
||||
}
|
||||
|
||||
} catch(error){
|
||||
console.log(error)
|
||||
console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
|
||||
}
|
||||
i=i+1
|
||||
}
|
||||
|
||||
// console.log("Sleeping for ~5secs so as to not be as noticeable to the cset-foretell servers")
|
||||
await sleep(5000 + Math.random()*1000) // don't be as noticeable
|
||||
|
||||
try{
|
||||
response = await fetchPage(i, cookie)
|
||||
}catch(error){
|
||||
console.log(error)
|
||||
console.log(`The program encountered some error when fetching page #${i}, so it won't appear on the final json. It is possible that this page wasn't actually a prediction question pages`)
|
||||
}
|
||||
}
|
||||
// let string = JSON.stringify(results,null, 2)
|
||||
// fs.writeFileSync('./data/csetforetell-questions.json', string);
|
||||
// console.log(results)
|
||||
await upsert(results, "csetforetell-questions")
|
||||
|
||||
|
||||
let end = Date.now()
|
||||
let difference = end-init
|
||||
console.log(`Took ${difference/1000} seconds, or ${difference/(1000*60)} minutes.`)
|
||||
}
|
||||
|
||||
|
||||
export async function csetforetell(){
|
||||
let cookie = process.env.CSETFORETELL_COOKIE || getCookie("csetforetell")
|
||||
await applyIfCookieExists(cookie, csetforetell_inner)
|
||||
}
|
||||
|
|
@ -122,8 +122,8 @@ async function fetchStats(questionUrl, cookie){
|
|||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": numforecasts,
|
||||
"num_forecasters": numforecasters,
|
||||
"numforecasts":Number(numforecasts),
|
||||
"numforecasters":Number(numforecasters),
|
||||
"stars": calculateStars("CSET-foretell", {numforecasts})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,8 +73,8 @@ async function processArray(arrayQuestions) {
|
|||
],
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": numforecasts,
|
||||
"numforecasters": numforecasters,
|
||||
"numforecasts": Number(numforecasts),
|
||||
"numforecasters": Number(numforecasters),
|
||||
"stars": calculateStars("Elicit", ({}))
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ async function processData(data) {
|
|||
],
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": predictionData.numForecasts,
|
||||
"numforecasts": Number(predictionData.numForecasts),
|
||||
"stars": calculateStars("FantasySCOTUS", ({}))
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ export async function foretold(){
|
|||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": question.measurementCount / 2,
|
||||
"numforecasts": Math.floor(Number(question.measurementCount) / 2),
|
||||
"stars": calculateStars("Foretold", ({ }))
|
||||
}
|
||||
/*liquidity: liquidity.toFixed(2),
|
||||
|
|
|
|||
|
|
@ -99,8 +99,8 @@ async function fetchStats(questionUrl, cookie) {
|
|||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": numforecasts,
|
||||
"numforecasters": numforecasters,
|
||||
"numforecasts": Number(numforecasts),
|
||||
"numforecasters": Number(numforecasters),
|
||||
"stars": calculateStars("Good Judgment Open", ({ numforecasts, minProbability, maxProbability }))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ async function fetchAllMarkets() { // for info which the polymarket graphql API
|
|||
|
||||
|
||||
async function processMarkets(markets) {
|
||||
let dateNow = new Date().toISOString()
|
||||
// console.log(markets)
|
||||
markets = markets.filter(market => market.close_date > dateNow)
|
||||
let results = await markets.map(market => {
|
||||
let probability = market.last_price/100
|
||||
let options = [
|
||||
|
|
@ -54,6 +57,12 @@ async function processMarkets(markets) {
|
|||
})
|
||||
return result
|
||||
})
|
||||
//console.log(results.length)
|
||||
console.log(results.map(result => result.title))
|
||||
console.log(results.map(result => result.title).length)
|
||||
console.log([...new Set(results.map(result => result.title))])
|
||||
console.log([...new Set(results.map(result => result.title))].length)
|
||||
// console.log([...new Set(results.map(result => result.title))].length)
|
||||
return results //resultsProcessed
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ export async function metaculus() {
|
|||
"options": options,
|
||||
"timestamp": new Date().toISOString(),
|
||||
"qualityindicators": {
|
||||
"numforecasts": result.number_of_predictions,
|
||||
"numforecasts": Number(result.number_of_predictions),
|
||||
"resolution_data": {
|
||||
"publish_time": result.publish_time,
|
||||
"resolution": result.resolution,
|
||||
|
|
|
|||
23
src/utils/algolia.js
Normal file
23
src/utils/algolia.js
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import algoliasearch from 'algoliasearch';
|
||||
import fs from "fs"
|
||||
|
||||
import { mongoReadWithReadCredentials } from "./mongo-wrapper.js"
|
||||
|
||||
const client = algoliasearch('96UD3NTQ7L', process.env.ALGOLIA_MASTER_API_KEY); // delete this when committing
|
||||
const index = client.initIndex('metaforecast');
|
||||
|
||||
export async function rebuildAlgoliaDatabase(){
|
||||
let records = await mongoReadWithReadCredentials("metaforecasts")
|
||||
// let string = JSON.stringify(json, null, 2)
|
||||
// fs.writeFileSync('metaforecasts.json', string);
|
||||
records = records.map(record => ({...record, has_numforecasts: record.has_numforecasts ? true : false}) )
|
||||
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
|
||||
|
||||
await index.clearObjects()
|
||||
index.saveObjects(records, { autoGenerateObjectIDIfNotExist: true }).then(() =>
|
||||
console.log("algolia search: done")
|
||||
).catch(error => {
|
||||
console.log("algolia search: error", error)
|
||||
})
|
||||
}
|
||||
// main()
|
||||
|
|
@ -20,6 +20,7 @@ import {smarkets} from "../platforms/smarkets-fetch.js"
|
|||
import {williamhill} from "../platforms/williamhill-fetch.js"
|
||||
import {mergeEverything} from "./mergeEverything.js"
|
||||
import {updateHistory} from "./history/updateHistory.js"
|
||||
import {rebuildAlgoliaDatabase} from "./algolia.js"
|
||||
import {rebuildNetlifySiteWithNewData} from "./rebuildNetliftySiteWithNewData.js"
|
||||
|
||||
/* Do everything */
|
||||
|
|
@ -39,7 +40,7 @@ export async function tryCatchTryAgain (fun) {
|
|||
}
|
||||
}
|
||||
export async function doEverything(){
|
||||
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildNetlifySiteWithNewData]
|
||||
let functions = [betfair, coupcast, csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, /* goodjudgment, */ goodjudgmentopen, hypermind, ladbrokes, kalshi, metaculus, omen, polymarket, predictit, rootclaim, smarkets, williamhill, mergeEverything, updateHistory, rebuildAlgoliaDatabase, rebuildNetlifySiteWithNewData]
|
||||
// Removed Good Judgment from the fetcher, doing it using cron instead because cloudflare blocks the utility on heroku.
|
||||
|
||||
console.log("")
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ let main = async () => {
|
|||
let string = JSON.stringify(json, null, 2)
|
||||
fs.writeFileSync('metaforecasts.json', string);
|
||||
}
|
||||
// main()
|
||||
main()
|
||||
|
||||
let extractQualityIndicators = async () => {
|
||||
let json = await mongoReadWithReadCredentials("metaforecasts")
|
||||
|
|
@ -18,4 +18,4 @@ let extractQualityIndicators = async () => {
|
|||
// let string = JSON.stringify(json, null, 2)
|
||||
// fs.writeFileSync('metaforecasts.json', string);
|
||||
}
|
||||
extractQualityIndicators()
|
||||
// extractQualityIndicators()
|
||||
Loading…
Reference in New Issue
Block a user