From f092c800ab94bafc954b77a10656c8d54df041f9 Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Wed, 14 Apr 2021 18:49:05 +0200 Subject: [PATCH] Updated database explanations --- .gitignore | 2 - README.md | 10 +-- src/utils/manualDownloadFromMongo.js | 10 +++ .../misc/process-forecasts-into-elicit.js | 76 ++++++++++--------- 4 files changed, 53 insertions(+), 45 deletions(-) create mode 100644 src/utils/manualDownloadFromMongo.js diff --git a/.gitignore b/.gitignore index 04b2539..ead0f31 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,3 @@ node_modules/ possiblenewsources.md herokulocation.md - -**/manualDownloadFromMongo.js diff --git a/README.md b/README.md index 650a565..b6b158f 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ This is a set of libraries and a command line interface that fetches probabilities/forecasts from prediction markets and forecasting platforms. -These forecasts are then used to power a search engine for probabilities, which can be found [here](https://metaforecast.org/) (try searching "Trump", "China" or "Semiconductors") (source code [here](https://github.com/QURIresearch/metaforecast-website-nextjs)). A json endpoint can be found [here](https://metaforecast.org/data/metaforecasts.json). +These forecasts are then used to power a search engine for probabilities, which can be found [here](https://metaforecast.org/) (try searching "Trump", "China" or "Semiconductors") (source code [here](https://github.com/QURIresearch/metaforecast-website-nextjs)). I also provide a datatabase, which can be accessed with a script similar to [this one](https://github.com/QURIresearch/metaforecasts/blob/master/src/utils/manualDownloadFromMongo.js). -I also created a search engine using Elicit's IDE, which uses GPT-3 to deliver vastly superior semantic search (as opposed to fuzzy word matching). If you have access to the Elicit IDE, you can use the action "Search Metaforecast database". +I also created a search engine using Elicit's IDE, which uses GPT-3 to deliver vastly superior semantic search (as opposed to fuzzy word matching). If you have access to the Elicit IDE, you can use the action "Search Metaforecast database". However, I'm not currently updating it regularly. ![](./metaforecasts.png) @@ -15,16 +15,14 @@ I also created a search engine using Elicit's IDE, which uses GPT-3 to deliver v ``git clone https://github.com/QURIresearch/metaforecasts`` ### 2. Enter your own process.env variables -The following environment variables are currently needed to run the `master` branch: +The following variables are currently needed to run the `master` branch: - `MONGODB_URL`, a string in the format `"mongodb+srv://:@/?retryWrites=true&w=majority&useNewUrlParser=true&useUnifiedTopology=true"` - `REBUIDNETLIFYHOOKURL`, a string in the format `"https://api.netlify.com/build_hooks/someprivatestring"` - `CSETFORETELL_COOKIE` - `GOODJUDGMENTOPENCOOKIE` - `HYPERMINDCOOKIE` -The cookie formats can be found in `src/input/privatekeys_example.json`; these session cookies are necessary to query CSET-foretell, Good Judgment Open and Hypermind. You can get these cookies by creating an account in said platforms and then making and inspecting a request (e.g., by making a prediction, or browsing questions). After doing this, you should create the environment variables. - -Alternatively, for fewer complications, have a look at the `commandlineinterface` branch, which instead of requiring environment variables only requires a `src/privatekeys.json`, in the same format as its `src/privatekeys_example.json`. Its disadvantages are that the command line tool in the `commandlineinterface` is more difficult to integrate with other services. +They can either be stored as process variables (e.g., something that can be accessed as `process.env.`), or as text in `src/input/privatekeys.json`, in the same format as `src/input/privatekeys_example.json`. These session cookies are necessary to query CSET-foretell, Good Judgment Open and Hypermind, and to access the MongoDB database I'm using to save data and history. You can get these cookies by creating an account in said platforms and then making and inspecting a request (e.g., by making a prediction, or browsing questions). After doing this, you should create the environment variables. ### 3. Actually run diff --git a/src/utils/manualDownloadFromMongo.js b/src/utils/manualDownloadFromMongo.js new file mode 100644 index 0000000..115e93a --- /dev/null +++ b/src/utils/manualDownloadFromMongo.js @@ -0,0 +1,10 @@ +import fs from "fs" + +import { mongoReadWithReadCredentials } from "./mongo-wrapper.js" + +let main = async () => { + let json = await mongoReadWithReadCredentials("metaforecasts") + let string = JSON.stringify(json, null, 2) + fs.writeFileSync('metaforecasts.json', string); +} +main() diff --git a/src/utils/misc/process-forecasts-into-elicit.js b/src/utils/misc/process-forecasts-into-elicit.js index c9624f6..281312d 100644 --- a/src/utils/misc/process-forecasts-into-elicit.js +++ b/src/utils/misc/process-forecasts-into-elicit.js @@ -1,48 +1,50 @@ /* Imports */ import fs from "fs" +import { mongoReadWithReadCredentials } from "../mongo-wrapper.js" /* Definitions */ let locationData = "./data/" /* Body */ -let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src -let data = JSON.parse(rawdata) - -let processDescription = (description) => { - if(description == null || description == undefined || description == ""){ - return "" - }else{ - description = description==null?"":description - .replaceAll("] (", "](") - .replaceAll(") )", "))") - .replaceAll("( [", "([") - .replaceAll(") ,", "),") - .replaceAll("\n", " ") - if(description.length > 1000){ - return(description.slice(0,1000)+"...") +// let rawdata = fs.readFileSync("./data/merged-questions.json") // run from topmost folder, not from src +async function main(){ + let data = await mongoReadWithReadCredentials("metaforecasts") //JSON.parse(rawdata) + let processDescription = (description) => { + if(description == null || description == undefined || description == ""){ + return "" }else{ - return(description) + description = description==null?"":description + .replaceAll("] (", "](") + .replaceAll(") )", "))") + .replaceAll("( [", "([") + .replaceAll(") ,", "),") + .replaceAll("\n", " ") + if(description.length > 1000){ + return(description.slice(0,1000)+"...") + }else{ + return(description) + } } } + + let results = [] + for(let datum of data){ + // do something + let description = processDescription(datum["description"]) + let forecasts = datum["qualityindicators"] ? datum["qualityindicators"].numforecasts : "unknown" + let stars = datum["qualityindicators"] ? datum["qualityindicators"].stars : 2 + results.push("Title: "+datum["title"]) + results.push("URL: "+datum["url"]) + results.push("Platform: "+datum["platform"]) + results.push("Description: "+description) + results.push("Number of forecasts: "+ forecasts) + results.push("Stars: "+forecasts) + results.push("\n") + } + + let string = results.join("\n") + string = string.replaceAll("\n\n", "\n") + + fs.writeFileSync("elicit-output.txt", string) } - -let results = [] -for(let datum of data){ - // do something - let description = processDescription(datum["Description"]) - let forecasts = datum["# Forecasts"] || "unknown" - results.push("Title: "+datum["Title"]) - results.push("URL: "+datum["URL"]) - results.push("Platform: "+datum["Platform"]) - results.push("Binary question?: "+datum["Binary question?"]) - results.push("Percentage: "+datum["Percentage"]) - results.push("Description: "+description) - results.push("# Forecasts: "+ forecasts) - results.push("Stars: "+datum["Stars"]) - results.push("\n") -} - -let string = results.join("\n") -string = string.replaceAll("\n\n", "\n") - -fs.writeFileSync("./data/elicit-output.txt", string) +main()