diff --git a/Procfile b/Procfile
index 37e372c..be48677 100644
--- a/Procfile
+++ b/Procfile
@@ -1 +1,3 @@
-worker: node src/utils/doEverythingForScheduler.js
+// worker: node src/utils/doEverythingForScheduler.js
+worker: node src/utils/testLoopHypothesis.js
+
diff --git a/src/platforms/goodjudmentopen-fetch.js b/src/platforms/goodjudmentopen-fetch.js
index 57da923..2847509 100644
--- a/src/platforms/goodjudmentopen-fetch.js
+++ b/src/platforms/goodjudmentopen-fetch.js
@@ -8,6 +8,7 @@ import { upsert } from "../utils/mongo-wrapper.js"
/* Definitions */
let htmlEndPoint = 'https://www.gjopen.com/questions?page='
+let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"]
/* Support functions */
@@ -27,10 +28,8 @@ function getcookie() {
process.exit()
}
*/
-
}
-
async function fetchPage(page, cookie) {
let response = await axios({
url: htmlEndPoint + page,
@@ -147,34 +146,38 @@ export async function goodjudgmentopen() {
// console.log(`Page #${i}`)
let htmlLines = response.split("\n")
let h5elements = htmlLines.filter(str => str.includes("
', "")
+ await sleep(1000 + Math.random() * 1000) // don't be as noticeable
+ try {
+ let moreinfo = await fetchStats(url, cookie)
+ if (moreinfo.isbinary) {
+ if (!moreinfo.crowdpercentage) { // then request again.
+ moreinfo = await fetchStats(url, cookie)
+ }
}
+ let question = ({
+ "title": title,
+ "url": url,
+ "platform": "Good Judgment Open",
+ ...moreinfo
+ })
+ if(j % 10 == 0){
+ console.log(`Page #${i}`)
+ console.log(question)
+ }
+ // console.log(question)
+ results.push(question)
+ } catch (error) {
+ console.log(error)
+ console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
}
- let question = ({
- "title": title,
- "url": url,
- "platform": "Good Judgment Open",
- ...moreinfo
- })
- if(i % 10 == 0){
- console.log(`Page #${i}`)
- console.log(question)
- }
- // console.log(question)
- results.push(question)
- } catch (error) {
- console.log(error)
- console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`)
}
+ j = j+1
}
i = i + 1
// console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers")
diff --git a/src/platforms/predictit-fetch.js b/src/platforms/predictit-fetch.js
index 179c132..60ea8f4 100644
--- a/src/platforms/predictit-fetch.js
+++ b/src/platforms/predictit-fetch.js
@@ -84,7 +84,7 @@ export async function predictit() {
}
})
- console.log(obj)
+ // console.log(obj)
results.push(obj)
}
//console.log(results)
diff --git a/src/utils/doEverything.js b/src/utils/doEverything.js
index 02f0751..0888710 100644
--- a/src/utils/doEverything.js
+++ b/src/utils/doEverything.js
@@ -35,8 +35,25 @@ export async function tryCatchTryAgain (fun) {
export async function doEverything(){
let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything]
+ console.log("")
+ console.log("")
+ console.log("")
+ console.log("")
+ console.log("================================")
+ console.log("STARTING UP")
+ console.log("================================")
+ console.log("")
+ console.log("")
+ console.log("")
+ console.log("")
+
for(let fun of functions){
+ console.log("")
+ console.log("")
+ console.log("****************************")
console.log(fun.name)
+ console.log("****************************")
await tryCatchTryAgain(fun)
+ console.log("****************************")
}
}
\ No newline at end of file
diff --git a/src/utils/mongo-wrapper.js b/src/utils/mongo-wrapper.js
index 680a308..8db08cc 100644
--- a/src/utils/mongo-wrapper.js
+++ b/src/utils/mongo-wrapper.js
@@ -30,7 +30,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca
const myDocument = await collection.findOne(filter);
// Print to the console
- console.log(myDocument.contentsArray.slice(0,10));
+ console.log(myDocument.contentsArray.slice(0,3));
} catch (err) {
console.log(err.stack);
}
diff --git a/src/utils/testLoopHypothesis.js b/src/utils/testLoopHypothesis.js
new file mode 100644
index 0000000..f8b90ea
--- /dev/null
+++ b/src/utils/testLoopHypothesis.js
@@ -0,0 +1,9 @@
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+let loop = async () => {
+ for (let i = 0; i < 6; i++) {
+ console.log(i)
+ await sleep(1000)
+ }
+}
+loop()