From 89c29053d4b765dc162b5da93a39de402c87d325 Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Thu, 8 Apr 2021 22:51:02 +0200 Subject: [PATCH] Test loop hypothesis --- Procfile | 4 +- src/platforms/goodjudmentopen-fetch.js | 51 ++++++++++++++------------ src/platforms/predictit-fetch.js | 2 +- src/utils/doEverything.js | 17 +++++++++ src/utils/mongo-wrapper.js | 2 +- src/utils/testLoopHypothesis.js | 9 +++++ 6 files changed, 58 insertions(+), 27 deletions(-) create mode 100644 src/utils/testLoopHypothesis.js diff --git a/Procfile b/Procfile index 37e372c..be48677 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,3 @@ -worker: node src/utils/doEverythingForScheduler.js +// worker: node src/utils/doEverythingForScheduler.js +worker: node src/utils/testLoopHypothesis.js + diff --git a/src/platforms/goodjudmentopen-fetch.js b/src/platforms/goodjudmentopen-fetch.js index 57da923..2847509 100644 --- a/src/platforms/goodjudmentopen-fetch.js +++ b/src/platforms/goodjudmentopen-fetch.js @@ -8,6 +8,7 @@ import { upsert } from "../utils/mongo-wrapper.js" /* Definitions */ let htmlEndPoint = 'https://www.gjopen.com/questions?page=' +let annoyingPromptUrls = ["https://www.gjopen.com/questions/1933-what-forecasting-questions-should-we-ask-what-questions-would-you-like-to-forecast-on-gjopen", "https://www.gjopen.com/questions/1779-are-there-any-forecasting-tips-tricks-and-experiences-you-would-like-to-share-and-or-discuss-with-your-fellow-forecasters"] /* Support functions */ @@ -27,10 +28,8 @@ function getcookie() { process.exit() } */ - } - async function fetchPage(page, cookie) { let response = await axios({ url: htmlEndPoint + page, @@ -147,34 +146,38 @@ export async function goodjudgmentopen() { // console.log(`Page #${i}`) let htmlLines = response.split("\n") let h5elements = htmlLines.filter(str => str.includes("
') let url = h5elementSplit[0].split('
', "") + await sleep(1000 + Math.random() * 1000) // don't be as noticeable + try { + let moreinfo = await fetchStats(url, cookie) + if (moreinfo.isbinary) { + if (!moreinfo.crowdpercentage) { // then request again. + moreinfo = await fetchStats(url, cookie) + } } + let question = ({ + "title": title, + "url": url, + "platform": "Good Judgment Open", + ...moreinfo + }) + if(j % 10 == 0){ + console.log(`Page #${i}`) + console.log(question) + } + // console.log(question) + results.push(question) + } catch (error) { + console.log(error) + console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`) } - let question = ({ - "title": title, - "url": url, - "platform": "Good Judgment Open", - ...moreinfo - }) - if(i % 10 == 0){ - console.log(`Page #${i}`) - console.log(question) - } - // console.log(question) - results.push(question) - } catch (error) { - console.log(error) - console.log(`We encountered some error when fetching the URL: ${url}, so it won't appear on the final json`) } + j = j+1 } i = i + 1 // console.log("Sleeping for 5secs so as to not be as noticeable to the gjopen servers") diff --git a/src/platforms/predictit-fetch.js b/src/platforms/predictit-fetch.js index 179c132..60ea8f4 100644 --- a/src/platforms/predictit-fetch.js +++ b/src/platforms/predictit-fetch.js @@ -84,7 +84,7 @@ export async function predictit() { } }) - console.log(obj) + // console.log(obj) results.push(obj) } //console.log(results) diff --git a/src/utils/doEverything.js b/src/utils/doEverything.js index 02f0751..0888710 100644 --- a/src/utils/doEverything.js +++ b/src/utils/doEverything.js @@ -35,8 +35,25 @@ export async function tryCatchTryAgain (fun) { export async function doEverything(){ let functions = [csetforetell, elicit, /* estimize, */ fantasyscotus, foretold, goodjudgment, goodjudgmentopen, hypermind, ladbrokes, metaculus, polymarket, predictit, omen, smarkets, williamhill, mergeEverything] + console.log("") + console.log("") + console.log("") + console.log("") + console.log("================================") + console.log("STARTING UP") + console.log("================================") + console.log("") + console.log("") + console.log("") + console.log("") + for(let fun of functions){ + console.log("") + console.log("") + console.log("****************************") console.log(fun.name) + console.log("****************************") await tryCatchTryAgain(fun) + console.log("****************************") } } \ No newline at end of file diff --git a/src/utils/mongo-wrapper.js b/src/utils/mongo-wrapper.js index 680a308..8db08cc 100644 --- a/src/utils/mongo-wrapper.js +++ b/src/utils/mongo-wrapper.js @@ -30,7 +30,7 @@ export async function upsert (contents, documentName, collectionName="metaforeca const myDocument = await collection.findOne(filter); // Print to the console - console.log(myDocument.contentsArray.slice(0,10)); + console.log(myDocument.contentsArray.slice(0,3)); } catch (err) { console.log(err.stack); } diff --git a/src/utils/testLoopHypothesis.js b/src/utils/testLoopHypothesis.js new file mode 100644 index 0000000..f8b90ea --- /dev/null +++ b/src/utils/testLoopHypothesis.js @@ -0,0 +1,9 @@ +let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + +let loop = async () => { + for (let i = 0; i < 6; i++) { + console.log(i) + await sleep(1000) + } +} +loop()