diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js index 50115e7..3fec3c3 100644 --- a/src/platforms/csetforetell-fetch.js +++ b/src/platforms/csetforetell-fetch.js @@ -45,6 +45,10 @@ async function fetchStats(questionUrl, cookie){ .then(res => res.data) //console.log(response) + if(response.includes("Sign up or sign in to forecast")){ + throw Error("Not logged in") + } + // Is binary? let isbinary = response.includes("binary?":true") // console.log(`is binary? ${isbinary}`) @@ -104,6 +108,8 @@ async function fetchStats(questionUrl, cookie){ let description = descriptionprocessed8 // Number of forecasts + //console.log(response) + //console.log(response.split("prediction_sets_count":")[1]) let numforecasts = response.split("prediction_sets_count":")[1].split(",")[0] // console.log(numforecasts) @@ -125,6 +131,16 @@ async function fetchStats(questionUrl, cookie){ return result } +function isNotSignedIn(html){ + + let isNotSignedInBool = html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") + if(isNotSignedInBool){ + console.log("Error: Not signed in.") + } + console.log(`isNotSignedIn? ${isNotSignedInBool}`) + return isNotSignedInBool +} + function isEnd(html){ let isEndBool = html.includes("No questions match your filter") if(isEndBool){ @@ -146,7 +162,7 @@ async function csetforetell_inner(cookie){ let results = [] let init = Date.now() // console.log("Downloading... This might take a couple of minutes. Results will be shown.") - while(!isEnd(response)){ + while(!isEnd(response) && !isNotSignedIn(response)){ let htmlLines = response.split("\n") let h4elements = htmlLines.filter(str => str.includes("
0){ + await upsert(results, "csetforetell-questions") + }else{ + console.log("Not updating results, as process was not signed in") + } let end = Date.now() let difference = end-init diff --git a/src/platforms/foretold-fetch.js b/src/platforms/foretold-fetch.js index f8fe931..1ecd312 100644 --- a/src/platforms/foretold-fetch.js +++ b/src/platforms/foretold-fetch.js @@ -6,7 +6,7 @@ import { upsert } from "../utils/mongo-wrapper.js" /* Definitions */ let graphQLendpoint = "https://api.foretold.io/graphql" -let highQualityCommunities = ["0104d8e8-07e4-464b-8b32-74ef22b49f21", "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", "cf663021-f87f-4632-ad82-962d889a2d39", "47ff5c49-9c20-4f3d-bd57-1897c35cd42d"] +let highQualityCommunities = ["0104d8e8-07e4-464b-8b32-74ef22b49f21", "c47c6bc8-2c9b-4a83-9583-d1ed80a40fa2", "cf663021-f87f-4632-ad82-962d889a2d39", "47ff5c49-9c20-4f3d-bd57-1897c35cd42d", "b2412a1d-0aa4-4e37-a12a-0aca9e440a96"] /* Support functions */ async function fetchAllCommunityQuestions(communityId) { diff --git a/src/platforms/goodjudmentopen-fetch.js b/src/platforms/goodjudmentopen-fetch.js index a062c19..e0a7b09 100644 --- a/src/platforms/goodjudmentopen-fetch.js +++ b/src/platforms/goodjudmentopen-fetch.js @@ -107,8 +107,23 @@ async function fetchStats(questionUrl, cookie) { return result } -function isEnd(html) { - return html.includes("No questions match your filter") +function isNotSignedIn(html){ + + let isNotSignedInBool = html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") + if(isNotSignedIn){ + console.log("Error: Not signed in.") + } + console.log(`isNotSignedIn? ${isNotSignedInBool}`) + return isNotSignedInBool +} + +function isEnd(html){ + let isEndBool = html.includes("No questions match your filter") + if(isEndBool){ + //console.log(html) + } + console.log(`IsEnd? ${isEndBool}`) + return isEndBool } function sleep(ms) { @@ -123,7 +138,7 @@ async function goodjudgmentopen_inner(cookie) { let results = [] let init = Date.now() // console.log("Downloading... This might take a couple of minutes. Results will be shown.") - while (!isEnd(response)) { + while(!isEnd(response) && !isNotSignedIn(response)){ // console.log(`Page #${i}`) let htmlLines = response.split("\n") let h5elements = htmlLines.filter(str => str.includes("
0){ + await upsert(results, "goodjudmentopen-questions") + }else{ + console.log("Not updating results, as process was not signed in") + } let end = Date.now() let difference = end - init diff --git a/src/utils/mongo-wrapper.js b/src/utils/mongo-wrapper.js index d7ffb4f..46db5cf 100644 --- a/src/utils/mongo-wrapper.js +++ b/src/utils/mongo-wrapper.js @@ -182,5 +182,5 @@ export async function mongoGetAllElements(databaseName = "metaforecastDatabase", } } -// mongoGetAllElements() -//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory") \ No newline at end of file +//mongoGetAllElements() +//mongoGetAllElements("metaforecastDatabase", "metaforecastHistory")