diff --git a/src/platforms/csetforetell-fetch.js b/src/platforms/csetforetell-fetch.js index 9f9c77f..50115e7 100644 --- a/src/platforms/csetforetell-fetch.js +++ b/src/platforms/csetforetell-fetch.js @@ -9,15 +9,18 @@ import {upsert} from "../utils/mongo-wrapper.js" /* Definitions */ let htmlEndPoint = 'https://www.cset-foretell.com/questions?page=' String.prototype.replaceAll = function replaceAll(search, replace) { return this.split(search).join(replace); } - +const DEBUG_MODE = "off" // "on" /* Support functions */ async function fetchPage(page, cookie){ + console.log(page) if(page==1){ cookie=cookie.split(";")[0] // Interesting that it otherwise doesn't work :( } + let urlEndpoint = htmlEndPoint+page + console.log(urlEndpoint) let response = await axios({ - url: htmlEndPoint+page, + url: urlEndpoint, method: 'GET', headers: ({ 'Content-Type': 'text/html', @@ -123,7 +126,12 @@ async function fetchStats(questionUrl, cookie){ } function isEnd(html){ - return html.includes("No questions match your filter") + let isEndBool = html.includes("No questions match your filter") + if(isEndBool){ + //console.log(html) + } + console.log(`IsEnd? ${isEndBool}`) + return isEndBool } function sleep(ms) { @@ -133,7 +141,7 @@ function sleep(ms) { /* Body */ async function csetforetell_inner(cookie){ - let i=0 + let i=1 let response = await fetchPage(i, cookie) let results = [] let init = Date.now() @@ -143,8 +151,9 @@ async function csetforetell_inner(cookie){ let htmlLines = response.split("\n") let h4elements = htmlLines.filter(str => str.includes("
') let url = h4elementSplit[0].split('