chore: tweaks

This commit is contained in:
NunoSempere 2021-08-08 19:42:21 +02:00
parent e2adcf95f4
commit 1980ac53e3
3 changed files with 12896 additions and 13004 deletions

File diff suppressed because one or more lines are too long

View File

@ -107,14 +107,15 @@ async function fetchStats(questionUrl, cookie) {
return result
}
function isNotSignedIn(html){
function isSignedIn(html){
let isNotSignedInBool = html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up")
if(isNotSignedIn){
let isSignedInBool = !( html.includes("You need to sign in or sign up before continuing") || html.includes("Sign up") )
// console.log(html)
if(!isSignedInBool){
console.log("Error: Not signed in.")
}
console.log(`isNotSignedIn? ${isNotSignedInBool}`)
return isNotSignedInBool
console.log(`is signed in? ${isSignedInBool}`)
return isSignedInBool
}
function isEnd(html){
@ -138,7 +139,7 @@ async function goodjudgmentopen_inner(cookie) {
let results = []
let init = Date.now()
// console.log("Downloading... This might take a couple of minutes. Results will be shown.")
while(!isEnd(response) && !isNotSignedIn(response)){
while(!isEnd(response) && isSignedIn(response)){
// console.log(`Page #${i}`)
let htmlLines = response.split("\n")
let h5elements = htmlLines.filter(str => str.includes("<h5><a href="))

View File

@ -10,14 +10,33 @@ export async function rebuildAlgoliaDatabase(){
let records = await mongoReadWithReadCredentials("metaforecasts")
// let string = JSON.stringify(json, null, 2)
// fs.writeFileSync('metaforecasts.json', string);
records = records.map(record => ({...record, has_numforecasts: record.has_numforecasts ? true : false}) )
records = records.map((record, index) => ({...record, has_numforecasts: record.numforecasts ? true : false, objectID: index}) )
// this is necessary to filter by missing attributes https://www.algolia.com/doc/guides/managing-results/refine-results/filtering/how-to/filter-by-null-or-missing-attributes/
await index.clearObjects()
/*
index.clearObjects().wait().then(response => {
console.log(response)
});
*/
if(index.exists()){
console.log("Index exists")
index.replaceAllObjects(records, { safe:true }).catch(error => console.log(error))
console.log(`Pushed ${records.length} records. Algolia will update asynchronously`)
}
/*await index.clearObjects()
console.log("Past data")
setTimeout(function(){
index.saveObjects(records, { autoGenerateObjectIDIfNotExist: true }).then(() =>
console.log("algolia search: done")
).catch(error => {
console.log("algolia search: error", error)
})
alert('hello');
}, 60*1000); // 1 minute seconds
*/
}
// main()
//rebuildAlgoliaDatabase()