/* Imports */
import fs from "fs"
import axios from "axios"
import toMarkdown from "./toMarkdown.js"
import {getstars} from "./stars.js"

/* Definitions */
let locationData = "./data/"

/* Support functions */
async function fetchPage(url){
  let response  = await axios({
    url: url,
    method: 'GET',
    headers: ({ 
    'Content-Type': 'text/html',
    }),
  })
  .then(res => res.data)
  //console.log(response)
  return response
}

/* Body */

async function main(){
  let rawdata = fs.readFileSync("./data/givewell-urls.txt")
  let data = rawdata.toString().split("\n").filter(url => url!="");
  console.log(data)
  let results = []
  for(let url of data){
    console.log(url)
    let page = await fetchPage(url) 
    
    // Title
    let titleraw = page.split('<meta name="twitter:title" content="')[1]
    let title = titleraw.split('" />')[0]
    
    // Description
    let internalforecasts = page.split("<h2").filter(section => section.includes("Internal forecast")||section.includes("internal forecast"))
    let description = "<h2 "+internalforecasts[1]
    
    let result = {
      "Title": title,
      "URL": url,
      "Platform": "GiveWell",
      "Binary question?": false,
      "Percentage": "none",
      "Description": description, 
      "Stars": getstars(3),
    }
    console.log(result)
    results.push(result)
  }
  let string = JSON.stringify(results,null,  2)
  fs.writeFileSync('./data/givewell-questions.json', string);
}
main()