Basic autocomplete functionality added

Still need to add support to the opensearch xml template, but otherwise
the main page functionality is working as expected

Adds new route '/autocomplete' that accepts a string query for both GET and POST requests, and returns an array of suggestions

Adds GET and POST tests for autocomplete search as well

app/request.py

@@ -1,10 +1,12 @@
 from io import BytesIO
+from lxml import etree
 import pycurl
 import random
 import urllib.parse as urlparse
 
-# Base search url
+# Core Google search URLs
 SEARCH_URL = 'https://www.google.com/search?gbv=1&q='
+AUTOCOMPLETE_URL = 'https://suggestqueries.google.com/complete/search?client=toolbar&'
 
 MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
 DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
@@ -76,6 +78,16 @@ class Request:
         else:
             return 'unicode-escape'
 
+    def autocomplete(self, query):
+        ac_query = dict(hl=self.language, q=query)
+        response = self.send(base_url=AUTOCOMPLETE_URL, query=urlparse.urlencode(ac_query))
+
+        if response:
+            dom = etree.fromstring(response)
+            return dom.xpath('//suggestion/@data')
+
+        return []
+
     def send(self, base_url=SEARCH_URL, query='', return_bytes=False):
         response_header = []
 

app/routes.py

@@ -6,7 +6,7 @@ import argparse
 import base64
 from bs4 import BeautifulSoup
 from cryptography.fernet import Fernet, InvalidToken
-from flask import g, make_response, request, redirect, render_template, send_file
+from flask import g, jsonify, make_response, request, redirect, render_template, send_file
 from functools import wraps
 import io
 import json
@@ -88,6 +88,17 @@ def opensearch():
     return response
 
 
+@app.route('/autocomplete', methods=['GET', 'POST'])
+def autocomplete():
+    request_params = request.args if request.method == 'GET' else request.form
+    q = request_params.get('q')
+
+    if not q:
+        return jsonify({'results': []})
+
+    return jsonify({'results': g.user_request.autocomplete(q)})
+
+
 @app.route('/search', methods=['GET', 'POST'])
 @auth_required
 def search():

app/static/css/search.css

@@ -0,0 +1,34 @@
+.autocomplete {
+    position: relative;
+    display: inline-block;
+    width: 100%;
+}
+
+.autocomplete-items {
+    position: absolute;
+    border: 1px solid #d4d4d4;
+    border-bottom: none;
+    border-top: none;
+    z-index: 99;
+
+    /*position the autocomplete items to be the same width as the container:*/
+    top: 100%;
+    left: 0;
+    right: 0;
+}
+
+.autocomplete-items div {
+    padding: 10px;
+    cursor: pointer;
+    background-color: #fff;
+    border-bottom: 1px solid #d4d4d4;
+}
+
+.autocomplete-items div:hover {
+    background-color: #e9e9e9;
+}
+
+.autocomplete-active {
+    background-color: #685e79 !important;
+    color: #ffffff;
+}

app/static/js/autocomplete.js

@@ -0,0 +1,79 @@
+function autocomplete(searchInput, autocompleteResults) {
+    let currentFocus;
+
+    searchInput.addEventListener("input", function () {
+        let autocompleteList, autocompleteItem, i, val = this.value;
+        closeAllLists();
+
+        if (!val) {
+            return false;
+        }
+
+        currentFocus = -1;
+        autocompleteList = document.createElement("div");
+        autocompleteList.setAttribute("id", this.id + "-autocomplete-list");
+        autocompleteList.setAttribute("class", "autocomplete-items");
+        this.parentNode.appendChild(autocompleteList);
+
+        for (i = 0; i < autocompleteResults.length; i++) {
+            if (autocompleteResults[i].substr(0, val.length).toUpperCase() === val.toUpperCase()) {
+                autocompleteItem = document.createElement("div");
+                autocompleteItem.innerHTML = "<strong>" + autocompleteResults[i].substr(0, val.length) + "</strong>";
+                autocompleteItem.innerHTML += autocompleteResults[i].substr(val.length);
+                autocompleteItem.innerHTML += "<input type='hidden' value='" + autocompleteResults[i] + "'>";
+                autocompleteItem.addEventListener("click", function () {
+                    searchInput.value = this.getElementsByTagName("input")[0].value;
+                    closeAllLists();
+                });
+                autocompleteList.appendChild(autocompleteItem);
+            }
+        }
+    });
+
+    searchInput.addEventListener("keydown", function (e) {
+        let suggestion = document.getElementById(this.id + "-autocomplete-list");
+        if (suggestion) suggestion = suggestion.getElementsByTagName("div");
+        if (e.keyCode === 40) { // down
+            currentFocus++;
+            addActive(suggestion);
+        } else if (e.keyCode === 38) { //up
+            currentFocus--;
+            addActive(suggestion);
+        } else if (e.keyCode === 13) { // enter
+            e.preventDefault();
+            if (currentFocus > -1) {
+                if (suggestion) suggestion[currentFocus].click();
+            }
+        }
+    });
+
+    function addActive(suggestion) {
+        if (!suggestion || !suggestion[currentFocus]) return false;
+        removeActive(suggestion);
+
+        if (currentFocus >= suggestion.length) currentFocus = 0;
+        if (currentFocus < 0) currentFocus = (suggestion.length - 1);
+
+        suggestion[currentFocus].classList.add("autocomplete-active");
+    }
+
+    function removeActive(suggestion) {
+        for (let i = 0; i < suggestion.length; i++) {
+            suggestion[i].classList.remove("autocomplete-active");
+        }
+    }
+
+    function closeAllLists(el) {
+        let suggestions = document.getElementsByClassName("autocomplete-items");
+        for (let i = 0; i < suggestions.length; i++) {
+            if (el !== suggestions[i] && el !== searchInput) {
+                suggestions[i].parentNode.removeChild(suggestions[i]);
+            }
+        }
+    }
+
+    // Close lists and search when user selects a suggestion
+    document.addEventListener("click", function (e) {
+        closeAllLists(e.target);
+    });
+}

app/static/js/controller.js

@@ -1,3 +1,21 @@
+const handleUserInput = searchBar => {
+    let xhrRequest = new XMLHttpRequest();
+    xhrRequest.open("POST", "/autocomplete");
+    xhrRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
+    xhrRequest.onload = function() {
+        if (xhrRequest.readyState === 4 && xhrRequest.status !== 200) {
+            alert("Error fetching autocomplete results");
+            return;
+        }
+
+        // Fill autocomplete with fetched results
+        let autocompleteResults = JSON.parse(xhrRequest.responseText);
+        autocomplete(searchBar, autocompleteResults["results"]);
+    };
+
+    xhrRequest.send('q=' + searchBar.value);
+};
+
 const setupSearchLayout = () => {
     // Setup search field
     const searchBar = document.getElementById("search-bar");
@@ -11,6 +29,8 @@ const setupSearchLayout = () => {
         if (event.keyCode === 13) {
             event.preventDefault();
             searchBtn.click();
+        } else {
+            handleUserInput(searchBar);
         }
     });
 };

app/templates/index.html

@@ -17,9 +17,11 @@
         <meta name="referrer" content="no-referrer">
         <meta name="msapplication-TileColor" content="#ffffff">
         <meta name="msapplication-TileImage" content="/static/img/favicon/ms-icon-144x144.png">
+        <script type="text/javascript" src="/static/js/autocomplete.js"></script>
         <script type="text/javascript" src="/static/js/controller.js"></script>
         <link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Whoogle Search">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <link rel="stylesheet" href="/static/css/search.css">
         <link rel="stylesheet" href="/static/css/main.css">
 	<title>Whoogle Search</title>
     </head>
@@ -28,7 +30,9 @@
             <img class="logo" src="/static/img/logo.png">
             <form action="/search" method="{{ request_type }}">
                 <div class="search-fields">
-                    <input type="text" name="q" id="search-bar" autofocus="autofocus">
+                    <div class="autocomplete">
+                        <input type="text" name="q" id="search-bar" autofocus="autofocus">
+                    </div>
                     <input type="submit" id="search-submit" value="Search">
                 </div>
             </form>

test/test_autocomplete.py

@@ -0,0 +1,20 @@
+from bs4 import BeautifulSoup
+from cryptography.fernet import Fernet
+from app.filter import Filter
+from datetime import datetime
+from dateutil.parser import *
+
+
+def test_autocomplete_get(client):
+    rv = client.get('/autocomplete?q=green+eggs+and')
+    assert rv._status_code == 200
+    assert len(rv.data) >= 1
+    assert b'green eggs and ham' in rv.data
+
+
+def test_autocomplete_post(client):
+    rv = client.post('/autocomplete', data=dict(q='the+cat+in+the'))
+    assert rv._status_code == 200
+    assert len(rv.data) >= 1
+    assert b'the cat in the hat' in rv.data
+