From 1d0c63c217b3166f3171af9ee53851af9270d6e6 Mon Sep 17 00:00:00 2001
From: Ben Busby <33362396+benbusby@users.noreply.github.com>
Date: Sat, 23 May 2020 15:38:37 -0600
Subject: [PATCH] Basic autocomplete functionality added
Still need to add support to the opensearch xml template, but otherwise
the main page functionality is working as expected
Adds new route '/autocomplete' that accepts a string query for both GET and POST requests, and returns an array of suggestions
Adds GET and POST tests for autocomplete search as well
---
app/request.py | 14 ++++++-
app/routes.py | 13 +++++-
app/static/css/search.css | 34 +++++++++++++++
app/static/js/autocomplete.js | 79 +++++++++++++++++++++++++++++++++++
app/static/js/controller.js | 20 +++++++++
app/templates/index.html | 6 ++-
test/test_autocomplete.py | 20 +++++++++
7 files changed, 183 insertions(+), 3 deletions(-)
create mode 100644 app/static/css/search.css
create mode 100644 app/static/js/autocomplete.js
create mode 100644 test/test_autocomplete.py
diff --git a/app/request.py b/app/request.py
index 07b6351..2b2e9ea 100644
--- a/app/request.py
+++ b/app/request.py
@@ -1,10 +1,12 @@
from io import BytesIO
+from lxml import etree
import pycurl
import random
import urllib.parse as urlparse
-# Base search url
+# Core Google search URLs
SEARCH_URL = 'https://www.google.com/search?gbv=1&q='
+AUTOCOMPLETE_URL = 'https://suggestqueries.google.com/complete/search?client=toolbar&'
MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
@@ -76,6 +78,16 @@ class Request:
else:
return 'unicode-escape'
+ def autocomplete(self, query):
+ ac_query = dict(hl=self.language, q=query)
+ response = self.send(base_url=AUTOCOMPLETE_URL, query=urlparse.urlencode(ac_query))
+
+ if response:
+ dom = etree.fromstring(response)
+ return dom.xpath('//suggestion/@data')
+
+ return []
+
def send(self, base_url=SEARCH_URL, query='', return_bytes=False):
response_header = []
diff --git a/app/routes.py b/app/routes.py
index 3f0d04c..348a352 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -6,7 +6,7 @@ import argparse
import base64
from bs4 import BeautifulSoup
from cryptography.fernet import Fernet, InvalidToken
-from flask import g, make_response, request, redirect, render_template, send_file
+from flask import g, jsonify, make_response, request, redirect, render_template, send_file
from functools import wraps
import io
import json
@@ -88,6 +88,17 @@ def opensearch():
return response
+@app.route('/autocomplete', methods=['GET', 'POST'])
+def autocomplete():
+ request_params = request.args if request.method == 'GET' else request.form
+ q = request_params.get('q')
+
+ if not q:
+ return jsonify({'results': []})
+
+ return jsonify({'results': g.user_request.autocomplete(q)})
+
+
@app.route('/search', methods=['GET', 'POST'])
@auth_required
def search():
diff --git a/app/static/css/search.css b/app/static/css/search.css
new file mode 100644
index 0000000..a79522b
--- /dev/null
+++ b/app/static/css/search.css
@@ -0,0 +1,34 @@
+.autocomplete {
+ position: relative;
+ display: inline-block;
+ width: 100%;
+}
+
+.autocomplete-items {
+ position: absolute;
+ border: 1px solid #d4d4d4;
+ border-bottom: none;
+ border-top: none;
+ z-index: 99;
+
+ /*position the autocomplete items to be the same width as the container:*/
+ top: 100%;
+ left: 0;
+ right: 0;
+}
+
+.autocomplete-items div {
+ padding: 10px;
+ cursor: pointer;
+ background-color: #fff;
+ border-bottom: 1px solid #d4d4d4;
+}
+
+.autocomplete-items div:hover {
+ background-color: #e9e9e9;
+}
+
+.autocomplete-active {
+ background-color: #685e79 !important;
+ color: #ffffff;
+}
\ No newline at end of file
diff --git a/app/static/js/autocomplete.js b/app/static/js/autocomplete.js
new file mode 100644
index 0000000..f65dd77
--- /dev/null
+++ b/app/static/js/autocomplete.js
@@ -0,0 +1,79 @@
+function autocomplete(searchInput, autocompleteResults) {
+ let currentFocus;
+
+ searchInput.addEventListener("input", function () {
+ let autocompleteList, autocompleteItem, i, val = this.value;
+ closeAllLists();
+
+ if (!val) {
+ return false;
+ }
+
+ currentFocus = -1;
+ autocompleteList = document.createElement("div");
+ autocompleteList.setAttribute("id", this.id + "-autocomplete-list");
+ autocompleteList.setAttribute("class", "autocomplete-items");
+ this.parentNode.appendChild(autocompleteList);
+
+ for (i = 0; i < autocompleteResults.length; i++) {
+ if (autocompleteResults[i].substr(0, val.length).toUpperCase() === val.toUpperCase()) {
+ autocompleteItem = document.createElement("div");
+ autocompleteItem.innerHTML = "" + autocompleteResults[i].substr(0, val.length) + "";
+ autocompleteItem.innerHTML += autocompleteResults[i].substr(val.length);
+ autocompleteItem.innerHTML += "";
+ autocompleteItem.addEventListener("click", function () {
+ searchInput.value = this.getElementsByTagName("input")[0].value;
+ closeAllLists();
+ });
+ autocompleteList.appendChild(autocompleteItem);
+ }
+ }
+ });
+
+ searchInput.addEventListener("keydown", function (e) {
+ let suggestion = document.getElementById(this.id + "-autocomplete-list");
+ if (suggestion) suggestion = suggestion.getElementsByTagName("div");
+ if (e.keyCode === 40) { // down
+ currentFocus++;
+ addActive(suggestion);
+ } else if (e.keyCode === 38) { //up
+ currentFocus--;
+ addActive(suggestion);
+ } else if (e.keyCode === 13) { // enter
+ e.preventDefault();
+ if (currentFocus > -1) {
+ if (suggestion) suggestion[currentFocus].click();
+ }
+ }
+ });
+
+ function addActive(suggestion) {
+ if (!suggestion || !suggestion[currentFocus]) return false;
+ removeActive(suggestion);
+
+ if (currentFocus >= suggestion.length) currentFocus = 0;
+ if (currentFocus < 0) currentFocus = (suggestion.length - 1);
+
+ suggestion[currentFocus].classList.add("autocomplete-active");
+ }
+
+ function removeActive(suggestion) {
+ for (let i = 0; i < suggestion.length; i++) {
+ suggestion[i].classList.remove("autocomplete-active");
+ }
+ }
+
+ function closeAllLists(el) {
+ let suggestions = document.getElementsByClassName("autocomplete-items");
+ for (let i = 0; i < suggestions.length; i++) {
+ if (el !== suggestions[i] && el !== searchInput) {
+ suggestions[i].parentNode.removeChild(suggestions[i]);
+ }
+ }
+ }
+
+ // Close lists and search when user selects a suggestion
+ document.addEventListener("click", function (e) {
+ closeAllLists(e.target);
+ });
+}
\ No newline at end of file
diff --git a/app/static/js/controller.js b/app/static/js/controller.js
index b3b4f3f..6dbfc66 100644
--- a/app/static/js/controller.js
+++ b/app/static/js/controller.js
@@ -1,3 +1,21 @@
+const handleUserInput = searchBar => {
+ let xhrRequest = new XMLHttpRequest();
+ xhrRequest.open("POST", "/autocomplete");
+ xhrRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
+ xhrRequest.onload = function() {
+ if (xhrRequest.readyState === 4 && xhrRequest.status !== 200) {
+ alert("Error fetching autocomplete results");
+ return;
+ }
+
+ // Fill autocomplete with fetched results
+ let autocompleteResults = JSON.parse(xhrRequest.responseText);
+ autocomplete(searchBar, autocompleteResults["results"]);
+ };
+
+ xhrRequest.send('q=' + searchBar.value);
+};
+
const setupSearchLayout = () => {
// Setup search field
const searchBar = document.getElementById("search-bar");
@@ -11,6 +29,8 @@ const setupSearchLayout = () => {
if (event.keyCode === 13) {
event.preventDefault();
searchBtn.click();
+ } else {
+ handleUserInput(searchBar);
}
});
};
diff --git a/app/templates/index.html b/app/templates/index.html
index af1879f..aab6a19 100644
--- a/app/templates/index.html
+++ b/app/templates/index.html
@@ -17,9 +17,11 @@
+
+
Whoogle Search
@@ -28,7 +30,9 @@
diff --git a/test/test_autocomplete.py b/test/test_autocomplete.py
new file mode 100644
index 0000000..7dd4df5
--- /dev/null
+++ b/test/test_autocomplete.py
@@ -0,0 +1,20 @@
+from bs4 import BeautifulSoup
+from cryptography.fernet import Fernet
+from app.filter import Filter
+from datetime import datetime
+from dateutil.parser import *
+
+
+def test_autocomplete_get(client):
+ rv = client.get('/autocomplete?q=green+eggs+and')
+ assert rv._status_code == 200
+ assert len(rv.data) >= 1
+ assert b'green eggs and ham' in rv.data
+
+
+def test_autocomplete_post(client):
+ rv = client.post('/autocomplete', data=dict(q='the+cat+in+the'))
+ assert rv._status_code == 200
+ assert len(rv.data) >= 1
+ assert b'the cat in the hat' in rv.data
+