Merge remote-tracking branch 'origin/master' into heroku-app

README.md

@@ -25,7 +25,8 @@ Contents
 - No AMP links
 - No URL tracking tags (i.e. utm=%s)
 - No referrer header
-- POST request search queries (when possible)
+- Autocomplete/search suggestions
+- POST request search and suggestion queries (when possible)
 - View images at full res without site redirect (currently mobile only)
 - Dark mode
 - Randomly generated User Agent
@@ -107,7 +108,7 @@ Description=Whoogle
 Type=simple
 User=root
 WorkingDirectory=<whoogle_directory>
-ExecStart=<whoogle_directory>/venv/bin/python3 -um app --host 0.0.0.0 --port 5000 
+ExecStart=<whoogle_directory>/venv/bin/python3 -um app --host 0.0.0.0 --port 5000
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
@@ -133,7 +134,7 @@ sudo systemctl start whoogle
 Through Docker Hub:
 ```bash
 docker pull benbusby/whoogle-search
-docker run --publish 5000:5000 --detach --name whoogle-search whoogle-search:latest
+docker run --publish 5000:5000 --detach --name whoogle-search benbusby/whoogle-search:latest
 ```
 
 or with docker-compose:
@@ -185,7 +186,7 @@ To filter by a range of time, append ":past <time>" to the end of your search, w
 
 ## Extra Steps
 ### Set Whoogle as your primary search engine
-*Note: If you're using a reverse proxy to run Whoogle Search, make sure the "Root URL" config option on the home page is set to your URL before going through these steps.* 
+*Note: If you're using a reverse proxy to run Whoogle Search, make sure the "Root URL" config option on the home page is set to your URL before going through these steps.*
 
 Update browser settings:
   - Firefox (Desktop)

app/filter.py

@@ -90,10 +90,19 @@ class Filter:
         for script in soup('script'):
             script.decompose()
 
+        # Remove google's language/time config
+        st_card = soup.find('div', id='st-card')
+        if st_card:
+            st_card.decompose()
+
         footer = soup.find('div', id='sfooter')
-        if footer is not None:
+        if footer:
             footer.decompose()
 
+        header = soup.find('header')
+        if header:
+            header.decompose()
+
         return soup
 
     def remove_ads(self, soup):

app/models/config.py

@@ -1,4 +1,3 @@
-
 class Config:
     # Derived from here:
     # https://sites.google.com/site/tomihasa/google-language-codes#searchlanguage
@@ -27,14 +26,14 @@ class Config:
         {'name': 'Hindi', 'value': 'lang_hi'},
         {'name': 'Hungarian', 'value': 'lang_hu'},
         {'name': 'Icelandic', 'value': 'lang_is'},
-        {'name': 'Indonesian',  'value': 'lang_id'},
+        {'name': 'Indonesian', 'value': 'lang_id'},
-        {'name': 'Italian',  'value': 'lang_it'},
+        {'name': 'Italian', 'value': 'lang_it'},
-        {'name': 'Japanese',  'value': 'lang_ja'},
+        {'name': 'Japanese', 'value': 'lang_ja'},
-        {'name': 'Korean',  'value': 'lang_ko'},
+        {'name': 'Korean', 'value': 'lang_ko'},
-        {'name': 'Latvian',  'value': 'lang_lv'},
+        {'name': 'Latvian', 'value': 'lang_lv'},
-        {'name': 'Lithuanian',  'value': 'lang_lt'},
+        {'name': 'Lithuanian', 'value': 'lang_lt'},
-        {'name': 'Norwegian',  'value': 'lang_no'},
+        {'name': 'Norwegian', 'value': 'lang_no'},
-        {'name': 'Persian',  'value': 'lang_fa'},
+        {'name': 'Persian', 'value': 'lang_fa'},
         {'name': 'Polish', 'value': 'lang_pl'},
         {'name': 'Portuguese', 'value': 'lang_pt'},
         {'name': 'Romanian', 'value': 'lang_ro'},
@@ -51,9 +50,257 @@ class Config:
         {'name': 'Vietnamese', 'value': 'lang_vi'},
     ]
 
+    COUNTRIES = [
+        {'name': 'Default (use server location)', 'value': ''},
+        {'name': 'Afghanistan', 'value': 'countryAF'},
+        {'name': 'Albania', 'value': 'countryAL'},
+        {'name': 'Algeria', 'value': 'countryDZ'},
+        {'name': 'American Samoa', 'value': 'countryAS'},
+        {'name': 'Andorra', 'value': 'countryAD'},
+        {'name': 'Angola', 'value': 'countryAO'},
+        {'name': 'Anguilla', 'value': 'countryAI'},
+        {'name': 'Antarctica', 'value': 'countryAQ'},
+        {'name': 'Antigua and Barbuda', 'value': 'countryAG'},
+        {'name': 'Argentina', 'value': 'countryAR'},
+        {'name': 'Armenia', 'value': 'countryAM'},
+        {'name': 'Aruba', 'value': 'countryAW'},
+        {'name': 'Australia', 'value': 'countryAU'},
+        {'name': 'Austria', 'value': 'countryAT'},
+        {'name': 'Azerbaijan', 'value': 'countryAZ'},
+        {'name': 'Bahamas', 'value': 'countryBS'},
+        {'name': 'Bahrain', 'value': 'countryBH'},
+        {'name': 'Bangladesh', 'value': 'countryBD'},
+        {'name': 'Barbados', 'value': 'countryBB'},
+        {'name': 'Belarus', 'value': 'countryBY'},
+        {'name': 'Belgium', 'value': 'countryBE'},
+        {'name': 'Belize', 'value': 'countryBZ'},
+        {'name': 'Benin', 'value': 'countryBJ'},
+        {'name': 'Bermuda', 'value': 'countryBM'},
+        {'name': 'Bhutan', 'value': 'countryBT'},
+        {'name': 'Bolivia', 'value': 'countryBO'},
+        {'name': 'Bosnia and Herzegovina', 'value': 'countryBA'},
+        {'name': 'Botswana', 'value': 'countryBW'},
+        {'name': 'Bouvet Island', 'value': 'countryBV'},
+        {'name': 'Brazil', 'value': 'countryBR'},
+        {'name': 'British Indian Ocean Territory', 'value': 'countryIO'},
+        {'name': 'Brunei Darussalam', 'value': 'countryBN'},
+        {'name': 'Bulgaria', 'value': 'countryBG'},
+        {'name': 'Burkina Faso', 'value': 'countryBF'},
+        {'name': 'Burundi', 'value': 'countryBI'},
+        {'name': 'Cambodia', 'value': 'countryKH'},
+        {'name': 'Cameroon', 'value': 'countryCM'},
+        {'name': 'Canada', 'value': 'countryCA'},
+        {'name': 'Cape Verde', 'value': 'countryCV'},
+        {'name': 'Cayman Islands', 'value': 'countryKY'},
+        {'name': 'Central African Republic', 'value': 'countryCF'},
+        {'name': 'Chad', 'value': 'countryTD'},
+        {'name': 'Chile', 'value': 'countryCL'},
+        {'name': 'China', 'value': 'countryCN'},
+        {'name': 'Christmas Island', 'value': 'countryCX'},
+        {'name': 'Cocos (Keeling) Islands', 'value': 'countryCC'},
+        {'name': 'Colombia', 'value': 'countryCO'},
+        {'name': 'Comoros', 'value': 'countryKM'},
+        {'name': 'Congo', 'value': 'countryCG'},
+        {'name': 'Congo, Democratic Republic of the', 'value': 'countryCD'},
+        {'name': 'Cook Islands', 'value': 'countryCK'},
+        {'name': 'Costa Rica', 'value': 'countryCR'},
+        {'name': 'Cote D\'ivoire', 'value': 'countryCI'},
+        {'name': 'Croatia (Hrvatska)', 'value': 'countryHR'},
+        {'name': 'Cuba', 'value': 'countryCU'},
+        {'name': 'Cyprus', 'value': 'countryCY'},
+        {'name': 'Czech Republic', 'value': 'countryCZ'},
+        {'name': 'Denmark', 'value': 'countryDK'},
+        {'name': 'Djibouti', 'value': 'countryDJ'},
+        {'name': 'Dominica', 'value': 'countryDM'},
+        {'name': 'Dominican Republic', 'value': 'countryDO'},
+        {'name': 'East Timor', 'value': 'countryTP'},
+        {'name': 'Ecuador', 'value': 'countryEC'},
+        {'name': 'Egypt', 'value': 'countryEG'},
+        {'name': 'El Salvador', 'value': 'countrySV'},
+        {'name': 'Equatorial Guinea', 'value': 'countryGQ'},
+        {'name': 'Eritrea', 'value': 'countryER'},
+        {'name': 'Estonia', 'value': 'countryEE'},
+        {'name': 'Ethiopia', 'value': 'countryET'},
+        {'name': 'European Union', 'value': 'countryEU'},
+        {'name': 'Falkland Islands (Malvinas)', 'value': 'countryFK'},
+        {'name': 'Faroe Islands', 'value': 'countryFO'},
+        {'name': 'Fiji', 'value': 'countryFJ'},
+        {'name': 'Finland', 'value': 'countryFI'},
+        {'name': 'France', 'value': 'countryFR'},
+        {'name': 'France\, Metropolitan', 'value': 'countryFX'},
+        {'name': 'French Guiana', 'value': 'countryGF'},
+        {'name': 'French Polynesia', 'value': 'countryPF'},
+        {'name': 'French Southern Territories', 'value': 'countryTF'},
+        {'name': 'Gabon', 'value': 'countryGA'},
+        {'name': 'Gambia', 'value': 'countryGM'},
+        {'name': 'Georgia', 'value': 'countryGE'},
+        {'name': 'Germany', 'value': 'countryDE'},
+        {'name': 'Ghana', 'value': 'countryGH'},
+        {'name': 'Gibraltar', 'value': 'countryGI'},
+        {'name': 'Greece', 'value': 'countryGR'},
+        {'name': 'Greenland', 'value': 'countryGL'},
+        {'name': 'Grenada', 'value': 'countryGD'},
+        {'name': 'Guadeloupe', 'value': 'countryGP'},
+        {'name': 'Guam', 'value': 'countryGU'},
+        {'name': 'Guatemala', 'value': 'countryGT'},
+        {'name': 'Guinea', 'value': 'countryGN'},
+        {'name': 'Guinea-Bissau', 'value': 'countryGW'},
+        {'name': 'Guyana', 'value': 'countryGY'},
+        {'name': 'Haiti', 'value': 'countryHT'},
+        {'name': 'Heard Island and Mcdonald Islands', 'value': 'countryHM'},
+        {'name': 'Holy See (Vatican City State)', 'value': 'countryVA'},
+        {'name': 'Honduras', 'value': 'countryHN'},
+        {'name': 'Hong Kong', 'value': 'countryHK'},
+        {'name': 'Hungary', 'value': 'countryHU'},
+        {'name': 'Iceland', 'value': 'countryIS'},
+        {'name': 'India', 'value': 'countryIN'},
+        {'name': 'Indonesia', 'value': 'countryID'},
+        {'name': 'Iran, Islamic Republic of', 'value': 'countryIR'},
+        {'name': 'Iraq', 'value': 'countryIQ'},
+        {'name': 'Ireland', 'value': 'countryIE'},
+        {'name': 'Israel', 'value': 'countryIL'},
+        {'name': 'Italy', 'value': 'countryIT'},
+        {'name': 'Jamaica', 'value': 'countryJM'},
+        {'name': 'Japan', 'value': 'countryJP'},
+        {'name': 'Jordan', 'value': 'countryJO'},
+        {'name': 'Kazakhstan', 'value': 'countryKZ'},
+        {'name': 'Kenya', 'value': 'countryKE'},
+        {'name': 'Kiribati', 'value': 'countryKI'},
+        {'name': 'Korea, Democratic People\'s Republic of', 'value': 'countryKP'},
+        {'name': 'Korea, Republic of', 'value': 'countryKR'},
+        {'name': 'Kuwait', 'value': 'countryKW'},
+        {'name': 'Kyrgyzstan', 'value': 'countryKG'},
+        {'name': 'Lao People\'s Democratic Republic', 'value': 'countryLA'},
+        {'name': 'Latvia', 'value': 'countryLV'},
+        {'name': 'Lebanon', 'value': 'countryLB'},
+        {'name': 'Lesotho', 'value': 'countryLS'},
+        {'name': 'Liberia', 'value': 'countryLR'},
+        {'name': 'Libyan Arab Jamahiriya', 'value': 'countryLY'},
+        {'name': 'Liechtenstein', 'value': 'countryLI'},
+        {'name': 'Lithuania', 'value': 'countryLT'},
+        {'name': 'Luxembourg', 'value': 'countryLU'},
+        {'name': 'Macao', 'value': 'countryMO'},
+        {'name': 'Macedonia, the Former Yugosalv Republic of', 'value': 'countryMK'},
+        {'name': 'Madagascar', 'value': 'countryMG'},
+        {'name': 'Malawi', 'value': 'countryMW'},
+        {'name': 'Malaysia', 'value': 'countryMY'},
+        {'name': 'Maldives', 'value': 'countryMV'},
+        {'name': 'Mali', 'value': 'countryML'},
+        {'name': 'Malta', 'value': 'countryMT'},
+        {'name': 'Marshall Islands', 'value': 'countryMH'},
+        {'name': 'Martinique', 'value': 'countryMQ'},
+        {'name': 'Mauritania', 'value': 'countryMR'},
+        {'name': 'Mauritius', 'value': 'countryMU'},
+        {'name': 'Mayotte', 'value': 'countryYT'},
+        {'name': 'Mexico', 'value': 'countryMX'},
+        {'name': 'Micronesia, Federated States of', 'value': 'countryFM'},
+        {'name': 'Moldova, Republic of', 'value': 'countryMD'},
+        {'name': 'Monaco', 'value': 'countryMC'},
+        {'name': 'Mongolia', 'value': 'countryMN'},
+        {'name': 'Montserrat', 'value': 'countryMS'},
+        {'name': 'Morocco', 'value': 'countryMA'},
+        {'name': 'Mozambique', 'value': 'countryMZ'},
+        {'name': 'Myanmar', 'value': 'countryMM'},
+        {'name': 'Namibia', 'value': 'countryNA'},
+        {'name': 'Nauru', 'value': 'countryNR'},
+        {'name': 'Nepal', 'value': 'countryNP'},
+        {'name': 'Netherlands', 'value': 'countryNL'},
+        {'name': 'Netherlands Antilles', 'value': 'countryAN'},
+        {'name': 'New Caledonia', 'value': 'countryNC'},
+        {'name': 'New Zealand', 'value': 'countryNZ'},
+        {'name': 'Nicaragua', 'value': 'countryNI'},
+        {'name': 'Niger', 'value': 'countryNE'},
+        {'name': 'Nigeria', 'value': 'countryNG'},
+        {'name': 'Niue', 'value': 'countryNU'},
+        {'name': 'Norfolk Island', 'value': 'countryNF'},
+        {'name': 'Northern Mariana Islands', 'value': 'countryMP'},
+        {'name': 'Norway', 'value': 'countryNO'},
+        {'name': 'Oman', 'value': 'countryOM'},
+        {'name': 'Pakistan', 'value': 'countryPK'},
+        {'name': 'Palau', 'value': 'countryPW'},
+        {'name': 'Palestinian Territory', 'value': 'countryPS'},
+        {'name': 'Panama', 'value': 'countryPA'},
+        {'name': 'Papua New Guinea', 'value': 'countryPG'},
+        {'name': 'Paraguay', 'value': 'countryPY'},
+        {'name': 'Peru', 'value': 'countryPE'},
+        {'name': 'Philippines', 'value': 'countryPH'},
+        {'name': 'Pitcairn', 'value': 'countryPN'},
+        {'name': 'Poland', 'value': 'countryPL'},
+        {'name': 'Portugal', 'value': 'countryPT'},
+        {'name': 'Puerto Rico', 'value': 'countryPR'},
+        {'name': 'Qatar', 'value': 'countryQA'},
+        {'name': 'Reunion', 'value': 'countryRE'},
+        {'name': 'Romania', 'value': 'countryRO'},
+        {'name': 'Russian Federation', 'value': 'countryRU'},
+        {'name': 'Rwanda', 'value': 'countryRW'},
+        {'name': 'Saint Helena', 'value': 'countrySH'},
+        {'name': 'Saint Kitts and Nevis', 'value': 'countryKN'},
+        {'name': 'Saint Lucia', 'value': 'countryLC'},
+        {'name': 'Saint Pierre and Miquelon', 'value': 'countryPM'},
+        {'name': 'Saint Vincent and the Grenadines', 'value': 'countryVC'},
+        {'name': 'Samoa', 'value': 'countryWS'},
+        {'name': 'San Marino', 'value': 'countrySM'},
+        {'name': 'Sao Tome and Principe', 'value': 'countryST'},
+        {'name': 'Saudi Arabia', 'value': 'countrySA'},
+        {'name': 'Senegal', 'value': 'countrySN'},
+        {'name': 'Serbia and Montenegro', 'value': 'countryCS'},
+        {'name': 'Seychelles', 'value': 'countrySC'},
+        {'name': 'Sierra Leone', 'value': 'countrySL'},
+        {'name': 'Singapore', 'value': 'countrySG'},
+        {'name': 'Slovakia', 'value': 'countrySK'},
+        {'name': 'Slovenia', 'value': 'countrySI'},
+        {'name': 'Solomon Islands', 'value': 'countrySB'},
+        {'name': 'Somalia', 'value': 'countrySO'},
+        {'name': 'South Africa', 'value': 'countryZA'},
+        {'name': 'South Georgia and the South Sandwich Islands', 'value': 'countryGS'},
+        {'name': 'Spain', 'value': 'countryES'},
+        {'name': 'Sri Lanka', 'value': 'countryLK'},
+        {'name': 'Sudan', 'value': 'countrySD'},
+        {'name': 'Suriname', 'value': 'countrySR'},
+        {'name': 'Svalbard and Jan Mayen', 'value': 'countrySJ'},
+        {'name': 'Swaziland', 'value': 'countrySZ'},
+        {'name': 'Sweden', 'value': 'countrySE'},
+        {'name': 'Switzerland', 'value': 'countryCH'},
+        {'name': 'Syrian Arab Republic', 'value': 'countrySY'},
+        {'name': 'Taiwan, Province of China', 'value': 'countryTW'},
+        {'name': 'Tajikistan', 'value': 'countryTJ'},
+        {'name': 'Tanzania, United Republic of', 'value': 'countryTZ'},
+        {'name': 'Thailand', 'value': 'countryTH'},
+        {'name': 'Togo', 'value': 'countryTG'},
+        {'name': 'Tokelau', 'value': 'countryTK'},
+        {'name': 'Tonga', 'value': 'countryTO'},
+        {'name': 'Trinidad and Tobago', 'value': 'countryTT'},
+        {'name': 'Tunisia', 'value': 'countryTN'},
+        {'name': 'Turkey', 'value': 'countryTR'},
+        {'name': 'Turkmenistan', 'value': 'countryTM'},
+        {'name': 'Turks and Caicos Islands', 'value': 'countryTC'},
+        {'name': 'Tuvalu', 'value': 'countryTV'},
+        {'name': 'Uganda', 'value': 'countryUG'},
+        {'name': 'Ukraine', 'value': 'countryUA'},
+        {'name': 'United Arab Emirates', 'value': 'countryAE'},
+        {'name': 'United Kingdom', 'value': 'countryUK'},
+        {'name': 'United States', 'value': 'countryUS'},
+        {'name': 'United States Minor Outlying Islands', 'value': 'countryUM'},
+        {'name': 'Uruguay', 'value': 'countryUY'},
+        {'name': 'Uzbekistan', 'value': 'countryUZ'},
+        {'name': 'Vanuatu', 'value': 'countryVU'},
+        {'name': 'Venezuela', 'value': 'countryVE'},
+        {'name': 'Vietnam', 'value': 'countryVN'},
+        {'name': 'Virgin Islands, British', 'value': 'countryVG'},
+        {'name': 'Virgin Islands, U.S.', 'value': 'countryVI'},
+        {'name': 'Wallis and Futuna', 'value': 'countryWF'},
+        {'name': 'Western Sahara', 'value': 'countryEH'},
+        {'name': 'Yemen', 'value': 'countryYE'},
+        {'name': 'Yugoslavia', 'value': 'countryYU'},
+        {'name': 'Zambia', 'value': 'countryZM'},
+        {'name': 'Zimbabwe', 'value': 'countryZW'}
+    ]
+
     def __init__(self, **kwargs):
         self.url = ''
         self.lang = 'lang_en'
+        self.ctry = ''
+        self.safe = True
         self.dark = False
         self.nojs = False
         self.near = ''
@@ -73,4 +320,4 @@ class Config:
         return delattr(self, name)
 
     def __contains__(self, name):
-        return hasattr(self, name)
+        return hasattr(self, name)

app/request.py

@@ -1,10 +1,12 @@
 from io import BytesIO
+from lxml import etree
 import pycurl
 import random
 import urllib.parse as urlparse
 
-# Base search url
+# Core Google search URLs
 SEARCH_URL = 'https://www.google.com/search?gbv=1&q='
+AUTOCOMPLETE_URL = 'https://suggestqueries.google.com/complete/search?client=toolbar&'
 
 MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
 DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
@@ -13,9 +15,7 @@ DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
 VALID_PARAMS = ['tbs', 'tbm', 'start', 'near']
 
 
-def gen_user_agent(normal_ua):
+def gen_user_agent(normal_ua, is_mobile):
-    is_mobile = 'Android' in normal_ua or 'iPhone' in normal_ua
-
     mozilla = random.choice(['Moo', 'Woah', 'Bro', 'Slow']) + 'zilla'
     firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox'
     linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux'
@@ -26,7 +26,7 @@ def gen_user_agent(normal_ua):
         return DESKTOP_UA.format(mozilla, linux, firefox)
 
 
-def gen_query(query, args, near_city=None, language='lang_en'):
+def gen_query(query, args, config, near_city=None):
     param_dict = {key: '' for key in VALID_PARAMS}
     # Use :past(hour/day/week/month/year) if available
     # example search "new restaurants :past month"
@@ -46,11 +46,13 @@ def gen_query(query, args, near_city=None, language='lang_en'):
         param_dict['start'] = '&start=' + args.get('start')
 
     # Search for results near a particular city, if available
-    if near_city is not None:
+    if near_city:
         param_dict['near'] = '&near=' + urlparse.quote(near_city)
 
     # Set language for results (lr) and interface (hl)
-    param_dict['lr'] = '&lr=' + language + '&hl=' + language.replace('lang_', '')
+    param_dict['lr'] = '&lr=' + config.lang + '&hl=' + config.lang.replace('lang_', '')
+    param_dict['cr'] = ('&cr=' + config.ctry) if config.ctry else ''
+    param_dict['safe'] = '&safe=' + ('active' if config.safe else 'off')
 
     for val in param_dict.values():
         if not val or val is None:
@@ -62,8 +64,9 @@ def gen_query(query, args, near_city=None, language='lang_en'):
 
 class Request:
     def __init__(self, normal_ua, language='lang_en'):
-        self.modified_user_agent = gen_user_agent(normal_ua)
         self.language = language
+        self.mobile = 'Android' in normal_ua or 'iPhone' in normal_ua
+        self.modified_user_agent = gen_user_agent(normal_ua, self.mobile)
 
     def __getitem__(self, name):
         return getattr(self, name)
@@ -74,6 +77,16 @@ class Request:
         else:
             return 'unicode-escape'
 
+    def autocomplete(self, query):
+        ac_query = dict(hl=self.language, q=query)
+        response = self.send(base_url=AUTOCOMPLETE_URL, query=urlparse.urlencode(ac_query))
+
+        if response:
+            dom = etree.fromstring(response)
+            return dom.xpath('//suggestion/@data')
+
+        return []
+
     def send(self, base_url=SEARCH_URL, query='', return_bytes=False):
         response_header = []
 

app/routes.py

@@ -3,13 +3,15 @@ from app.filter import Filter, get_first_link
 from app.models.config import Config
 from app.request import Request, gen_query
 import argparse
+import base64
 from bs4 import BeautifulSoup
 from cryptography.fernet import Fernet, InvalidToken
-from flask import g, make_response, request, redirect, render_template, send_file
+from flask import g, jsonify, make_response, request, redirect, render_template, send_file
 from functools import wraps
 import io
 import json
 import os
+from pycurl import error as pycurl_error
 import urllib.parse as urlparse
 import waitress
 
@@ -64,7 +66,9 @@ def index():
                            bg=bg,
                            ua=g.user_request.modified_user_agent,
                            languages=Config.LANGUAGES,
+                           countries=Config.COUNTRIES,
                            current_lang=g.user_config.lang,
+                           current_ctry=g.user_config.ctry,
                            version_number=app.config['VERSION_NUMBER'],
                            request_type='get' if g.user_config.get_only else 'post')
 
@@ -87,6 +91,19 @@ def opensearch():
     return response
 
 
+@app.route('/autocomplete', methods=['GET', 'POST'])
+def autocomplete():
+    request_params = request.args if request.method == 'GET' else request.form
+    q = request_params.get('q')
+
+    if not q and not request.data:
+        return jsonify({'?': []})
+    elif request.data:
+        q = urlparse.unquote_plus(request.data.decode('utf-8').replace('q=', ''))
+
+    return jsonify([q, g.user_request.autocomplete(q)])
+
+
 @app.route('/search', methods=['GET', 'POST'])
 @auth_required
 def search():
@@ -111,7 +128,7 @@ def search():
     mobile = 'Android' in user_agent or 'iPhone' in user_agent
 
     content_filter = Filter(mobile, g.user_config, secret_key=app.secret_key)
-    full_query = gen_query(q, request_params, content_filter.near, language=g.user_config.lang)
+    full_query = gen_query(q, request_params, g.user_config, content_filter.near)
     get_body = g.user_request.send(query=full_query)
     dirty_soup = BeautifulSoup(content_filter.reskin(get_body), 'html.parser')
 
@@ -120,7 +137,14 @@ def search():
     else:
         formatted_results = content_filter.clean(dirty_soup)
 
-    return render_template('display.html', query=urlparse.unquote(q), response=formatted_results)
+    return render_template(
+        'display.html',
+        query=urlparse.unquote(q),
+        response=formatted_results,
+        search_header=render_template(
+            'header.html',
+            q=urlparse.unquote(q),
+            mobile=g.user_request.mobile))
 
 
 @app.route('/config', methods=['GET', 'POST'])
@@ -164,17 +188,24 @@ def imgres():
 def tmp():
     cipher_suite = Fernet(app.secret_key)
     img_url = cipher_suite.decrypt(request.args.get('image_url').encode()).decode()
-    file_data = g.user_request.send(base_url=img_url, return_bytes=True)
-    tmp_mem = io.BytesIO()
-    tmp_mem.write(file_data)
-    tmp_mem.seek(0)
 
-    return send_file(
+    try:
-        tmp_mem,
+        file_data = g.user_request.send(base_url=img_url, return_bytes=True)
-        as_attachment=True,
+        tmp_mem = io.BytesIO()
-        attachment_filename='tmp.png',
+        tmp_mem.write(file_data)
-        mimetype='image/png'
+        tmp_mem.seek(0)
-    )
+
+        return send_file(
+            tmp_mem,
+            as_attachment=True,
+            attachment_filename='tmp.png',
+            mimetype='image/png'
+        )
+    except pycurl_error:
+        pass
+
+    empty_gif = base64.b64decode('R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==')
+    return send_file(io.BytesIO(empty_gif), mimetype='image/gif')
 
 
 @app.route('/window')

app/static/css/header.css

@@ -0,0 +1,55 @@
+header {
+    font-family: Roboto,HelveticaNeue,Arial,sans-serif;
+    font-size: 14px;
+    line-height: 20px;
+    color: #3C4043;
+    word-wrap: break-word;
+}
+
+.logo-link, .logo-letter {
+    text-decoration: none !important;
+    letter-spacing: -1px;
+    text-align: center;
+    border-radius: 2px 0 0 0;
+}
+
+.mobile-logo {
+    font: 22px/36px Futura, Arial, sans-serif;
+    padding-left: 5px;
+}
+
+.logo-div {
+    letter-spacing: -1px;
+    text-align: center;
+    font: 22pt Futura, Arial, sans-serif;
+    padding: 10px 0 5px 0;
+    height: 37px;
+    font-smoothing: antialiased;
+}
+
+.search-div {
+    border-radius: 8px 8px 0 0;
+    box-shadow: 0 1px 6px rgba(32, 33, 36, 0.18);
+    margin-top: 10px;
+}
+
+.search-form {
+    height: 39px;
+    display: flex;
+    width: 100%;
+}
+
+.search-input {
+    background: none;
+    margin: 2px 4px 2px 8px;
+    display: block;
+    font-size: 16px;
+    padding: 0 0 0 8px;
+    flex: 1;
+    height: 35px;
+    outline: none;
+    border: none;
+    width: 100%;
+    -webkit-tap-highlight-color: rgba(0,0,0,0);
+    overflow: hidden;
+}

app/static/css/search.css

@@ -0,0 +1,34 @@
+.autocomplete {
+    position: relative;
+    display: inline-block;
+    width: 100%;
+}
+
+.autocomplete-items {
+    position: absolute;
+    border: 1px solid #d4d4d4;
+    border-bottom: none;
+    border-top: none;
+    z-index: 99;
+
+    /*position the autocomplete items to be the same width as the container:*/
+    top: 100%;
+    left: 0;
+    right: 0;
+}
+
+.autocomplete-items div {
+    padding: 10px;
+    cursor: pointer;
+    background-color: #fff;
+    border-bottom: 1px solid #d4d4d4;
+}
+
+.autocomplete-items div:hover {
+    background-color: #e9e9e9;
+}
+
+.autocomplete-active {
+    background-color: #685e79 !important;
+    color: #ffffff;
+}

app/static/js/autocomplete.js

@@ -0,0 +1,98 @@
+const handleUserInput = searchBar => {
+    let xhrRequest = new XMLHttpRequest();
+    xhrRequest.open("POST", "/autocomplete");
+    xhrRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
+    xhrRequest.onload = function() {
+        if (xhrRequest.readyState === 4 && xhrRequest.status !== 200) {
+            alert("Error fetching autocomplete results");
+            return;
+        }
+
+        // Fill autocomplete with fetched results
+        let autocompleteResults = JSON.parse(xhrRequest.responseText);
+        autocomplete(searchBar, autocompleteResults[1]);
+    };
+
+    xhrRequest.send('q=' + searchBar.value);
+};
+
+const autocomplete = (searchInput, autocompleteResults) => {
+    let currentFocus;
+
+    searchInput.addEventListener("input", function () {
+        let autocompleteList, autocompleteItem, i, val = this.value;
+        closeAllLists();
+
+        if (!val || !autocompleteResults) {
+            return false;
+        }
+
+        currentFocus = -1;
+        autocompleteList = document.createElement("div");
+        autocompleteList.setAttribute("id", this.id + "-autocomplete-list");
+        autocompleteList.setAttribute("class", "autocomplete-items");
+        this.parentNode.appendChild(autocompleteList);
+
+        for (i = 0; i < autocompleteResults.length; i++) {
+            if (autocompleteResults[i].substr(0, val.length).toUpperCase() === val.toUpperCase()) {
+                autocompleteItem = document.createElement("div");
+                autocompleteItem.innerHTML = "<strong>" + autocompleteResults[i].substr(0, val.length) + "</strong>";
+                autocompleteItem.innerHTML += autocompleteResults[i].substr(val.length);
+                autocompleteItem.innerHTML += "<input type=\"hidden\" value=\"" + autocompleteResults[i] + "\">";
+                autocompleteItem.addEventListener("click", function () {
+                    searchInput.value = this.getElementsByTagName("input")[0].value;
+                    closeAllLists();
+                    document.getElementById("search-form").submit();
+                });
+                autocompleteList.appendChild(autocompleteItem);
+            }
+        }
+    });
+
+    searchInput.addEventListener("keydown", function (e) {
+        let suggestion = document.getElementById(this.id + "-autocomplete-list");
+        if (suggestion) suggestion = suggestion.getElementsByTagName("div");
+        if (e.keyCode === 40) { // down
+            currentFocus++;
+            addActive(suggestion);
+        } else if (e.keyCode === 38) { //up
+            currentFocus--;
+            addActive(suggestion);
+        } else if (e.keyCode === 13) { // enter
+            e.preventDefault();
+            if (currentFocus > -1) {
+                if (suggestion) suggestion[currentFocus].click();
+            }
+        }
+    });
+
+    const addActive = suggestion => {
+        if (!suggestion || !suggestion[currentFocus]) return false;
+        removeActive(suggestion);
+
+        if (currentFocus >= suggestion.length) currentFocus = 0;
+        if (currentFocus < 0) currentFocus = (suggestion.length - 1);
+
+        suggestion[currentFocus].classList.add("autocomplete-active");
+    };
+
+    const removeActive = suggestion => {
+        for (let i = 0; i < suggestion.length; i++) {
+            suggestion[i].classList.remove("autocomplete-active");
+        }
+    };
+
+    const closeAllLists = el => {
+        let suggestions = document.getElementsByClassName("autocomplete-items");
+        for (let i = 0; i < suggestions.length; i++) {
+            if (el !== suggestions[i] && el !== searchInput) {
+                suggestions[i].parentNode.removeChild(suggestions[i]);
+            }
+        }
+    };
+
+    // Close lists and search when user selects a suggestion
+    document.addEventListener("click", function (e) {
+        closeAllLists(e.target);
+    });
+};

app/static/js/controller.js

@@ -11,6 +11,8 @@ const setupSearchLayout = () => {
         if (event.keyCode === 13) {
             event.preventDefault();
             searchBtn.click();
+        } else {
+            handleUserInput(searchBar);
         }
     });
 };
@@ -20,6 +22,7 @@ const fillConfigValues = () => {
     const near = document.getElementById("config-near");
     const noJS = document.getElementById("config-nojs");
     const dark = document.getElementById("config-dark");
+    const safe = document.getElementById("config-safe");
     const url  = document.getElementById("config-url");
     const newTab  = document.getElementById("config-new-tab");
     const getOnly = document.getElementById("config-get-only");
@@ -39,6 +42,7 @@ const fillConfigValues = () => {
         near.value = configSettings["near"] ? configSettings["near"] : "";
         noJS.checked = !!configSettings["nojs"];
         dark.checked = !!configSettings["dark"];
+        safe.checked = !!configSettings["safe"];
         getOnly.checked = !!configSettings["get_only"];
         newTab.checked = !!configSettings["new_tab"];
 

app/templates/display.html

@@ -5,9 +5,13 @@
         <link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Whoogle Search">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
         <meta name="referrer" content="no-referrer">
+        <script type="text/javascript" src="/static/js/autocomplete.js"></script>
+        <link rel="stylesheet" href="/static/css/search.css">
+        <link rel="stylesheet" href="/static/css/header.css">
         <title>{{ query }} - Whoogle Search</title>
     </head>
     <body>
+        {{ search_header|safe }}
         {{ response|safe }}
     </body>
 </html>

app/templates/header.html

@@ -0,0 +1,53 @@
+{% if mobile %}
+    <header>
+        <div class="bz1lBb">
+            <form class="Pg70bf" id="search-form" method="POST">
+                <a class="logo-link mobile-logo"
+                   href="/"
+                   style="display:flex; justify-content:center; align-items:center; color:#685e79; font-size:18px; ">
+                    <span class="V6gwVd">Wh</span><span class="iWkuvd">o</span><span class="cDrQ7">o</span><span
+                        class="V6gwVd">g</span><span class="ntlR9">l</span><span
+                        class="iWkuvd tJ3Myc">e</span>
+                </a>
+                <div class="H0PQec" style="width: 100%;">
+                    <div class="sbc esbc autocomplete">
+                        <input id="search-bar" autocapitalize="none" autocomplete="off" class="noHIxc" name="q"
+                               spellcheck="false" type="text" value="{{ q }}">
+                        <div class="sc"></div>
+                    </div>
+                </div>
+            </form>
+        </div>
+    </header>
+{% else %}
+    <header>
+        <div class="logo-div">
+            <a class="logo-link" href="/">
+                <span class="V6gwVd logo-letter">Wh</span><span class="iWkuvd logo-letter">o</span><span
+                    class="cDrQ7 logo-letter">o</span><span class="V6gwVd logo-letter">g</span><span
+                    class="ntlR9 logo-letter">l</span><span class="iWkuvd tJ3Myc logo-letter">e</span>
+            </a>
+        </div>
+        <div class="search-div">
+            <form id="search-form" class="search-form" id="sf" method="POST">
+                <div class="autocomplete" style="width: 100%; flex: 1">
+                    <div style="width: 100%; display: flex">
+                        <input id="search-bar" autocapitalize="none" autocomplete="off" class="noHIxc" name="q"
+                               spellcheck="false" type="text" value="{{ q }}">
+                        <div class="sc"></div>
+                    </div>
+                </div>
+            </form>
+        </div>
+    </header>
+{% endif %}
+
+<script>
+    const searchBar = document.getElementById("search-bar");
+
+    searchBar.addEventListener("keyup", function (event) {
+        if (event.keyCode !== 13) {
+            handleUserInput(searchBar);
+        }
+    });
+</script>

app/templates/index.html

@@ -17,18 +17,22 @@
         <meta name="referrer" content="no-referrer">
         <meta name="msapplication-TileColor" content="#ffffff">
         <meta name="msapplication-TileImage" content="/static/img/favicon/ms-icon-144x144.png">
+        <script type="text/javascript" src="/static/js/autocomplete.js"></script>
         <script type="text/javascript" src="/static/js/controller.js"></script>
         <link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Whoogle Search">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <link rel="stylesheet" href="/static/css/search.css">
         <link rel="stylesheet" href="/static/css/main.css">
 	<title>Whoogle Search</title>
     </head>
     <body id="main" style="display: none; background-color: {{ bg }}">
         <div class="search-container">
             <img class="logo" src="/static/img/logo.png">
-            <form action="/search" method="{{ request_type }}">
+            <form id="search-form" action="/search" method="{{ request_type }}">
                 <div class="search-fields">
-                    <input type="text" name="q" id="search-bar" autofocus="autofocus">
+                    <div class="autocomplete">
+                        <input type="text" name="q" id="search-bar" autofocus="autofocus">
+                    </div>
                     <input type="submit" id="search-submit" value="Search">
                 </div>
             </form>
@@ -41,6 +45,19 @@
                             <!-- TODO: Add option to regenerate user agent? -->
                             <span class="ua-span">User Agent: {{ ua }}</span>
                         </div>
+                        <div class="config-div">
+                            <label for="config-ctry">Country: </label>
+                            <select name="ctry" id="config-ctry">
+                                {% for ctry in countries %}
+                                    <option value="{{ ctry.value }}"
+                                            {% if ctry.value in current_ctry %}
+                                            selected
+                                            {% endif %}>
+                                        {{ ctry.name }}
+                                    </option>
+                                {% endfor %}
+                            </select>
+                        </div>
                         <div class="config-div">
                             <label for="config-lang">Language: </label>
                             <select name="lang" id="config-lang">
@@ -66,6 +83,10 @@
                             <label for="config-dark">Dark Mode: </label>
                             <input type="checkbox" name="dark" id="config-dark">
                         </div>
+                        <div class="config-div">
+                            <label for="config-safe">Safe Search: </label>
+                            <input type="checkbox" name="safe" id="config-safe">
+                        </div>
                         <div class="config-div">
                             <label for="config-new-tab">Open Links in New Tab: </label>
                             <input type="checkbox" name="new_tab" id="config-new-tab">

app/templates/opensearch.xml

@@ -7,7 +7,9 @@
   <Url type="text/html" method="{{ request_type }}" template="{{ main_url }}/search">
     <Param name="q" value="{searchTerms}"/>
   </Url>
-  <Url type="application/x-suggestions+json" template="{{ main_url }}/search"/>
+  <Url type="application/x-suggestions+json" method="{{ request_type }}" template="{{ main_url }}/autocomplete">
+    <Param name="q" value="{searchTerms}"/>
+  </Url>
   <moz:SearchForm>{{ main_url }}/search</moz:SearchForm>
 </OpenSearchDescription>
 

requirements.txt

@@ -6,6 +6,7 @@ cryptography==2.8
 Flask==1.1.1
 itsdangerous==1.1.0
 Jinja2==2.10.3
+lxml==4.5.1
 MarkupSafe==1.1.1
 pycparser==2.19
 pycurl==7.43.0.4

test/test_autocomplete.py

@@ -0,0 +1,12 @@
+def test_autocomplete_get(client):
+    rv = client.get('/autocomplete?q=green+eggs+and')
+    assert rv._status_code == 200
+    assert len(rv.data) >= 1
+    assert b'green eggs and ham' in rv.data
+
+
+def test_autocomplete_post(client):
+    rv = client.post('/autocomplete', data=dict(q='the+cat+in+the'))
+    assert rv._status_code == 200
+    assert len(rv.data) >= 1
+    assert b'the cat in the hat' in rv.data

test/test_results.py

@@ -64,4 +64,4 @@ def test_recent_results(client):
                 date = parse(date_span)
                 assert (current_date - date).days <= (num_days + 5)  # Date can have a little bit of wiggle room
             except ParserError:
-                assert ' ago' in date_span
+                pass