Major refactor of requests and session management

- Switches from pycurl to requests library
  - Allows for less janky decoding, especially with non-latin character
  sets
- Adds session level management of user configs
  - Allows for each session to set its own config (people are probably
  going to complain about this, though not sure if it'll be the same
  number of people who are upset that their friends/family have to share
  their config)
- Updates key gen/regen to more aggressively swap out keys after each
request

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 *.pem
 config.json
 test/static
+flask_session/
 
 # pip stuff
 build/

app/__init__.py

@@ -1,12 +1,21 @@
+from app.utils.misc import generate_user_keys
 from cryptography.fernet import Fernet
 from flask import Flask
+from flask_session import Session
 import os
 
 app = Flask(__name__, static_folder=os.path.dirname(os.path.abspath(__file__)) + '/static')
-app.secret_key = Fernet.generate_key()
+app.user_elements = {}
+app.config['SECRET_KEY'] = os.urandom(128)
+app.config['SESSION_TYPE'] = 'filesystem'
 app.config['VERSION_NUMBER'] = '0.1.4'
 app.config['APP_ROOT'] = os.getenv('APP_ROOT', os.path.dirname(os.path.abspath(__file__)))
 app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config['APP_ROOT'], 'static'))
-app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', app.config['STATIC_FOLDER']) + '/config.json'
+app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', app.config['STATIC_FOLDER'] + '/config')
+app.config['SESSION_FILE_DIR'] = app.config['CONFIG_PATH']
+app.config['SESSION_COOKIE_SECURE'] = True
+
+sess = Session()
+sess.init_app(app)
 
 from app import routes

app/filter.py

@@ -17,14 +17,9 @@ data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAD0lEQVR42m
 def get_first_link(soup):
     # Replace hrefs with only the intended destination (no "utm" type tags)
     for a in soup.find_all('a', href=True):
-        href = a['href'].replace('https://www.google.com', '')
-        
-        result_link = urlparse.urlparse(href)
-        query_link = parse_qs(result_link.query)['q'][0] if '?q=' in href else ''
-
         # Return the first search result URL
-        if 'url?q=' in href:
+        if 'url?q=' in a['href']:
-            return filter_link_args(href)
+            return filter_link_args(a['href'])
 
 
 def filter_link_args(query_link):
@@ -52,7 +47,7 @@ def filter_link_args(query_link):
 
 
 class Filter:
-    def __init__(self, mobile=False, config=None, secret_key=''):
+    def __init__(self, user_keys: dict, mobile=False, config=None):
         if config is None:
             config = {}
 
@@ -61,11 +56,16 @@ class Filter:
         self.nojs = config['nojs'] if 'nojs' in config else False
         self.new_tab = config['new_tab'] if 'new_tab' in config else False
         self.mobile = mobile
-        self.secret_key = secret_key
+        self.user_keys = user_keys
+        self._elements = 0
 
     def __getitem__(self, name):
         return getattr(self, name)
 
+    @property
+    def elements(self):
+        return self._elements
+
     def reskin(self, page):
         # Aesthetic only re-skinning
         page = page.replace('>G<', '>Wh<')
@@ -76,11 +76,29 @@ class Filter:
 
         return page
 
+    def encrypt_path(self, msg, is_element=False):
+        # Encrypts path to avoid plaintext results in logs
+        if is_element:
+            # Element paths are tracked differently in order for the element key to be regenerated
+            # once all elements have been loaded
+            enc_path = Fernet(self.user_keys['element_key']).encrypt(msg.encode()).decode()
+            self._elements += 1
+            return enc_path
+
+        return Fernet(self.user_keys['text_key']).encrypt(msg.encode()).decode()
+
     def clean(self, soup):
         self.remove_ads(soup)
-        self.update_image_paths(soup)
         self.update_styling(soup)
-        self.update_links(soup)
+
+        for img in [_ for _ in soup.find_all('img') if 'src' in _.attrs]:
+            self.update_element_src(img, 'image/png')
+
+        for audio in [_ for _ in soup.find_all('audio') if 'src' in _.attrs]:
+            self.update_element_src(audio, 'audio/mpeg')
+
+        for link in soup.find_all('a', href=True):
+            self.update_link(link)
 
         input_form = soup.find('form')
         if input_form is not None:
@@ -116,25 +134,24 @@ class Filter:
         for div in ad_divs:
             div.decompose()
 
-    def update_image_paths(self, soup):
+    def update_element_src(self, element, mimetype):
-        for img in [_ for _ in soup.find_all('img') if 'src' in _.attrs]:
+        element_src = element['src']
-            img_src = img['src']
+        if element_src.startswith('//'):
-            if img_src.startswith('//'):
+            element_src = 'https:' + element_src
-                img_src = 'https:' + img_src
+        elif element_src.startswith(LOGO_URL):
-            elif img_src.startswith(LOGO_URL):
+            # Re-brand with Whoogle logo
-                # Re-brand with Whoogle logo
+            element['src'] = '/static/img/logo.png'
-                img['src'] = '/static/img/logo.png'
+            element['style'] = 'height:40px;width:162px'
-                img['style'] = 'height:40px;width:162px'
+            return
-                continue
+        elif element_src.startswith(GOOG_IMG):
-            elif img_src.startswith(GOOG_IMG):
+            element['src'] = BLANK_B64
-                img['src'] = BLANK_B64
+            return
-                continue
 
-            enc_src = Fernet(self.secret_key).encrypt(img_src.encode())
+        element['src'] = '/element?url=' + self.encrypt_path(element_src, is_element=True) + \
-            img['src'] = '/tmp?image_url=' + enc_src.decode()
+                         '&type=' + urlparse.quote(mimetype)
-            # TODO: Non-mobile image results link to website instead of image
+        # TODO: Non-mobile image results link to website instead of image
-            # if not self.mobile:
+        # if not self.mobile:
-            # img.append(BeautifulSoup(FULL_RES_IMG.format(img_src), 'html.parser'))
+        # img.append(BeautifulSoup(FULL_RES_IMG.format(element_src), 'html.parser'))
 
     def update_styling(self, soup):
         # Remove unnecessary button(s)
@@ -170,45 +187,43 @@ class Filter:
             for href_element in soup.findAll('a'):
                 href_element['style'] = 'color: white' if href_element['href'].startswith('/search') else ''
 
-    def update_links(self, soup):
+    def update_link(self, link):
-        # Replace hrefs with only the intended destination (no "utm" type tags)
+        # Replace href with only the intended destination (no "utm" type tags)
-        for a in soup.find_all('a', href=True):
+        href = link['href'].replace('https://www.google.com', '')
-            href = a['href'].replace('https://www.google.com', '')
+        if '/advanced_search' in href:
-            if '/advanced_search' in href:
+            link.decompose()
-                a.decompose()
+            return
-                continue
+        elif self.new_tab:
-            elif self.new_tab:
+            link['target'] = '_blank'
-                a['target'] = '_blank'
 
-            result_link = urlparse.urlparse(href)
+        result_link = urlparse.urlparse(href)
-            query_link = parse_qs(result_link.query)['q'][0] if '?q=' in href else ''
+        query_link = parse_qs(result_link.query)['q'][0] if '?q=' in href else ''
 
-            if query_link.startswith('/'):
+        if query_link.startswith('/'):
-                a['href'] = 'https://google.com' + query_link
+            link['href'] = 'https://google.com' + query_link
-            elif '/search?q=' in href:
+        elif '/search?q=' in href:
-                enc_result = Fernet(self.secret_key).encrypt(query_link.encode())
+            new_search = '/search?q=' + self.encrypt_path(query_link)
-                new_search = '/search?q=' + enc_result.decode()
 
-                query_params = parse_qs(urlparse.urlparse(href).query)
+            query_params = parse_qs(urlparse.urlparse(href).query)
-                for param in VALID_PARAMS:
+            for param in VALID_PARAMS:
-                    param_val = query_params[param][0] if param in query_params else ''
+                param_val = query_params[param][0] if param in query_params else ''
-                    new_search += '&' + param + '=' + param_val
+                new_search += '&' + param + '=' + param_val
-                a['href'] = new_search
+            link['href'] = new_search
-            elif 'url?q=' in href:
+        elif 'url?q=' in href:
-                # Strip unneeded arguments
+            # Strip unneeded arguments
-                a['href'] = filter_link_args(query_link)
+            link['href'] = filter_link_args(query_link)
 
-                # Add no-js option
+            # Add no-js option
-                if self.nojs:
+            if self.nojs:
-                    gen_nojs(soup, a['href'], a)
+                gen_nojs(link)
-            else:
+        else:
-                a['href'] = href
+            link['href'] = href
 
 
-def gen_nojs(soup, link, sibling):
+def gen_nojs(sibling):
-    nojs_link = soup.new_tag('a')
+    nojs_link = BeautifulSoup().new_tag('a')
-    nojs_link['href'] = '/window?location=' + link
+    nojs_link['href'] = '/window?location=' + sibling['href']
     nojs_link['style'] = 'display:block;width:100%;'
     nojs_link.string = 'NoJS Link: ' + nojs_link['href']
     sibling.append(BeautifulSoup('<br><hr><br>', 'html.parser'))
-    sibling.append(nojs_link)
+    sibling.append(nojs_link)

app/request.py

@@ -1,7 +1,7 @@
 from io import BytesIO
 from lxml import etree
-import pycurl
 import random
+import requests
 import urllib.parse as urlparse
 
 # Core Google search URLs
@@ -15,7 +15,7 @@ DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
 VALID_PARAMS = ['tbs', 'tbm', 'start', 'near']
 
 
-def gen_user_agent(normal_ua, is_mobile):
+def gen_user_agent(is_mobile):
     mozilla = random.choice(['Moo', 'Woah', 'Bro', 'Slow']) + 'zilla'
     firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox'
     linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux'
@@ -66,17 +66,11 @@ class Request:
     def __init__(self, normal_ua, language='lang_en'):
         self.language = language
         self.mobile = 'Android' in normal_ua or 'iPhone' in normal_ua
-        self.modified_user_agent = gen_user_agent(normal_ua, self.mobile)
+        self.modified_user_agent = gen_user_agent(self.mobile)
 
     def __getitem__(self, name):
         return getattr(self, name)
 
-    def get_decode_value(self):
-        if 'lang_zh' in self.language:
-            return 'gb2312'
-        else:
-            return 'unicode-escape'
-
     def autocomplete(self, query):
         ac_query = dict(hl=self.language, q=query)
         response = self.send(base_url=AUTOCOMPLETE_URL, query=urlparse.urlencode(ac_query))
@@ -88,19 +82,13 @@ class Request:
         return []
 
     def send(self, base_url=SEARCH_URL, query='', return_bytes=False):
-        response_header = []
+        headers = {
+            'User-Agent': self.modified_user_agent
+        }
 
-        b_obj = BytesIO()
+        response = requests.get(base_url + query, headers=headers)
-        crl = pycurl.Curl()
-        crl.setopt(crl.URL, base_url + query)
-        crl.setopt(crl.USERAGENT, self.modified_user_agent)
-        crl.setopt(crl.WRITEDATA, b_obj)
-        crl.setopt(crl.HEADERFUNCTION, response_header.append)
-        crl.setopt(pycurl.FOLLOWLOCATION, 1)
-        crl.perform()
-        crl.close()
 
         if return_bytes:
-            return b_obj.getvalue()
+            return response.content
         else:
-            return b_obj.getvalue().decode(self.get_decode_value(), 'ignore')
+            return response.text

app/routes.py

@@ -1,18 +1,20 @@
 from app import app
-from app.filter import Filter, get_first_link
 from app.models.config import Config
-from app.request import Request, gen_query
+from app.request import Request
+from app.utils.misc import generate_user_keys, valid_user_session
+from app.utils.routing_utils import *
 import argparse
 import base64
 from bs4 import BeautifulSoup
-from cryptography.fernet import Fernet, InvalidToken
+from cryptography.fernet import Fernet
-from flask import g, jsonify, make_response, request, redirect, render_template, send_file
+from flask import g, jsonify, make_response, request, redirect, render_template, send_file, session
 from functools import wraps
 import io
 import json
 import os
-from pycurl import error as pycurl_error
 import urllib.parse as urlparse
+from requests import exceptions
+import uuid
 import waitress
 
 
@@ -34,17 +36,22 @@ def auth_required(f):
 
 @app.before_request
 def before_request_func():
-    # Always redirect to https if HTTPS_ONLY is set (otherwise default to false)
+    # Generate secret key for user if unavailable
+    if not valid_user_session(session):
+        session['config'] = {'url': request.url_root}
+        session['keys'] = generate_user_keys()
+        session['uuid'] = str(uuid.uuid4())
+
+    if session['uuid'] not in app.user_elements:
+        app.user_elements.update({session['uuid']: 0})
+
+    # Always redirect to https if HTTPS_ONLY is set (otherwise default to False)
     https_only = os.getenv('HTTPS_ONLY', False)
-    config_path = app.config['CONFIG_PATH']
 
     if https_only and request.url.startswith('http://'):
-        https_url = request.url.replace('http://', 'https://', 1)
+        return redirect(request.url.replace('http://', 'https://', 1), code=308)
-        code = 308
-        return redirect(https_url, code=code)
 
-    json_config = json.load(open(config_path)) if os.path.exists(config_path) else {'url': request.url_root}
+    g.user_config = Config(**session['config'])
-    g.user_config = Config(**json_config)
 
     if not g.user_config.url:
         g.user_config.url = request.url_root.replace('http://', 'https://') if https_only else request.url_root
@@ -53,6 +60,16 @@ def before_request_func():
     g.app_location = g.user_config.url
 
 
+@app.after_request
+def after_request_func(response):
+    # Regenerate element key if all elements have been served to user
+    if app.user_elements[session['uuid']] <= 0 and '/element' in request.url:
+        session['keys']['element_key'] = Fernet.generate_key()
+        app.user_elements[session['uuid']] = 0
+
+    return response
+
+
 @app.errorhandler(404)
 def unknown_page(e):
     return redirect(g.app_location)
@@ -62,14 +79,11 @@ def unknown_page(e):
 @auth_required
 def index():
     return render_template('index.html',
-                           dark_mode=g.user_config.dark,
                            ua=g.user_request.modified_user_agent,
                            languages=Config.LANGUAGES,
                            countries=Config.COUNTRIES,
-                           current_lang=g.user_config.lang,
+                           config=g.user_config,
-                           current_ctry=g.user_config.ctry,
+                           version_number=app.config['VERSION_NUMBER'])
-                           version_number=app.config['VERSION_NUMBER'],
-                           request_type='get' if g.user_config.get_only else 'post')
 
 
 @app.route('/opensearch.xml', methods=['GET'])
@@ -103,52 +117,37 @@ def autocomplete():
 @app.route('/search', methods=['GET', 'POST'])
 @auth_required
 def search():
-    request_params = request.args if request.method == 'GET' else request.form
+    # Clear previous elements and generate a new key each time a new search is performed
-    q = request_params.get('q')
+    app.user_elements[session['uuid']] = 0
+    session['keys']['element_key'] = Fernet.generate_key()
 
-    if q is None or len(q) == 0:
+    search_util = RoutingUtils(request, g.user_config, session)
+    query = search_util.new_search_query()
+
+    # Redirect to home if invalid/blank search
+    if not query:
         return redirect('/')
-    else:
-        # Attempt to decrypt if this is an internal link
-        try:
-            q = Fernet(app.secret_key).decrypt(q.encode()).decode()
-        except InvalidToken:
-            pass
 
-    feeling_lucky = q.startswith('! ')
+    # Generate response and number of external elements from the page
+    response, elements = search_util.generate_response()
+    if search_util.feeling_lucky:
+        return redirect(response, code=303)
 
-    if feeling_lucky:  # Well do you, punk?
+    # Keep count of external elements to fetch before element key can be regenerated
-        q = q[2:]
+    app.user_elements[session['uuid']] = elements
-
-    user_agent = request.headers.get('User-Agent')
-    mobile = 'Android' in user_agent or 'iPhone' in user_agent
-
-    content_filter = Filter(mobile, g.user_config, secret_key=app.secret_key)
-    full_query = gen_query(q, request_params, g.user_config, content_filter.near)
-    get_body = g.user_request.send(query=full_query)
-    dirty_soup = BeautifulSoup(content_filter.reskin(get_body), 'html.parser')
-
-    if feeling_lucky:
-        return redirect(get_first_link(dirty_soup), 303)  # Using 303 so the browser performs a GET request for the URL
-    else:
-        formatted_results = content_filter.clean(dirty_soup)
-
-    # Set search type to be used in the header template to allow for repeated searches
-    # in the same category
-    search_type = request_params.get('tbm') if 'tbm' in request_params else ''
 
     return render_template(
         'display.html',
-        query=urlparse.unquote(q),
+        query=urlparse.unquote(query),
-        search_type=search_type,
+        search_type=search_util.search_type,
         dark_mode=g.user_config.dark,
-        response=formatted_results,
+        response=response,
         search_header=render_template(
             'header.html',
             dark_mode=g.user_config.dark,
-            q=urlparse.unquote(q),
+            query=urlparse.unquote(query),
-            search_type=search_type,
+            search_type=search_util.search_type,
-            mobile=g.user_request.mobile) if 'isch' not in search_type else '')
+            mobile=g.user_request.mobile) if 'isch' not in search_util.search_type else '')
 
 
 @app.route('/config', methods=['GET', 'POST'])
@@ -161,10 +160,7 @@ def config():
         if 'url' not in config_data or not config_data['url']:
             config_data['url'] = g.user_config.url
 
-        with open(app.config['CONFIG_PATH'], 'w') as config_file:
+        session['config'] = config_data
-            config_file.write(json.dumps(config_data, indent=4))
-            config_file.close()
-
         return redirect(config_data['url'])
 
 
@@ -187,25 +183,22 @@ def imgres():
     return redirect(request.args.get('imgurl'))
 
 
-@app.route('/tmp')
+@app.route('/element')
 @auth_required
-def tmp():
+def element():
-    cipher_suite = Fernet(app.secret_key)
+    cipher_suite = Fernet(session['keys']['element_key'])
-    img_url = cipher_suite.decrypt(request.args.get('image_url').encode()).decode()
+    src_url = cipher_suite.decrypt(request.args.get('url').encode()).decode()
+    src_type = request.args.get('type')
 
     try:
-        file_data = g.user_request.send(base_url=img_url, return_bytes=True)
+        file_data = g.user_request.send(base_url=src_url, return_bytes=True)
+        app.user_elements[session['uuid']] -= 1
         tmp_mem = io.BytesIO()
         tmp_mem.write(file_data)
         tmp_mem.seek(0)
 
-        return send_file(
+        return send_file(tmp_mem, mimetype=src_type)
-            tmp_mem,
+    except exceptions.RequestException:
-            as_attachment=True,
-            attachment_filename='tmp.png',
-            mimetype='image/png'
-        )
-    except pycurl_error:
         pass
 
     empty_gif = base64.b64decode('R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==')

app/static/config/.gitignore

@@ -0,0 +1 @@
+*

app/templates/header.html

@@ -15,7 +15,7 @@
                                style="background-color: {{ '#000' if dark_mode else '#fff' }};
                                        color: {{ '#685e79' if dark_mode else '#000' }};
                                        border: {{ '1px solid #685e79' if dark_mode else '' }}"
-                               spellcheck="false" type="text" value="{{ q }}">
+                               spellcheck="false" type="text" value="{{ query }}">
                         <input name="tbm" value="{{ search_type }}" style="display: none">
                         <div class="sc"></div>
                     </div>
@@ -37,7 +37,7 @@
                 <div class="autocomplete" style="width: 100%; flex: 1">
                     <div style="width: 100%; display: flex">
                         <input id="search-bar" autocapitalize="none" autocomplete="off" class="noHIxc" name="q"
-                               spellcheck="false" type="text" value="{{ q }}"
+                               spellcheck="false" type="text" value="{{ query }}"
                                style="background-color: {{ '#000' if dark_mode else '#fff' }};
                                        color: {{ '#685e79' if dark_mode else '#000' }};
                                        border: {{ '1px solid #685e79' if dark_mode else '' }}">

app/templates/index.html

@@ -21,14 +21,14 @@
         <script type="text/javascript" src="/static/js/controller.js"></script>
         <link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Whoogle Search">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
-        <link rel="stylesheet" href="/static/css/{{ 'search-dark' if dark_mode else 'search' }}.css">
+        <link rel="stylesheet" href="/static/css/{{ 'search-dark' if config.dark else 'search' }}.css">
         <link rel="stylesheet" href="/static/css/main.css">
 	<title>Whoogle Search</title>
     </head>
-    <body id="main" style="display: none; background-color: {{ '#000' if dark_mode else '#fff' }}">
+    <body id="main" style="display: none; background-color: {{ '#000' if config.dark else '#fff' }}">
         <div class="search-container">
             <img class="logo" src="/static/img/logo.png">
-            <form id="search-form" action="/search" method="{{ request_type }}">
+            <form id="search-form" action="/search" method="{{ 'get' if config.get_only else 'post' }}">
                 <div class="search-fields">
                     <div class="autocomplete">
                         <input type="text" name="q" id="search-bar" autofocus="autofocus">
@@ -50,7 +50,7 @@
                             <select name="ctry" id="config-ctry">
                                 {% for ctry in countries %}
                                     <option value="{{ ctry.value }}"
-                                            {% if ctry.value in current_ctry %}
+                                            {% if ctry.value in config.ctry %}
                                             selected
                                             {% endif %}>
                                         {{ ctry.name }}
@@ -63,7 +63,7 @@
                             <select name="lang" id="config-lang">
                                 {% for lang in languages %}
                                 <option value="{{ lang.value }}"
-                                    {% if lang.value in current_lang %}
+                                    {% if lang.value in config.lang %}
                                         selected
                                     {% endif %}>
                                     {{ lang.name }}

app/utils/misc.py

@@ -0,0 +1,20 @@
+from cryptography.fernet import Fernet
+
+SESSION_VALS = ['uuid', 'config', 'keys']
+
+
+def generate_user_keys():
+    # Generate/regenerate unique key per user
+    return {
+        'element_key': Fernet.generate_key(),
+        'text_key': Fernet.generate_key()
+    }
+
+
+def valid_user_session(session):
+    # Generate secret key for user if unavailable
+    for value in SESSION_VALS:
+        if value not in session:
+            return False
+
+    return True

app/utils/routing_utils.py

@@ -0,0 +1,69 @@
+from app import app
+from app.filter import Filter, get_first_link
+from app.request import gen_query
+from bs4 import BeautifulSoup
+from cryptography.fernet import Fernet, InvalidToken
+from flask import g
+from typing import Any, Tuple
+
+
+class RoutingUtils:
+    def __init__(self, request, config, session):
+        self.request_params = request.args if request.method == 'GET' else request.form
+        self.user_agent = request.headers.get('User-Agent')
+        self.feeling_lucky = False
+        self.config = config
+        self.session = session
+        self.query = ''
+        self.search_type = self.request_params.get('tbm') if 'tbm' in self.request_params else ''
+
+    def __getitem__(self, name):
+        return getattr(self, name)
+
+    def __setitem__(self, name, value):
+        return setattr(self, name, value)
+
+    def __delitem__(self, name):
+        return delattr(self, name)
+
+    def __contains__(self, name):
+        return hasattr(self, name)
+
+    def new_search_query(self) -> str:
+        app.user_elements[self.session['uuid']] = 0
+        self.session['keys']['element_key'] = Fernet.generate_key()
+
+        q = self.request_params.get('q')
+
+        if q is None or len(q) == 0:
+            return ''
+        else:
+            # Attempt to decrypt if this is an internal link
+            try:
+                q = Fernet(self.session['keys']['text_key']).decrypt(q.encode()).decode()
+            except InvalidToken:
+                pass
+
+        # Reset text key
+        self.session['keys']['text_key'] = Fernet.generate_key()
+
+        # Format depending on whether or not the query is a "feeling lucky" query
+        self.feeling_lucky = q.startswith('! ')
+        self.query = q[2:] if self.feeling_lucky else q
+        return self.query
+
+    def generate_response(self) -> Tuple[Any, int]:
+        mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent
+
+        content_filter = Filter(self.session['keys'], mobile=mobile, config=self.config)
+        full_query = gen_query(self.query, self.request_params, self.config, content_filter.near)
+        get_body = g.user_request.send(query=full_query)
+
+        # Produce cleanable html soup from response
+        html_soup = BeautifulSoup(content_filter.reskin(get_body), 'html.parser')
+
+        if self.feeling_lucky:
+            return get_first_link(html_soup), 1
+        else:
+            formatted_results = content_filter.clean(html_soup)
+            return formatted_results, content_filter.elements

requirements.txt

@@ -4,15 +4,16 @@ cffi==1.13.2
 Click==7.0
 cryptography==2.8
 Flask==1.1.1
+Flask-Session==0.3.2
 itsdangerous==1.1.0
 Jinja2==2.10.3
 lxml==4.5.1
 MarkupSafe==1.1.1
 pycparser==2.19
-pycurl==7.43.0.4
 pyOpenSSL==19.1.0
 pytest==5.4.1
 python-dateutil==2.8.1
+requests==2.23.0
 six==1.14.0
 soupsieve==1.9.5
 Werkzeug==0.16.0

test/test_misc.py

@@ -0,0 +1,36 @@
+from app.utils.misc import generate_user_keys, valid_user_session
+
+
+def test_generate_user_keys():
+    keys = generate_user_keys()
+    assert 'text_key' in keys
+    assert 'element_key' in keys
+    assert keys['text_key'] not in keys['element_key']
+
+
+def test_valid_session(client):
+    with client.session_transaction() as session:
+        assert not valid_user_session(session)
+
+        session['uuid'] = 'test'
+        session['keys'] = generate_user_keys()
+        session['config'] = {}
+
+        assert valid_user_session(session)
+
+
+def test_request_key_generation(client):
+    text_key = ''
+    rv = client.get('/search?q=test+1')
+    assert rv._status_code == 200
+
+    with client.session_transaction() as session:
+        assert valid_user_session(session)
+        text_key = session['keys']['text_key']
+
+    rv = client.get('/search?q=test+2')
+    assert rv._status_code == 200
+
+    with client.session_transaction() as session:
+        assert valid_user_session(session)
+        assert text_key not in session['keys']['text_key']

test/test_results.py

@@ -1,13 +1,13 @@
 from bs4 import BeautifulSoup
-from cryptography.fernet import Fernet
 from app.filter import Filter
+from app.utils.misc import generate_user_keys
 from datetime import datetime
 from dateutil.parser import *
 
 
 def get_search_results(data):
-    secret_key = Fernet.generate_key()
+    secret_key = generate_user_keys()
-    soup = Filter(secret_key=secret_key).clean(BeautifulSoup(data, 'html.parser'))
+    soup = Filter(user_keys=secret_key).clean(BeautifulSoup(data, 'html.parser'))
 
     main_divs = soup.find('div', {'id': 'main'})
     assert len(main_divs) > 1

test/test_routes.py

@@ -1,10 +1,13 @@
+from app.models.config import Config
 import json
 import random
 
 demo_config = {
     'near': random.choice(['Seattle', 'New York', 'San Francisco']),
     'dark_mode': str(random.getrandbits(1)),
-    'nojs': str(random.getrandbits(1))
+    'nojs': str(random.getrandbits(1)),
+    'lang': random.choice(Config.LANGUAGES)['value'],
+    'ctry': random.choice(Config.COUNTRIES)['value']
 }
 
 
@@ -17,6 +20,7 @@ def test_search(client):
     rv = client.get('/search?q=test')
     assert rv._status_code == 200
 
+
 def test_feeling_lucky(client):
     rv = client.get('/search?q=!%20test')
     assert rv._status_code == 303