From e0c88359f577bcc82ba4c323991a728b3a4114a5 Mon Sep 17 00:00:00 2001
From: Ben Busby <33362396+benbusby@users.noreply.github.com>
Date: Sat, 30 May 2020 09:55:02 -0600
Subject: [PATCH] Result formatting and removal of unused elements

- Fixed question section formatting from results page (added appropriate
padding and made questions styled as italic)
- Removed user agent display from main config settings
---
 app/filter.py            | 24 ++++++++++++++++++------
 app/request.py           |  1 -
 app/routes.py            |  1 -
 app/templates/index.html |  4 ----
 4 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/app/filter.py b/app/filter.py
index 3ef65ba..4a4856f 100644
--- a/app/filter.py
+++ b/app/filter.py
@@ -1,5 +1,6 @@
 from app.request import VALID_PARAMS
 from bs4 import BeautifulSoup
+from bs4.element import ResultSet
 from cryptography.fernet import Fernet
 import re
 import urllib.parse as urlparse
@@ -57,6 +58,7 @@ class Filter:
         self.new_tab = config['new_tab'] if 'new_tab' in config else False
         self.mobile = mobile
         self.user_keys = user_keys
+        self.main_divs = ResultSet('')
         self._elements = 0
 
     def __getitem__(self, name):
@@ -88,7 +90,9 @@ class Filter:
         return Fernet(self.user_keys['text_key']).encrypt(msg.encode()).decode()
 
     def clean(self, soup):
-        self.remove_ads(soup)
+        self.main_divs = soup.find('div', {'id': 'main'})
+        self.remove_ads()
+        self.fix_question_section()
         self.update_styling(soup)
 
         for img in [_ for _ in soup.find_all('img') if 'src' in _.attrs]:
@@ -123,16 +127,24 @@ class Filter:
 
         return soup
 
-    def remove_ads(self, soup):
-        main_divs = soup.find('div', {'id': 'main'})
-        if main_divs is None:
+    def remove_ads(self):
+        if not self.main_divs:
             return
-        result_divs = main_divs.find_all('div', recursive=False)
 
-        for div in [_ for _ in result_divs]:
+        for div in [_ for _ in self.main_divs.find_all('div', recursive=True)]:
             has_ad = len([_ for _ in div.find_all('span', recursive=True) if 'ad' == _.text.lower()])
             _ = div.decompose() if has_ad else None
 
+    def fix_question_section(self):
+        if not self.main_divs:
+            return
+
+        question_divs = [_ for _ in self.main_divs.find_all('div', recursive=False) if len(_.find_all('h2')) > 0]
+        for x in question_divs:
+            questions = [_ for _ in x.find_all('div', recursive=True) if _.text.endswith('?')]
+            for question in questions:
+                question['style'] = 'padding: 10px; font-style: italic;'
+
     def update_element_src(self, element, mimetype):
         element_src = element['src']
         if element_src.startswith('//'):
diff --git a/app/request.py b/app/request.py
index de60638..12e1c1e 100644
--- a/app/request.py
+++ b/app/request.py
@@ -1,4 +1,3 @@
-from io import BytesIO
 from lxml import etree
 import random
 import requests
diff --git a/app/routes.py b/app/routes.py
index 26dcd1d..e1e5812 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -79,7 +79,6 @@ def unknown_page(e):
 @auth_required
 def index():
     return render_template('index.html',
-                           ua=g.user_request.modified_user_agent,
                            languages=Config.LANGUAGES,
                            countries=Config.COUNTRIES,
                            config=g.user_config,
diff --git a/app/templates/index.html b/app/templates/index.html
index e85d796..2fba873 100644
--- a/app/templates/index.html
+++ b/app/templates/index.html
@@ -41,10 +41,6 @@
             <div class="content">
                 <div class="config-fields">
                     <form id="config-form" action="/config" method="post">
-                        <div class="config-div">
-                            <!-- TODO: Add option to regenerate user agent? -->
-                            <span class="ua-span">User Agent: {{ ua }}</span>
-                        </div>
                         <div class="config-div">
                             <label for="config-ctry">Country: </label>
                             <select name="ctry" id="config-ctry">