open.source/whoogle-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feature/public-instance-sessions

Browse Source
This commit is contained in:
Ben Busby 2021-11-01 17:05:15 -06:00
commit 50d592bcec
No known key found for this signature in database
GPG Key ID: 339B7B7EB5333D14
13 changed files with 363 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/request.py
View File

@ -9,8 +9,6 @@ import os
from stem import Signal, SocketError
from stem.control import Controller
SEARCH_URL = 'https://www.google.com/search?gbv=1&num=' + str(
os.getenv('WHOOGLE_RESULTS_PER_PAGE', 10)) + '&q='
MAPS_URL = 'https://maps.google.com/maps'
AUTOCOMPLETE_URL = ('https://suggestqueries.google.com/'
'complete/search?client=toolbar&')
@ -151,6 +149,8 @@ class Request:
"""
def __init__(self, normal_ua, root_path, config: Config):
self.search_url = 'https://www.google.com/search?gbv=1&num=' + str(
os.getenv('WHOOGLE_RESULTS_PER_PAGE', 10)) + '&q='
# Send heartbeat to Tor, used in determining if the user can or cannot
# enable Tor for future requests
send_tor_signal(Signal.HEARTBEAT)
@ -224,7 +224,7 @@ class Request:
return [_.attrib['data'] for _ in
root.findall('.//suggestion/[@data]')]
def send(self, base_url=SEARCH_URL, query='', attempt=0,
def send(self, base_url='', query='', attempt=0,
force_mobile=False) -> Response:
"""Sends an outbound request to a URL. Optionally sends the request
using Tor, if enabled by the user.
@ -285,7 +285,7 @@ class Request:
disable=True)
response = requests.get(
base_url + query,
(base_url or self.search_url) + query,
proxies=self.proxies,
headers=headers,
cookies=cookies)
@ -295,6 +295,6 @@ class Request:
attempt += 1
if attempt > 10:
raise TorError("Tor query failed -- max attempts exceeded 10")
return self.send(base_url, query, attempt)
return self.send((base_url or self.search_url), query, attempt)
return response

17
app/routes.py
View File

@ -14,7 +14,7 @@ from app import app
from app.models.config import Config
from app.request import Request, TorError
from app.utils.bangs import resolve_bang
from app.utils.misc import read_config_bool
from app.utils.misc import read_config_bool, get_client_ip
from app.utils.results import add_ip_card
from app.utils.results import bold_search_terms
from app.utils.search import *
@ -301,10 +301,11 @@ def search():
# Return 503 if temporarily blocked by captcha
resp_code = 503 if has_captcha(str(response)) else 200
response = bold_search_terms(response, query)
# Feature to display IP address
if search_util.check_kw_ip():
html_soup = bsoup(response, "html.parser")
response = add_ip_card(html_soup, request.remote_addr)
html_soup = bsoup(str(response), 'html.parser')
response = add_ip_card(html_soup, get_client_ip(request))
return render_template(
'display.html',
@ -322,7 +323,7 @@ def search():
is_translation=any(
_ in query.lower() for _ in [translation['translate'], 'translate']
) and not search_util.search_type, # Standard search queries only
response=html.unescape(str(response)),
response=response,
version_number=app.config['VERSION_NUMBER'],
search_header=(render_template(
'header.html',
@ -434,7 +435,13 @@ def window():
for script in results('script'):
script.decompose()
return render_template('display.html', response=results)
return render_template(
'display.html',
response=results,
translation=app.config['TRANSLATIONS'][
g.user_config.get_localization_lang()
]
)
def run_app() -> None:

1
app/static/css/main.css
View File

@ -102,6 +102,7 @@ button::-moz-focus-inner {
}
.open {
overflow-y: scroll;
padding-bottom: 20px;
}

2
app/static/js/controller.js
View File

@ -26,7 +26,7 @@ const setupConfigLayout = () => {
if (content.style.maxHeight) {
content.style.maxHeight = null;
} else {
content.style.maxHeight = content.scrollHeight + "px";
content.style.maxHeight = "400px";
}
content.classList.toggle("open");

45
app/templates/index.html
View File

@ -36,9 +36,20 @@
<link rel="stylesheet" href="{{ cb_url('main.css') }}">
<noscript>
<style>
#main { display: inherit !important; }
.content { max-height: 720px; padding: 18px; border-radius: 10px; }
.collapsible { display: none; }
#main {
display: inherit !important;
}
.content {
max-height: 400px;
padding: 18px;
border-radius: 10px;
overflow-y: scroll;
}
.collapsible {
display: none;
}
</style>
</noscript>
<style>{{ config.style }}</style>
@ -115,19 +126,24 @@
</div>
<div class="config-div config-div-near">
<label for="config-near">{{ translation['config-near'] }}: </label>
<input type="text" name="near" id="config-near" placeholder="{{ translation['config-near-help'] }}" value="{{ config.near }}">
<input type="text" name="near" id="config-near"
placeholder="{{ translation['config-near-help'] }}" value="{{ config.near }}">
</div>
<div class="config-div config-div-block">
<label for="config-block">{{ translation['config-block'] }}: </label>
<input type="text" name="block" id="config-block" placeholder="{{ translation['config-block-help'] }}" value="{{ config.block }}">
<input type="text" name="block" id="config-block"
placeholder="{{ translation['config-block-help'] }}" value="{{ config.block }}">
</div>
<div class="config-div config-div-block">
<label for="config-block-title">{{ translation['config-block-title'] }}: </label>
<input type="text" name="block_title" id="config-block" placeholder="{{ translation['config-block-title-help'] }}" value="{{ config.block_title }}">
<input type="text" name="block_title" id="config-block"
placeholder="{{ translation['config-block-title-help'] }}"
value="{{ config.block_title }}">
</div>
<div class="config-div config-div-block">
<label for="config-block-url">{{ translation['config-block-url'] }}: </label>
<input type="text" name="block_url" id="config-block" placeholder="{{ translation['config-block-url-help'] }}" value="{{ config.block_url }}">
<input type="text" name="block_url" id="config-block"
placeholder="{{ translation['config-block-url-help'] }}" value="{{ config.block_url }}">
</div>
<div class="config-div config-div-nojs">
<label for="config-nojs">{{ translation['config-nojs'] }}: </label>
@ -162,24 +178,29 @@
</div>
<div class="config-div config-div-new-tab">
<label for="config-new-tab">{{ translation['config-new-tab'] }}: </label>
<input type="checkbox" name="new_tab" id="config-new-tab" {{ 'checked' if config.new_tab else '' }}>
<input type="checkbox" name="new_tab"
id="config-new-tab" {{ 'checked' if config.new_tab else '' }}>
</div>
<div class="config-div config-div-view-image">
<label for="config-view-image">{{ translation['config-images'] }}: </label>
<input type="checkbox" name="view_image" id="config-view-image" {{ 'checked' if config.view_image else '' }}>
<input type="checkbox" name="view_image"
id="config-view-image" {{ 'checked' if config.view_image else '' }}>
<div><span class="info-text"> — {{ translation['config-images-help'] }}</span></div>
</div>
<div class="config-div config-div-tor">
<label for="config-tor">{{ translation['config-tor'] }}: {{ '' if tor_available else 'Unavailable' }}</label>
<input type="checkbox" name="tor" id="config-tor" {{ '' if tor_available else 'hidden' }} {{ 'checked' if config.tor else '' }}>
<input type="checkbox" name="tor"
id="config-tor" {{ '' if tor_available else 'hidden' }} {{ 'checked' if config.tor else '' }}>
</div>
<div class="config-div config-div-get-only">
<label for="config-get-only">{{ translation['config-get-only'] }}: </label>
<input type="checkbox" name="get_only" id="config-get-only" {{ 'checked' if config.get_only else '' }}>
<input type="checkbox" name="get_only"
id="config-get-only" {{ 'checked' if config.get_only else '' }}>
</div>
<div class="config-div config-div-get-only">
<label for="config-accept-language">Set Accept-Language: </label>
<input type="checkbox" name="accept_language" id="config-accept-language" {{ 'checked' if config.accept_language else '' }}>
<input type="checkbox" name="accept_language"
id="config-accept-language" {{ 'checked' if config.accept_language else '' }}>
</div>
<div class="config-div config-div-root-url">
<label for="config-url">{{ translation['config-url'] }}: </label>

7
app/templates/opensearch.xml
View File

File diff suppressed because one or more lines are too long

10
app/utils/bangs.py
View File

@ -55,9 +55,11 @@ def resolve_bang(query: str, bangs_dict: dict) -> str:
query = query.lower()
split_query = query.split(' ')
for operator in bangs_dict.keys():
if operator not in split_query:
if operator not in split_query \
and operator[1:] + operator[0] not in split_query:
continue
return bangs_dict[operator]['url'].format(
query.replace(operator, '').strip())
return bangs_dict[operator]['url'].replace(
'{}',
query.replace(operator if operator in split_query
else operator[1:] + operator[0], '').strip(), 1)
return ''

8
app/utils/misc.py
View File

@ -1,3 +1,4 @@
from flask import Request
import hashlib
import os
@ -15,3 +16,10 @@ def read_config_bool(var: str) -> bool:
if val.isdigit():
return bool(int(val))
return False
def get_client_ip(r: Request) -> str:
if r.environ.get('HTTP_X_FORWARDED_FOR') is None:
return r.environ['REMOTE_ADDR']
else:
return r.environ['HTTP_X_FORWARDED_FOR']

15
app/utils/results.py
View File

@ -1,4 +1,5 @@
from bs4 import BeautifulSoup, NavigableString
import html
import os
import urllib.parse as urlparse
from urllib.parse import parse_qs
@ -52,11 +53,15 @@ def bold_search_terms(response: str, query: str) -> BeautifulSoup:
if len(element) == len(target_word):
return
element.replace_with(
if not re.match('.*[a-zA-Z0-9].*', target_word) or (
element.parent and element.parent.name == 'style'):
return
element.replace_with(BeautifulSoup(
re.sub(fr'\b((?![{{}}<>-]){target_word}(?![{{}}<>-]))\b',
r'<b>\1</b>',
element,
flags=re.I)
html.escape(element),
flags=re.I), 'html.parser')
)
# Split all words out of query, grouping the ones wrapped in quotes
@ -173,9 +178,7 @@ def append_nojs(result: BeautifulSoup) -> None:
"""
nojs_link = BeautifulSoup(features='html.parser').new_tag('a')
nojs_link['href'] = '/window?location=' + result['href']
nojs_link['style'] = 'display:block;width:100%;'
nojs_link.string = 'NoJS Link: ' + nojs_link['href']
result.append(BeautifulSoup('<br><hr><br>', 'html.parser'))
nojs_link.string = ' NoJS Link'
result.append(nojs_link)

4
run
View File

@ -12,6 +12,10 @@ SUBDIR="${1:-app}"
export APP_ROOT="$SCRIPT_DIR/$SUBDIR"
export STATIC_FOLDER="$APP_ROOT/static"
# Clear out build directory
rm -f "$SCRIPT_DIR"/app/static/build/*.js
rm -f "$SCRIPT_DIR"/app/static/build/*.css
# Check for regular vs test run
if [[ "$SUBDIR" == "test" ]]; then
# Set up static files for testing

5
test/test_routes.py
View File

@ -36,6 +36,11 @@ def test_ddg_bang(client):
assert rv._status_code == 302
assert rv.headers.get('Location').startswith('https://www.reddit.com')
# Move '!' to end of the bang
rv = client.get('/search?q=gitlab%20w!')
assert rv._status_code == 302
assert rv.headers.get('Location').startswith('https://en.wikipedia.org')
# Ensure bang is case insensitive
rv = client.get('/search?q=!GH%20whoogle')
assert rv._status_code == 302