Merge branch 'main' into dev-header-tabs

This commit is contained in:
Ben Busby 2022-02-04 16:39:22 -07:00 committed by GitHub
commit 3f5e76045f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 189 additions and 71 deletions

View File

@ -2,10 +2,13 @@ name: buildx
on:
workflow_run:
workflows: ["docker_tests"]
workflows: ["docker_main"]
branches: [main]
types:
- completed
push:
tags:
- '*'
jobs:
on-success:
@ -21,10 +24,17 @@ jobs:
uses: crazy-max/ghaction-docker-buildx@v1
with:
version: latest
- name: log in to docker hub
run: |
echo "${{ secrets.DOCKER_PASSWORD }}" | \
docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to ghcr.io
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: build and push the image
if: startsWith(github.ref, 'refs/heads/main') && github.actor == 'benbusby'
run: |
@ -33,6 +43,9 @@ jobs:
docker buildx build --push \
--tag benbusby/whoogle-search:latest \
--platform linux/amd64,linux/arm/v7,linux/arm64 .
docker buildx build --push \
--tag ghcr.io/benbusby/whoogle-search:latest \
--platform linux/amd64,linux/arm/v7,linux/arm64 .
- name: build and push tag
if: startsWith(github.ref, 'refs/tags')
run: |
@ -41,3 +54,6 @@ jobs:
docker buildx build --push \
--tag benbusby/whoogle-search:${GITHUB_REF#refs/*/v}\
--platform linux/amd64,linux/arm/v7,linux/arm64 .
docker buildx build --push \
--tag ghcr.io/benbusby/whoogle-search:${GITHUB_REF#refs/*/v}\
--platform linux/amd64,linux/arm/v7,linux/arm64 .

28
.github/workflows/docker_main.yml vendored Normal file
View File

@ -0,0 +1,28 @@
name: docker_main
on:
workflow_run:
workflows: ["tests"]
branches: [main]
types:
- completed
# TODO: Needs refactoring to use reusable workflows and share w/ docker_tests
jobs:
on-success:
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/checkout@v2
- name: build and test (docker)
run: |
docker build --tag whoogle-search:test .
docker run --publish 5000:5000 --detach --name whoogle-search-nocompose whoogle-search:test
sleep 15
docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1
- name: build and test (docker-compose)
run: |
docker rm -f whoogle-search-nocompose
WHOOGLE_IMAGE="whoogle-search:test" docker-compose up --detach
sleep 15
docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1

View File

@ -1,21 +1,26 @@
name: docker_tests
on:
workflow_run:
workflows: ["tests"]
branches: [main]
types:
- completed
push:
branches: main
pull_request:
branches: main
jobs:
on-success:
docker:
runs-on: ubuntu-latest
steps:
- name: checkout code
uses: actions/checkout@v2
- name: build and test
- name: build and test (docker)
run: |
docker build --tag whoogle-search:test .
docker run --publish 5000:5000 --detach --name whoogle-search whoogle-search:test
docker run --publish 5000:5000 --detach --name whoogle-search-nocompose whoogle-search:test
sleep 15
docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1
- name: build and test (docker-compose)
run: |
docker rm -f whoogle-search-nocompose
WHOOGLE_IMAGE="whoogle-search:test" docker-compose up --detach
sleep 15
docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1

19
.github/workflows/scan.yml vendored Normal file
View File

@ -0,0 +1,19 @@
name: scan
on:
schedule:
- cron: '0 0 * * *'
jobs:
scan:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Build the container image
run: |
docker build --tag whoogle-search:test .
- name: Initiate grype scan
run: |
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b .
chmod +x ./grype
./grype whoogle-search:test --only-fixed

View File

@ -14,9 +14,11 @@ RUN pip install --prefix /install --no-warn-script-location --no-cache-dir -r re
FROM python:3.8-alpine
RUN apk add --update --no-cache tor curl bash openrc
RUN apk add --update --no-cache tor curl openrc
# libcurl4-openssl-dev
RUN apk -U upgrade
ARG DOCKER_USER=whoogle
ARG DOCKER_USERID=927
ARG config_dir=/config
@ -69,14 +71,15 @@ COPY app/ app/
COPY run .
#COPY whoogle.env .
# Allow writing symlinks to build dir
RUN chown 102:102 app/static/build
# Create user/group to run as
RUN adduser -D -g $DOCKER_USERID -u $DOCKER_USERID $DOCKER_USER
# Fix ownership / permissions
RUN chown -R ${DOCKER_USER}:${DOCKER_USER} /whoogle /var/lib/tor
# Allow writing symlinks to build dir
RUN chown $DOCKER_USERID:$DOCKER_USERID app/static/build
USER $DOCKER_USER:$DOCKER_USER
EXPOSE $EXPOSE_PORT

View File

@ -512,12 +512,11 @@ A lot of the app currently piggybacks on Google's existing support for fetching
| Website | Country | Language | Cloudflare |
|-|-|-|-|
| [https://whoogle.fossho.st](https://whoogle.fossho.st) | 🇺🇸 US | Multi-choice | |
| [https://whoogle.fossho.st](https://whoogle.fossho.st) (down until further notice) | 🇺🇸 US | Multi-choice | |
| [https://search.albony.xyz](https://search.albony.xyz/) | 🇮🇳 IN | Multi-choice | |
| [https://whoogle.sdf.org](https://whoogle.sdf.org) | 🇺🇸 US | Multi-choice |
| [https://search.garudalinux.org](https://search.garudalinux.org) | 🇩🇪 DE | Multi-choice | |
| [https://whooglesearch.net](https://whooglesearch.net) | 🇩🇪 DE | Spanish | |
| [https://search.exonip.de](https://search.exonip.de) | 🇳🇱 NL | Multi-choice | |
| [https://s.alefvanoon.xyz](https://s.alefvanoon.xyz) | 🇺🇸 US | Multi-choice | ✅ |
| [https://www.whooglesearch.ml](https://www.whooglesearch.ml) | 🇺🇸 US | English | |
| [https://search.sethforprivacy.com](https://search.sethforprivacy.com) | 🇩🇪 DE | English | |

View File

@ -2,13 +2,14 @@ from app.filter import clean_query
from app.request import send_tor_signal
from app.utils.session import generate_user_key
from app.utils.bangs import gen_bangs_json
from app.utils.misc import gen_file_hash
from app.utils.misc import gen_file_hash, read_config_bool
from flask import Flask
from flask_session import Session
import json
import logging.config
import os
from stem import Signal
import threading
from dotenv import load_dotenv
app = Flask(__name__, static_folder=os.path.dirname(
@ -29,7 +30,7 @@ if os.getenv('HTTPS_ONLY'):
app.config['SESSION_COOKIE_NAME'] = '__Secure-session'
app.config['SESSION_COOKIE_SECURE'] = True
app.config['VERSION_NUMBER'] = '0.7.0'
app.config['VERSION_NUMBER'] = '0.7.1'
app.config['APP_ROOT'] = os.getenv(
'APP_ROOT',
os.path.dirname(os.path.abspath(__file__)))
@ -58,7 +59,7 @@ app.config['CONFIG_PATH'] = os.getenv(
app.config['DEFAULT_CONFIG'] = os.path.join(
app.config['CONFIG_PATH'],
'config.json')
app.config['CONFIG_DISABLE'] = os.getenv('WHOOGLE_CONFIG_DISABLE', '')
app.config['CONFIG_DISABLE'] = read_config_bool('WHOOGLE_CONFIG_DISABLE')
app.config['SESSION_FILE_DIR'] = os.path.join(
app.config['CONFIG_PATH'],
'session')
@ -98,7 +99,11 @@ if not os.path.exists(app.config['SESSION_FILE_DIR']):
if not os.path.exists(app.config['BANG_PATH']):
os.makedirs(app.config['BANG_PATH'])
if not os.path.exists(app.config['BANG_FILE']):
gen_bangs_json(app.config['BANG_FILE'])
json.dump({}, open(app.config['BANG_FILE'], 'w'))
bangs_thread = threading.Thread(
target=gen_bangs_json,
args=(app.config['BANG_FILE'],))
bangs_thread.start()
# Build new mapping of static files for cache busting
if not os.path.exists(app.config['BUILD_FOLDER']):

View File

@ -12,6 +12,8 @@ import urllib.parse as urlparse
from urllib.parse import parse_qs
import os
minimal_mode_sections = ['Top stories', 'Images', 'People also ask']
def extract_q(q_str: str, href: str) -> str:
"""Extracts the 'q' element from a result link. This is typically
@ -180,7 +182,11 @@ class Filter:
for result in self.main_divs:
result_children = pull_child_divs(result)
if minimal_mode:
if len(result_children) in (1, 3):
if any(f">{x}</span" in str(s) for s in result_children
for x in minimal_mode_sections):
result.decompose()
continue
if len(result_children) < self.RESULT_CHILD_LIMIT:
continue
else:
if len(result_children) < self.RESULT_CHILD_LIMIT:

View File

@ -3,6 +3,7 @@ import base64
import io
import os
import json
import os
import pickle
import urllib.parse as urlparse
import uuid
@ -28,7 +29,7 @@ from requests.models import PreparedRequest
from cryptography.fernet import Fernet
# Load DDG bang json files only on init
bang_json = json.load(open(app.config['BANG_FILE']))
bang_json = json.load(open(app.config['BANG_FILE'])) or {}
# Check the newest version of WHOOGLE
update = bsoup(get(app.config['RELEASES_URL']).text, 'html.parser')
@ -103,6 +104,8 @@ def session_required(f):
@app.before_request
def before_request_func():
global bang_json
g.request_params = (
request.args if request.method == 'GET' else request.form
)
@ -152,6 +155,15 @@ def before_request_func():
g.app_location = g.user_config.url
# Attempt to reload bangs json if not generated yet
if not bang_json and os.path.getsize(app.config['BANG_FILE']) > 4:
try:
bang_json = json.load(open(app.config['BANG_FILE']))
except json.decoder.JSONDecodeError:
# Ignore decoding error, can occur if file is still
# being written
pass
@app.after_request
def after_request_func(resp):

View File

@ -59,21 +59,27 @@ select {
.ZINbbc {
overflow: hidden;
box-shadow: 0 0 0 0 !important;
background-color: var(--whoogle-dark-result-bg) !important;
}
.KP7LCb {
box-shadow: 0 0 0 0 !important;
}
.BVG0Nb {
background-color: var(--whoogle-dark-result-bg) !important;
box-shadow: 0 0 0 0 !important;
background-color: var(--whoogle-dark-page-bg) !important;
}
.ZINbbc.luh4tb {
background: var(--whoogle-dark-result-bg) !important;
}
.bRsWnc {
background-color: var(--whoogle-dark-result-bg) !important;
}
.BVG0Nb {
background-color: var(--whoogle-dark-page-bg) !important;
}
.x54gtf {
background-color: var(--whoogle-dark-divider) !important;
}

View File

@ -41,6 +41,10 @@ select {
background-color: var(--whoogle-result-bg) !important;
}
.ZINbbc.luh4tb {
background: var(--whoogle-result-bg) !important;
}
.bRsWnc {
background-color: var(--whoogle-result-bg) !important;
}

View File

@ -1,3 +1,12 @@
body {
display: block !important;
margin: auto !important;
}
.vvjwJb {
font-size: 16px !important;
}
.autocomplete {
position: relative;
display: inline-block;
@ -43,3 +52,9 @@ details summary span {
.ip-text-div {
padding-top: 0 !important;
}
@media (min-width: 801px) {
body {
min-width: 736px !important;
}
}

View File

@ -35,7 +35,7 @@
"dark": "dark",
"system": "system",
"ratelimit": "Instance has been ratelimited",
"continue-search": "Continue your search with ",
"continue-search": "Continue your search with Farside",
"all": "All",
"images": "Images",
"maps": "Maps",
@ -79,7 +79,7 @@
"dark": "donker",
"system": "systeeminstellingen",
"ratelimit": "Instantie is beperkt in snelheid",
"continue-search": "Ga verder met zoeken met "
"continue-search": "Ga verder met zoeken met Farside"
},
"lang_de": {
"search": "Suchen",
@ -117,7 +117,7 @@
"dark": "dunkel",
"system": "Systemeinstellung",
"ratelimit": "Instanz wurde ratenbegrenzt",
"continue-search": "Setzen Sie Ihre Suche fort mit "
"continue-search": "Setzen Sie Ihre Suche fort mit Farside"
},
"lang_es": {
"search": "Buscar",
@ -155,7 +155,7 @@
"dark": "oscuro",
"system": "configuración del sistema",
"ratelimit": "La instancia ha sido ratelimited",
"continue-search": "Continúe su búsqueda con "
"continue-search": "Continúe su búsqueda con Farside"
},
"lang_it": {
"search": "Cerca",
@ -193,7 +193,7 @@
"dark": "notte",
"system": "impostazioni di sistema",
"ratelimit": "L'istanza è stata limitata alla velocità",
"continue-search": "Continua la tua ricerca con "
"continue-search": "Continua la tua ricerca con Farside"
},
"lang_pt": {
"search": "Pesquisar",
@ -231,7 +231,7 @@
"dark": "escuro",
"system": "configuração de sistema",
"ratelimit": "A instância foi limitada pela taxa",
"continue-search": "Continue sua pesquisa com "
"continue-search": "Continue sua pesquisa com Farside"
},
"lang_ru": {
"search": "Поиск",
@ -269,7 +269,7 @@
"dark": "темное",
"system": "системное",
"ratelimit": "Число экземпляров ограничено",
"continue-search": "Продолжайте поиск с "
"continue-search": "Продолжайте поиск с Farside"
},
"lang_zh-CN": {
"search": "搜索",
@ -307,7 +307,7 @@
"dark": "黑暗的",
"system": "系统设置",
"ratelimit": "实例已被限速",
"continue-search": "继续搜索 "
"continue-search": "继续搜索 Farside"
},
"lang_si": {
"search": "සොයන්න",
@ -320,13 +320,13 @@
"config-block": "අවහිර",
"config-block-help": "අල්ප විරාම වලින් වෙන් වූ අඩවි ලැයිස්තුව",
"config-block-title": "මාතෘකාව අනුව අවහිර කරන්න",
"config-block-title-help": "රෙජෙක්ස් භාවිතා කරන්න",
"config-block-url": "යූආර්එල් මඟින් අවහිර කරන්න",
"config-block-title-help": "වාක්‍යවිධි භාවිතා කරන්න",
"config-block-url": "ඒ.ස.නි. මඟින් අවහිර කරන්න",
"config-block-url-help": "රෙජෙක්ස් භාවිතා කරන්න",
"config-theme": "තේමාව",
"config-nojs": "නෝජේඑස් සබැඳි පෙන්වන්න",
"config-dark": "අඳුරු ආකාරය",
"config-safe": "ආරක්ෂිත සෙවුම",
"config-safe": "ආරක්ෂිත සෙවුම",
"config-alts": "සමාජ මාධ්‍ය සබැඳි ප්‍රතිස්ථාපනය කරන්න",
"config-alts-help": "ට්විටර්/යූ ටියුබ්/ඉන්ස්ටග්‍රෑම් ආදී සබැඳි පෞද්ගලිකත්වයට ගරු කරන විකල්ප සමඟ ප්‍රතිස්ථාපනය කරයි.",
"config-new-tab": "නව තීරුවකින් සබැඳි විවෘත කරන්න",
@ -344,8 +344,8 @@
"light": "දීප්තිමත්",
"dark": "අඳුරු",
"system": "පද්ධතිය",
"ratelimit": "උදාහරණය අනුපාත කර ඇත",
"continue-search": "සමඟ ඔබේ සෙවීම දිගටම කරගෙන යන්න"
"ratelimit": "සේවාදායකය අනුපාතනය කර ඇත",
"continue-search": "Farside සමඟ ඔබගේ සෙවුම කරගෙන යන්න"
},
"lang_fr": {
"search": "Chercher",
@ -383,7 +383,7 @@
"dark": "sombre",
"system": "système",
"ratelimit": "Le débit de l'instance a été limité",
"continue-search": "Continuez votre recherche avec "
"continue-search": "Continuez votre recherche avec Farside"
},
"lang_fa": {
"search": "جستجو",
@ -421,7 +421,7 @@
"dark": "تیره",
"system": "سیستم",
"ratelimit": "نمونه با نرخ محدود شده است",
"continue-search": "جستجوی خود را با "
"continue-search": "Farside جستجوی خود را با "
},
"lang_cs": {
"search": "Hledat",
@ -459,7 +459,7 @@
"dark": "Tmavý",
"system": "Systémový",
"ratelimit": "Instance byla omezena sazbou",
"continue-search": "Pokračujte ve vyhledávání pomocí "
"continue-search": "Pokračujte ve vyhledávání pomocí Farside"
},
"lang_zh-TW": {
"search": "搜尋",
@ -497,7 +497,7 @@
"dark": "黑暗的",
"system": "依系統",
"ratelimit": "實例已被限速",
"continue-search": "繼續搜索 "
"continue-search": "繼續搜索 Farside"
},
"lang_bg": {
"search": "Търсене",
@ -535,7 +535,7 @@
"dark": "тъмна",
"system": "системна",
"ratelimit": "Екземплярът е с ограничена скорост",
"continue-search": "Продължете търсенето си с "
"continue-search": "Продължете търсенето си с Farside"
},
"lang_hi": {
"search": "खोज",
@ -573,7 +573,7 @@
"dark": "अंधेरा",
"system": "प्रणाली",
"ratelimit": "इंस्टेंस को सीमित कर दिया गया है",
"continue-search": "के साथ अपनी खोज जारी रखें "
"continue-search": "के साथ अपनी खोज जारी रखें Farside"
},
"lang_ja": {
"search": "検索",
@ -611,6 +611,6 @@
"dark": "ダーク",
"system": "自動",
"ratelimit": "インスタンスはレート制限されています",
"continue-search": "で検索を続ける "
"continue-search": "で検索を続ける Farside"
}
}

View File

@ -21,7 +21,7 @@
<hr>
<p>
{% if blocked is defined %}
<h4>{{ translation['continue-search'] }} <a class="link" href="https://github.com/benbusby/farside">Farside</a>!</h4>
<h4><a class="link" href="https://farside.link">{{ translation['continue-search'] }}</a></h4>
Whoogle:
<br>
<a class="link-color" href="{{farside}}/whoogle/search?q={{query}}{{params}}">

View File

@ -35,6 +35,7 @@ def gen_bangs_json(bangs_file: str) -> None:
}
json.dump(bangs_data, open(bangs_file, 'w'))
print('* Finished creating ddg bangs json')
def resolve_bang(query: str, bangs_dict: dict) -> str:

View File

@ -3,7 +3,7 @@ name: whoogle
description: A self hosted search engine on Kubernetes
type: application
version: 0.1.0
appVersion: 0.7.0
appVersion: 0.7.1
icon: https://github.com/benbusby/whoogle-search/raw/main/app/static/img/favicon/favicon-96x96.png

View File

@ -4,22 +4,22 @@ version: "2.4"
services:
whoogle-search:
image: benbusby/whoogle-search
image: ${WHOOGLE_IMAGE:-benbusby/whoogle-search}
container_name: whoogle-search
restart: unless-stopped
pids_limit: 50
mem_limit: 256mb
memswap_limit: 256mb
# user debian-tor from tor package
user: '102'
user: whoogle
security_opt:
- no-new-privileges
cap_drop:
- ALL
tmpfs:
- /config/:size=10M,uid=102,gid=102,mode=1700
- /var/lib/tor/:size=10M,uid=102,gid=102,mode=1700
- /run/tor/:size=1M,uid=102,gid=102,mode=1700
- /config/:size=10M,uid=927,gid=927,mode=1700
- /var/lib/tor/:size=10M,uid=927,gid=927,mode=1700
- /run/tor/:size=1M,uid=927,gid=927,mode=1700
#environment: # Uncomment to configure environment variables
# Basic auth configuration, uncomment to enable
#- WHOOGLE_USER=<auth username>

View File

@ -1,6 +1,5 @@
https://s.alefvanoon.xyz
https://search.albony.xyz
https://search.exonip.de
https://search.garudalinux.org
https://search.sethforprivacy.com
https://whoogle.fossho.st

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
if [ "$(whoami)" != "root" ]; then
tor -f /etc/tor/torrc

8
run
View File

@ -1,11 +1,11 @@
#!/bin/bash
#!/bin/sh
# Usage:
# ./run # Runs the full web app
# ./run test # Runs the testing suite
set -euo pipefail
set -eu
SCRIPT_DIR="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
SCRIPT_DIR="$(CDPATH= command cd -- "$(dirname -- "$0")" && pwd -P)"
# Set directory to serve static content from
SUBDIR="${1:-app}"
@ -17,7 +17,7 @@ rm -f "$SCRIPT_DIR"/app/static/build/*.js
rm -f "$SCRIPT_DIR"/app/static/build/*.css
# Check for regular vs test run
if [[ "$SUBDIR" == "test" ]]; then
if [ "$SUBDIR" = "test" ]; then
# Set up static files for testing
rm -rf "$STATIC_FOLDER"
ln -s "$SCRIPT_DIR/app/static" "$STATIC_FOLDER"

View File

@ -13,7 +13,7 @@ setuptools.setup(
author='Ben Busby',
author_email='contact@benbusby.com',
name='whoogle-search',
version='0.7.0' + optional_dev_tag,
version='0.7.1' + optional_dev_tag,
include_package_data=True,
install_requires=requirements,
description='Self-hosted, ad-free, privacy-respecting metasearch engine',