diff --git a/.github/workflows/buildx.yml b/.github/workflows/buildx.yml index 8d201ce..7cd2767 100644 --- a/.github/workflows/buildx.yml +++ b/.github/workflows/buildx.yml @@ -2,10 +2,13 @@ name: buildx on: workflow_run: - workflows: ["docker_tests"] + workflows: ["docker_main"] branches: [main] types: - completed + push: + tags: + - '*' jobs: on-success: @@ -21,10 +24,17 @@ jobs: uses: crazy-max/ghaction-docker-buildx@v1 with: version: latest - - name: log in to docker hub - run: | - echo "${{ secrets.DOCKER_PASSWORD }}" | \ - docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to ghcr.io + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: build and push the image if: startsWith(github.ref, 'refs/heads/main') && github.actor == 'benbusby' run: | @@ -33,6 +43,9 @@ jobs: docker buildx build --push \ --tag benbusby/whoogle-search:latest \ --platform linux/amd64,linux/arm/v7,linux/arm64 . + docker buildx build --push \ + --tag ghcr.io/benbusby/whoogle-search:latest \ + --platform linux/amd64,linux/arm/v7,linux/arm64 . - name: build and push tag if: startsWith(github.ref, 'refs/tags') run: | @@ -41,3 +54,6 @@ jobs: docker buildx build --push \ --tag benbusby/whoogle-search:${GITHUB_REF#refs/*/v}\ --platform linux/amd64,linux/arm/v7,linux/arm64 . + docker buildx build --push \ + --tag ghcr.io/benbusby/whoogle-search:${GITHUB_REF#refs/*/v}\ + --platform linux/amd64,linux/arm/v7,linux/arm64 . diff --git a/.github/workflows/docker_main.yml b/.github/workflows/docker_main.yml new file mode 100644 index 0000000..c41b693 --- /dev/null +++ b/.github/workflows/docker_main.yml @@ -0,0 +1,28 @@ +name: docker_main + +on: + workflow_run: + workflows: ["tests"] + branches: [main] + types: + - completed + +# TODO: Needs refactoring to use reusable workflows and share w/ docker_tests +jobs: + on-success: + runs-on: ubuntu-latest + steps: + - name: checkout code + uses: actions/checkout@v2 + - name: build and test (docker) + run: | + docker build --tag whoogle-search:test . + docker run --publish 5000:5000 --detach --name whoogle-search-nocompose whoogle-search:test + sleep 15 + docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1 + - name: build and test (docker-compose) + run: | + docker rm -f whoogle-search-nocompose + WHOOGLE_IMAGE="whoogle-search:test" docker-compose up --detach + sleep 15 + docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1 diff --git a/.github/workflows/docker_tests.yml b/.github/workflows/docker_tests.yml index e3a3a2f..52d1222 100644 --- a/.github/workflows/docker_tests.yml +++ b/.github/workflows/docker_tests.yml @@ -1,21 +1,26 @@ name: docker_tests on: - workflow_run: - workflows: ["tests"] - branches: [main] - types: - - completed + push: + branches: main + pull_request: + branches: main jobs: - on-success: + docker: runs-on: ubuntu-latest steps: - name: checkout code uses: actions/checkout@v2 - - name: build and test + - name: build and test (docker) run: | docker build --tag whoogle-search:test . - docker run --publish 5000:5000 --detach --name whoogle-search whoogle-search:test + docker run --publish 5000:5000 --detach --name whoogle-search-nocompose whoogle-search:test + sleep 15 + docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1 + - name: build and test (docker-compose) + run: | + docker rm -f whoogle-search-nocompose + WHOOGLE_IMAGE="whoogle-search:test" docker-compose up --detach sleep 15 docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1 diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml new file mode 100644 index 0000000..8790e89 --- /dev/null +++ b/.github/workflows/scan.yml @@ -0,0 +1,19 @@ +name: scan + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build the container image + run: | + docker build --tag whoogle-search:test . + - name: Initiate grype scan + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b . + chmod +x ./grype + ./grype whoogle-search:test --only-fixed diff --git a/Dockerfile b/Dockerfile index c9f2f20..b6f6483 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,11 @@ RUN pip install --prefix /install --no-warn-script-location --no-cache-dir -r re FROM python:3.8-alpine -RUN apk add --update --no-cache tor curl bash openrc +RUN apk add --update --no-cache tor curl openrc # libcurl4-openssl-dev +RUN apk -U upgrade + ARG DOCKER_USER=whoogle ARG DOCKER_USERID=927 ARG config_dir=/config @@ -69,19 +71,20 @@ COPY app/ app/ COPY run . #COPY whoogle.env . -# Allow writing symlinks to build dir -RUN chown 102:102 app/static/build - # Create user/group to run as -RUN adduser -D -g $DOCKER_USERID -u $DOCKER_USERID $DOCKER_USER +RUN adduser -D -g $DOCKER_USERID -u $DOCKER_USERID $DOCKER_USER + # Fix ownership / permissions RUN chown -R ${DOCKER_USER}:${DOCKER_USER} /whoogle /var/lib/tor +# Allow writing symlinks to build dir +RUN chown $DOCKER_USERID:$DOCKER_USERID app/static/build + USER $DOCKER_USER:$DOCKER_USER EXPOSE $EXPOSE_PORT -HEALTHCHECK --interval=30s --timeout=5s \ +HEALTHCHECK --interval=30s --timeout=5s \ CMD curl -f http://localhost:${EXPOSE_PORT}/healthz || exit 1 CMD misc/tor/start-tor.sh & ./run diff --git a/README.md b/README.md index 5acef07..87c0ff1 100644 --- a/README.md +++ b/README.md @@ -512,12 +512,11 @@ A lot of the app currently piggybacks on Google's existing support for fetching | Website | Country | Language | Cloudflare | |-|-|-|-| -| [https://whoogle.fossho.st](https://whoogle.fossho.st) | 🇺🇸 US | Multi-choice | | +| [https://whoogle.fossho.st](https://whoogle.fossho.st) (down until further notice) | 🇺🇸 US | Multi-choice | | | [https://search.albony.xyz](https://search.albony.xyz/) | 🇮🇳 IN | Multi-choice | | | [https://whoogle.sdf.org](https://whoogle.sdf.org) | 🇺🇸 US | Multi-choice | | [https://search.garudalinux.org](https://search.garudalinux.org) | 🇩🇪 DE | Multi-choice | | | [https://whooglesearch.net](https://whooglesearch.net) | 🇩🇪 DE | Spanish | | -| [https://search.exonip.de](https://search.exonip.de) | 🇳🇱 NL | Multi-choice | | | [https://s.alefvanoon.xyz](https://s.alefvanoon.xyz) | 🇺🇸 US | Multi-choice | ✅ | | [https://www.whooglesearch.ml](https://www.whooglesearch.ml) | 🇺🇸 US | English | | | [https://search.sethforprivacy.com](https://search.sethforprivacy.com) | 🇩🇪 DE | English | | diff --git a/app/__init__.py b/app/__init__.py index d0dc7cd..078e190 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -2,13 +2,14 @@ from app.filter import clean_query from app.request import send_tor_signal from app.utils.session import generate_user_key from app.utils.bangs import gen_bangs_json -from app.utils.misc import gen_file_hash +from app.utils.misc import gen_file_hash, read_config_bool from flask import Flask from flask_session import Session import json import logging.config import os from stem import Signal +import threading from dotenv import load_dotenv app = Flask(__name__, static_folder=os.path.dirname( @@ -29,7 +30,7 @@ if os.getenv('HTTPS_ONLY'): app.config['SESSION_COOKIE_NAME'] = '__Secure-session' app.config['SESSION_COOKIE_SECURE'] = True -app.config['VERSION_NUMBER'] = '0.7.0' +app.config['VERSION_NUMBER'] = '0.7.1' app.config['APP_ROOT'] = os.getenv( 'APP_ROOT', os.path.dirname(os.path.abspath(__file__))) @@ -58,7 +59,7 @@ app.config['CONFIG_PATH'] = os.getenv( app.config['DEFAULT_CONFIG'] = os.path.join( app.config['CONFIG_PATH'], 'config.json') -app.config['CONFIG_DISABLE'] = os.getenv('WHOOGLE_CONFIG_DISABLE', '') +app.config['CONFIG_DISABLE'] = read_config_bool('WHOOGLE_CONFIG_DISABLE') app.config['SESSION_FILE_DIR'] = os.path.join( app.config['CONFIG_PATH'], 'session') @@ -98,7 +99,11 @@ if not os.path.exists(app.config['SESSION_FILE_DIR']): if not os.path.exists(app.config['BANG_PATH']): os.makedirs(app.config['BANG_PATH']) if not os.path.exists(app.config['BANG_FILE']): - gen_bangs_json(app.config['BANG_FILE']) + json.dump({}, open(app.config['BANG_FILE'], 'w')) + bangs_thread = threading.Thread( + target=gen_bangs_json, + args=(app.config['BANG_FILE'],)) + bangs_thread.start() # Build new mapping of static files for cache busting if not os.path.exists(app.config['BUILD_FOLDER']): diff --git a/app/filter.py b/app/filter.py index 0df2386..fa7e604 100644 --- a/app/filter.py +++ b/app/filter.py @@ -12,6 +12,8 @@ import urllib.parse as urlparse from urllib.parse import parse_qs import os +minimal_mode_sections = ['Top stories', 'Images', 'People also ask'] + def extract_q(q_str: str, href: str) -> str: """Extracts the 'q' element from a result link. This is typically @@ -180,7 +182,11 @@ class Filter: for result in self.main_divs: result_children = pull_child_divs(result) if minimal_mode: - if len(result_children) in (1, 3): + if any(f">{x} 4: + try: + bang_json = json.load(open(app.config['BANG_FILE'])) + except json.decoder.JSONDecodeError: + # Ignore decoding error, can occur if file is still + # being written + pass + @app.after_request def after_request_func(resp): diff --git a/app/static/css/dark-theme.css b/app/static/css/dark-theme.css index 38df90a..4634fa0 100644 --- a/app/static/css/dark-theme.css +++ b/app/static/css/dark-theme.css @@ -59,19 +59,25 @@ select { .ZINbbc { overflow: hidden; - background-color: var(--whoogle-dark-result-bg) !important; + box-shadow: 0 0 0 0 !important; + background-color: var(--whoogle-dark-result-bg) !important; +} + +.KP7LCb { + box-shadow: 0 0 0 0 !important; } .BVG0Nb { - background-color: var(--whoogle-dark-result-bg) !important; + box-shadow: 0 0 0 0 !important; + background-color: var(--whoogle-dark-page-bg) !important; +} + +.ZINbbc.luh4tb { + background: var(--whoogle-dark-result-bg) !important; } .bRsWnc { - background-color: var(--whoogle-dark-result-bg) !important; -} - -.BVG0Nb { - background-color: var(--whoogle-dark-page-bg) !important; + background-color: var(--whoogle-dark-result-bg) !important; } .x54gtf { diff --git a/app/static/css/light-theme.css b/app/static/css/light-theme.css index 2562555..56810ed 100644 --- a/app/static/css/light-theme.css +++ b/app/static/css/light-theme.css @@ -34,15 +34,19 @@ select { .ZINbbc { overflow: hidden; - background-color: var(--whoogle-result-bg) !important; + background-color: var(--whoogle-result-bg) !important; } .BVG0Nb { - background-color: var(--whoogle-result-bg) !important; + background-color: var(--whoogle-result-bg) !important; +} + +.ZINbbc.luh4tb { + background: var(--whoogle-result-bg) !important; } .bRsWnc { - background-color: var(--whoogle-result-bg) !important; + background-color: var(--whoogle-result-bg) !important; } .x54gtf { diff --git a/app/static/css/search.css b/app/static/css/search.css index 1e76664..23484a2 100644 --- a/app/static/css/search.css +++ b/app/static/css/search.css @@ -1,3 +1,12 @@ +body { + display: block !important; + margin: auto !important; +} + +.vvjwJb { + font-size: 16px !important; +} + .autocomplete { position: relative; display: inline-block; @@ -43,3 +52,9 @@ details summary span { .ip-text-div { padding-top: 0 !important; } + +@media (min-width: 801px) { + body { + min-width: 736px !important; + } +} diff --git a/app/static/settings/translations.json b/app/static/settings/translations.json index 4797d0d..41e8415 100644 --- a/app/static/settings/translations.json +++ b/app/static/settings/translations.json @@ -35,7 +35,7 @@ "dark": "dark", "system": "system", "ratelimit": "Instance has been ratelimited", - "continue-search": "Continue your search with ", + "continue-search": "Continue your search with Farside", "all": "All", "images": "Images", "maps": "Maps", @@ -79,7 +79,7 @@ "dark": "donker", "system": "systeeminstellingen", "ratelimit": "Instantie is beperkt in snelheid", - "continue-search": "Ga verder met zoeken met " + "continue-search": "Ga verder met zoeken met Farside" }, "lang_de": { "search": "Suchen", @@ -117,7 +117,7 @@ "dark": "dunkel", "system": "Systemeinstellung", "ratelimit": "Instanz wurde ratenbegrenzt", - "continue-search": "Setzen Sie Ihre Suche fort mit " + "continue-search": "Setzen Sie Ihre Suche fort mit Farside" }, "lang_es": { "search": "Buscar", @@ -155,7 +155,7 @@ "dark": "oscuro", "system": "configuración del sistema", "ratelimit": "La instancia ha sido ratelimited", - "continue-search": "Continúe su búsqueda con " + "continue-search": "Continúe su búsqueda con Farside" }, "lang_it": { "search": "Cerca", @@ -193,7 +193,7 @@ "dark": "notte", "system": "impostazioni di sistema", "ratelimit": "L'istanza è stata limitata alla velocità", - "continue-search": "Continua la tua ricerca con " + "continue-search": "Continua la tua ricerca con Farside" }, "lang_pt": { "search": "Pesquisar", @@ -231,7 +231,7 @@ "dark": "escuro", "system": "configuração de sistema", "ratelimit": "A instância foi limitada pela taxa", - "continue-search": "Continue sua pesquisa com " + "continue-search": "Continue sua pesquisa com Farside" }, "lang_ru": { "search": "Поиск", @@ -269,7 +269,7 @@ "dark": "темное", "system": "системное", "ratelimit": "Число экземпляров ограничено", - "continue-search": "Продолжайте поиск с " + "continue-search": "Продолжайте поиск с Farside" }, "lang_zh-CN": { "search": "搜索", @@ -307,7 +307,7 @@ "dark": "黑暗的", "system": "系统设置", "ratelimit": "实例已被限速", - "continue-search": "继续搜索 " + "continue-search": "继续搜索 Farside" }, "lang_si": { "search": "සොයන්න", @@ -320,13 +320,13 @@ "config-block": "අවහිර", "config-block-help": "අල්ප විරාම වලින් වෙන් වූ අඩවි ලැයිස්තුව", "config-block-title": "මාතෘකාව අනුව අවහිර කරන්න", - "config-block-title-help": "රෙජෙක්ස් භාවිතා කරන්න", - "config-block-url": "යූආර්එල් මඟින් අවහිර කරන්න", + "config-block-title-help": "වාක්‍යවිධි භාවිතා කරන්න", + "config-block-url": "ඒ.ස.නි. මඟින් අවහිර කරන්න", "config-block-url-help": "රෙජෙක්ස් භාවිතා කරන්න", "config-theme": "තේමාව", "config-nojs": "නෝජේඑස් සබැඳි පෙන්වන්න", "config-dark": "අඳුරු ආකාරය", - "config-safe": "ආරක්ෂිත සෙවුම", + "config-safe": "ආරක්‍ෂිත සෙවුම", "config-alts": "සමාජ මාධ්‍ය සබැඳි ප්‍රතිස්ථාපනය කරන්න", "config-alts-help": "ට්විටර්/යූ ටියුබ්/ඉන්ස්ටග්‍රෑම් ආදී සබැඳි පෞද්ගලිකත්වයට ගරු කරන විකල්ප සමඟ ප්‍රතිස්ථාපනය කරයි.", "config-new-tab": "නව තීරුවකින් සබැඳි විවෘත කරන්න", @@ -344,8 +344,8 @@ "light": "දීප්තිමත්", "dark": "අඳුරු", "system": "පද්ධතිය", - "ratelimit": "උදාහරණය අනුපාත කර ඇත", - "continue-search": "සමඟ ඔබේ සෙවීම දිගටම කරගෙන යන්න" + "ratelimit": "සේවාදායකය අනුපාතනය කර ඇත", + "continue-search": "Farside සමඟ ඔබගේ සෙවුම කරගෙන යන්න" }, "lang_fr": { "search": "Chercher", @@ -383,7 +383,7 @@ "dark": "sombre", "system": "système", "ratelimit": "Le débit de l'instance a été limité", - "continue-search": "Continuez votre recherche avec " + "continue-search": "Continuez votre recherche avec Farside" }, "lang_fa": { "search": "جستجو", @@ -421,7 +421,7 @@ "dark": "تیره", "system": "سیستم", "ratelimit": "نمونه با نرخ محدود شده است", - "continue-search": "جستجوی خود را با " + "continue-search": "Farside جستجوی خود را با " }, "lang_cs": { "search": "Hledat", @@ -459,7 +459,7 @@ "dark": "Tmavý", "system": "Systémový", "ratelimit": "Instance byla omezena sazbou", - "continue-search": "Pokračujte ve vyhledávání pomocí " + "continue-search": "Pokračujte ve vyhledávání pomocí Farside" }, "lang_zh-TW": { "search": "搜尋", @@ -497,7 +497,7 @@ "dark": "黑暗的", "system": "依系統", "ratelimit": "實例已被限速", - "continue-search": "繼續搜索 " + "continue-search": "繼續搜索 Farside" }, "lang_bg": { "search": "Търсене", @@ -535,7 +535,7 @@ "dark": "тъмна", "system": "системна", "ratelimit": "Екземплярът е с ограничена скорост", - "continue-search": "Продължете търсенето си с " + "continue-search": "Продължете търсенето си с Farside" }, "lang_hi": { "search": "खोज", @@ -573,7 +573,7 @@ "dark": "अंधेरा", "system": "प्रणाली", "ratelimit": "इंस्टेंस को सीमित कर दिया गया है", - "continue-search": "के साथ अपनी खोज जारी रखें " + "continue-search": "के साथ अपनी खोज जारी रखें Farside" }, "lang_ja": { "search": "検索", @@ -611,6 +611,6 @@ "dark": "ダーク", "system": "自動", "ratelimit": "インスタンスはレート制限されています", - "continue-search": "で検索を続ける " + "continue-search": "で検索を続ける Farside" } } diff --git a/app/templates/error.html b/app/templates/error.html index 687327f..58f7f42 100644 --- a/app/templates/error.html +++ b/app/templates/error.html @@ -21,7 +21,7 @@

{% if blocked is defined %} -

{{ translation['continue-search'] }} Farside!

+

{{ translation['continue-search'] }}

Whoogle:
diff --git a/app/utils/bangs.py b/app/utils/bangs.py index 23b3c04..4a1a3cb 100644 --- a/app/utils/bangs.py +++ b/app/utils/bangs.py @@ -35,6 +35,7 @@ def gen_bangs_json(bangs_file: str) -> None: } json.dump(bangs_data, open(bangs_file, 'w')) + print('* Finished creating ddg bangs json') def resolve_bang(query: str, bangs_dict: dict) -> str: diff --git a/charts/whoogle/Chart.yaml b/charts/whoogle/Chart.yaml index 41d2a0e..f1561f4 100644 --- a/charts/whoogle/Chart.yaml +++ b/charts/whoogle/Chart.yaml @@ -3,7 +3,7 @@ name: whoogle description: A self hosted search engine on Kubernetes type: application version: 0.1.0 -appVersion: 0.7.0 +appVersion: 0.7.1 icon: https://github.com/benbusby/whoogle-search/raw/main/app/static/img/favicon/favicon-96x96.png diff --git a/docker-compose.yml b/docker-compose.yml index 84c0502..676a03c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,22 +4,22 @@ version: "2.4" services: whoogle-search: - image: benbusby/whoogle-search + image: ${WHOOGLE_IMAGE:-benbusby/whoogle-search} container_name: whoogle-search restart: unless-stopped pids_limit: 50 mem_limit: 256mb memswap_limit: 256mb # user debian-tor from tor package - user: '102' + user: whoogle security_opt: - no-new-privileges cap_drop: - ALL tmpfs: - - /config/:size=10M,uid=102,gid=102,mode=1700 - - /var/lib/tor/:size=10M,uid=102,gid=102,mode=1700 - - /run/tor/:size=1M,uid=102,gid=102,mode=1700 + - /config/:size=10M,uid=927,gid=927,mode=1700 + - /var/lib/tor/:size=10M,uid=927,gid=927,mode=1700 + - /run/tor/:size=1M,uid=927,gid=927,mode=1700 #environment: # Uncomment to configure environment variables # Basic auth configuration, uncomment to enable #- WHOOGLE_USER= diff --git a/misc/instances.txt b/misc/instances.txt index 4d6da01..23694bd 100644 --- a/misc/instances.txt +++ b/misc/instances.txt @@ -1,6 +1,5 @@ https://s.alefvanoon.xyz https://search.albony.xyz -https://search.exonip.de https://search.garudalinux.org https://search.sethforprivacy.com https://whoogle.fossho.st diff --git a/misc/tor/start-tor.sh b/misc/tor/start-tor.sh index e29241f..7e7e282 100755 --- a/misc/tor/start-tor.sh +++ b/misc/tor/start-tor.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh if [ "$(whoami)" != "root" ]; then tor -f /etc/tor/torrc diff --git a/run b/run index 9593201..2cc3cfe 100755 --- a/run +++ b/run @@ -1,11 +1,11 @@ -#!/bin/bash +#!/bin/sh # Usage: # ./run # Runs the full web app # ./run test # Runs the testing suite -set -euo pipefail +set -eu -SCRIPT_DIR="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_DIR="$(CDPATH= command cd -- "$(dirname -- "$0")" && pwd -P)" # Set directory to serve static content from SUBDIR="${1:-app}" @@ -17,7 +17,7 @@ rm -f "$SCRIPT_DIR"/app/static/build/*.js rm -f "$SCRIPT_DIR"/app/static/build/*.css # Check for regular vs test run -if [[ "$SUBDIR" == "test" ]]; then +if [ "$SUBDIR" = "test" ]; then # Set up static files for testing rm -rf "$STATIC_FOLDER" ln -s "$SCRIPT_DIR/app/static" "$STATIC_FOLDER" diff --git a/setup.py b/setup.py index 8779ca8..f8f22c0 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setuptools.setup( author='Ben Busby', author_email='contact@benbusby.com', name='whoogle-search', - version='0.7.0' + optional_dev_tag, + version='0.7.1' + optional_dev_tag, include_package_data=True, install_requires=requirements, description='Self-hosted, ad-free, privacy-respecting metasearch engine',