diff --git a/.gitignore b/.gitignore index 614e9936..14da8a03 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ vendor/ # calibre-web *.db *.log +cps/cache .idea/ *.bak diff --git a/cps.py b/cps.py index 277da288..aee7a7f5 100755 --- a/cps.py +++ b/cps.py @@ -44,6 +44,7 @@ from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.search_metadata import meta from cps.error_handler import init_errorhandler +from cps.schedule import register_scheduled_tasks, register_startup_tasks try: from cps.kobo import kobo, get_kobo_activated @@ -79,6 +80,11 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + # Register scheduled tasks + register_scheduled_tasks() + register_startup_tasks() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index 2dfad699..96593a46 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -159,6 +159,7 @@ def create_app(): config.store_calibre_uuid(calibre_db, db.Library_Id) return app + @babel.localeselector def get_locale(): # if a user is logged in, use the locale from the user settings diff --git a/cps/admin.py b/cps/admin.py index 4876d421..5d297d2b 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -40,11 +40,13 @@ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text from . import constants, logger, helper, services, cli -from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, kobo_sync_status +from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, \ + kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ valid_email, check_username from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config +from .services.worker import WorkerThread from . import debug_info, _BABEL_TRANSLATIONS from functools import wraps @@ -1635,6 +1637,45 @@ def update_mailsettings(): return edit_mailsettings() +@admi.route("/admin/scheduledtasks") +@login_required +@admin_required +def edit_scheduledtasks(): + content = config.get_scheduled_task_settings() + return render_title_template("schedule_edit.html", config=content, title=_(u"Edit Scheduled Tasks Settings")) + + +@admi.route("/admin/scheduledtasks", methods=["POST"]) +@login_required +@admin_required +def update_scheduledtasks(): + to_save = request.form.to_dict() + _config_int(to_save, "schedule_start_time") + _config_int(to_save, "schedule_end_time") + _config_checkbox(to_save, "schedule_generate_book_covers") + _config_checkbox(to_save, "schedule_generate_series_covers") + + try: + config.save() + flash(_(u"Scheduled tasks settings updated"), category="success") + + # Cancel any running tasks + schedule.end_scheduled_tasks() + + # Re-register tasks with new settings + schedule.register_scheduled_tasks() + except IntegrityError as ex: + ub.session.rollback() + log.error("An unknown error occurred while saving scheduled tasks settings") + flash(_(u"An unknown error occurred. Please try again later."), category="error") + except OperationalError: + ub.session.rollback() + log.error("Settings DB is not Writeable") + flash(_("Settings DB is not Writeable"), category="error") + + return edit_scheduledtasks() + + @admi.route("/admin/user/", methods=["GET", "POST"]) @login_required @admin_required @@ -1911,3 +1952,13 @@ def extract_dynamic_field_from_filter(user, filtr): def extract_user_identifier(user, filtr): dynamic_field = extract_dynamic_field_from_filter(user, filtr) return extract_user_data_from_field(user, dynamic_field) + + +@admi.route("/ajax/canceltask", methods=['POST']) +@login_required +@admin_required +def cancel_task(): + task_id = request.get_json().get('task_id', None) + worker = WorkerThread.get_instance() + worker.end_task(task_id) + return "" diff --git a/cps/config_sql.py b/cps/config_sql.py index 01523a01..a8beaabb 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -134,13 +134,18 @@ class _Settings(_Base): config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) - config_unicode_filename =Column(Boolean, default=False) + config_unicode_filename = Column(Boolean, default=False) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) config_reverse_proxy_login_header_name = Column(String) config_allow_reverse_proxy_header_login = Column(Boolean, default=False) + schedule_start_time = Column(Integer, default=4) + schedule_end_time = Column(Integer, default=6) + schedule_generate_book_covers = Column(Boolean, default=False) + schedule_generate_series_covers = Column(Boolean, default=False) + def __repr__(self): return self.__class__.__name__ @@ -171,7 +176,6 @@ class _ConfigSQL(object): if change: self.save() - def _read_from_storage(self): if self._settings is None: log.debug("_ConfigSQL._read_from_storage") @@ -255,6 +259,8 @@ class _ConfigSQL(object): return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0) or (self.mail_gmail_token != {} and self.mail_server_type == 1)) + def get_scheduled_task_settings(self): + return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')} def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): """Possibly updates a field of this object. @@ -290,7 +296,6 @@ class _ConfigSQL(object): storage[k] = v return storage - def load(self): '''Load all configuration values from the underlying storage.''' s = self._read_from_storage() # type: _Settings @@ -411,6 +416,7 @@ def autodetect_calibre_binary(): return element return "" + def autodetect_unrar_binary(): if sys.platform == "win32": calibre_path = ["C:\\program files\\WinRar\\unRAR.exe", @@ -422,6 +428,7 @@ def autodetect_unrar_binary(): return element return "" + def autodetect_kepubify_binary(): if sys.platform == "win32": calibre_path = ["C:\\program files\\kepubify\\kepubify-windows-64Bit.exe", @@ -433,6 +440,7 @@ def autodetect_kepubify_binary(): return element return "" + def _migrate_database(session): # make sure the table is created, if it does not exist _Base.metadata.create_all(session.bind) diff --git a/cps/constants.py b/cps/constants.py index f9003125..04e2404a 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -23,6 +23,9 @@ from sqlalchemy import __version__ as sql_version sqlalchemy_version2 = ([int(x) for x in sql_version.split('.')] >= [2, 0, 0]) +# APP_MODE - production, development, or test +APP_MODE = os.environ.get('APP_MODE', 'production') + # if installed via pip this variable is set to true (empty file with name .HOMEDIR present) HOME_CONFIG = os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), '.HOMEDIR')) @@ -35,6 +38,10 @@ STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static') TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates') TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') +# Cache dir - use CACHE_DIR environment variable, otherwise use the default directory: cps/cache +DEFAULT_CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') +CACHE_DIR = os.environ.get('CACHE_DIR', DEFAULT_CACHE_DIR) + if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"), ".calibre-web") if not os.path.exists(home_dir): @@ -162,6 +169,19 @@ NIGHTLY_VERSION[1] = '$Format:%cI$' # NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57' # NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00' +# CACHE +CACHE_TYPE_THUMBNAILS = 'thumbnails' + +# Thumbnail Types +THUMBNAIL_TYPE_COVER = 1 +THUMBNAIL_TYPE_SERIES = 2 +THUMBNAIL_TYPE_AUTHOR = 3 + +# Thumbnails Sizes +COVER_THUMBNAIL_ORIGINAL = 0 +COVER_THUMBNAIL_SMALL = 1 +COVER_THUMBNAIL_MEDIUM = 2 +COVER_THUMBNAIL_LARGE = 3 # clean-up the module namespace del sys, os, namedtuple diff --git a/cps/db.py b/cps/db.py index a3b93d89..3e86f7a5 100644 --- a/cps/db.py +++ b/cps/db.py @@ -450,11 +450,11 @@ class CalibreDB(): """ self.session = None if self._init: - self.initSession(expire_on_commit) + self.init_session(expire_on_commit) self.instances.add(self) - def initSession(self, expire_on_commit=True): + def init_session(self, expire_on_commit=True): self.session = self.session_factory() self.session.expire_on_commit = expire_on_commit self.update_title_sort(self.config) @@ -603,7 +603,7 @@ class CalibreDB(): autoflush=True, bind=cls.engine)) for inst in cls.instances: - inst.initSession() + inst.init_session() cls._init = True return True @@ -720,7 +720,8 @@ class CalibreDB(): randm = self.session.query(Books) \ .filter(self.common_filters(allow_show_archived)) \ .order_by(func.random()) \ - .limit(self.config.config_random_books).all() + .limit(self.config.config_random_books) \ + .all() else: randm = false() if join_archive_read: diff --git a/cps/editbooks.py b/cps/editbooks.py index fcf043a5..1258f641 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -686,6 +686,7 @@ def upload_cover(request, book): abort(403) ret, message = helper.save_cover(requested_file, book.path) if ret is True: + helper.clear_cover_thumbnail_cache(book.id) return True else: flash(message, category="error") @@ -809,6 +810,7 @@ def edit_book(book_id): if result is True: book.has_cover = 1 modif_date = True + helper.clear_cover_thumbnail_cache(book.id) else: flash(error, category="error") diff --git a/cps/fs.py b/cps/fs.py new file mode 100644 index 00000000..0171a5d5 --- /dev/null +++ b/cps/fs.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +from . import logger +from .constants import CACHE_DIR +from os import makedirs, remove +from os.path import isdir, isfile, join +from shutil import rmtree + + +class FileSystem: + _instance = None + _cache_dir = CACHE_DIR + + def __new__(cls): + if cls._instance is None: + cls._instance = super(FileSystem, cls).__new__(cls) + cls.log = logger.create() + return cls._instance + + def get_cache_dir(self, cache_type=None): + if not isdir(self._cache_dir): + try: + makedirs(self._cache_dir) + except OSError: + self.log.info(f'Failed to create path {self._cache_dir} (Permission denied).') + return False + + path = join(self._cache_dir, cache_type) + if cache_type and not isdir(path): + try: + makedirs(path) + except OSError: + self.log.info(f'Failed to create path {path} (Permission denied).') + return False + + return path if cache_type else self._cache_dir + + def get_cache_file_dir(self, filename, cache_type=None): + path = join(self.get_cache_dir(cache_type), filename[:2]) + if not isdir(path): + try: + makedirs(path) + except OSError: + self.log.info(f'Failed to create path {path} (Permission denied).') + return False + + return path + + def get_cache_file_path(self, filename, cache_type=None): + return join(self.get_cache_file_dir(filename, cache_type), filename) if filename else None + + def get_cache_file_exists(self, filename, cache_type=None): + path = self.get_cache_file_path(filename, cache_type) + return isfile(path) + + def delete_cache_dir(self, cache_type=None): + if not cache_type and isdir(self._cache_dir): + try: + rmtree(self._cache_dir) + except OSError: + self.log.info(f'Failed to delete path {self._cache_dir} (Permission denied).') + return False + + path = join(self._cache_dir, cache_type) + if cache_type and isdir(path): + try: + rmtree(path) + except OSError: + self.log.info(f'Failed to delete path {path} (Permission denied).') + return False + + def delete_cache_file(self, filename, cache_type=None): + path = self.get_cache_file_path(filename, cache_type) + if isfile(path): + try: + remove(path) + except OSError: + self.log.info(f'Failed to delete path {path} (Permission denied).') + return False diff --git a/cps/helper.py b/cps/helper.py index b5495930..9a3c0580 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -34,7 +34,7 @@ from babel.units import format_unit from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ from flask_login import current_user -from sqlalchemy.sql.expression import true, false, and_, text, func +from sqlalchemy.sql.expression import true, false, and_, or_, text, func from sqlalchemy.exc import InvalidRequestError, OperationalError from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash @@ -49,12 +49,14 @@ except ImportError: from . import calibre_db, cli from .tasks.convert import TaskConvert -from . import logger, config, get_locale, db, ub, kobo_sync_status +from . import logger, config, get_locale, db, ub, kobo_sync_status, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES from .subproc_wrapper import process_wait -from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS +from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS, STAT_ENDED, \ + STAT_CANCELLED from .tasks.mail import TaskEmail +from .tasks.thumbnail import TaskClearCoverThumbnailCache log = logger.create() @@ -497,6 +499,7 @@ def upload_new_file_gdrive(book_id, first_author, renamed_author, title, title_d return error + def update_dir_structure_gdrive(book_id, first_author, renamed_author): error = False book = calibre_db.get_book(book_id) @@ -633,6 +636,7 @@ def uniq(inpt): output.append(x) return output + def check_email(email): email = valid_email(email) if ub.session.query(ub.User).filter(func.lower(ub.User.email) == email.lower()).first(): @@ -679,6 +683,7 @@ def update_dir_structure(book_id, def delete_book(book, calibrepath, book_format): + clear_cover_thumbnail_cache(book.id) if config.config_use_google_drive: return delete_book_gdrive(book, book_format) else: @@ -692,19 +697,29 @@ def get_cover_on_failure(use_generic_cover): return None -def get_book_cover(book_id): +def get_book_cover(book_id, resolution=None): book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - return get_book_cover_internal(book, use_generic_cover_on_failure=True) + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) -def get_book_cover_with_uuid(book_uuid, - use_generic_cover_on_failure=True): +def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): book = calibre_db.get_book_by_uuid(book_uuid) return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, use_generic_cover_on_failure): +def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: + + # Send the book cover thumbnail if it exists in cache + if resolution: + thumbnail = get_book_cover_thumbnail(book, resolution) + if thumbnail: + cache = fs.FileSystem() + if cache.get_cache_file_exists(thumbnail.filename, CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_file_dir(thumbnail.filename, CACHE_TYPE_THUMBNAILS), + thumbnail.filename) + + # Send the book cover from Google Drive if configured if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -718,6 +733,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): except Exception as ex: log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) + + # Send the book cover from the Calibre directory else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): @@ -728,6 +745,56 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): return get_cover_on_failure(use_generic_cover_on_failure) +def get_book_cover_thumbnail(book, resolution): + if book and book.has_cover: + return ub.session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == THUMBNAIL_TYPE_COVER) \ + .filter(ub.Thumbnail.entity_id == book.id) \ + .filter(ub.Thumbnail.resolution == resolution) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ + .first() + + +def get_series_thumbnail_on_failure(series_id, resolution): + book = calibre_db.session \ + .query(db.Books) \ + .join(db.books_series_link) \ + .join(db.Series) \ + .filter(db.Series.id == series_id) \ + .filter(db.Books.has_cover == 1) \ + .first() + + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) + + +def get_series_cover_thumbnail(series_id, resolution=None): + return get_series_cover_internal(series_id, resolution) + + +def get_series_cover_internal(series_id, resolution=None): + # Send the series thumbnail if it exists in cache + if resolution: + thumbnail = get_series_thumbnail(series_id, resolution) + if thumbnail: + cache = fs.FileSystem() + if cache.get_cache_file_exists(thumbnail.filename, CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_file_dir(thumbnail.filename, CACHE_TYPE_THUMBNAILS), + thumbnail.filename) + + return get_series_thumbnail_on_failure(series_id, resolution) + + +def get_series_thumbnail(series_id, resolution): + return ub.session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == THUMBNAIL_TYPE_SERIES) \ + .filter(ub.Thumbnail.entity_id == series_id) \ + .filter(ub.Thumbnail.resolution == resolution) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ + .first() + + # saves book cover from url def save_cover_from_url(url, book_path): try: @@ -920,12 +987,22 @@ def render_task_status(tasklist): ret['status'] = _(u'Started') elif task.stat == STAT_FINISH_SUCCESS: ret['status'] = _(u'Finished') + elif task.stat == STAT_ENDED: + ret['status'] = _(u'Ended') + elif task.stat == STAT_CANCELLED: + ret['status'] = _(u'Cancelled') else: ret['status'] = _(u'Unknown Status') - ret['taskMessage'] = "{}: {}".format(_(task.name), task.message) + ret['taskMessage'] = "{}: {}".format(_(task.name), task.message) if task.message else _(task.name) ret['progress'] = "{} %".format(int(task.progress * 100)) ret['user'] = escape(user) # prevent xss + + # Hidden fields + ret['id'] = task.id + ret['stat'] = task.stat + ret['is_cancellable'] = task.is_cancellable + renderedtasklist.append(ret) return renderedtasklist @@ -994,3 +1071,7 @@ def get_download_link(book_id, book_format, client): return do_download_file(book, book_format, client, data1, headers) else: abort(404) + + +def clear_cover_thumbnail_cache(book_id): + WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id)) diff --git a/cps/jinjia.py b/cps/jinjia.py index 06e99141..f88b36a2 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -31,8 +31,7 @@ from flask import Blueprint, request, url_for from flask_babel import get_locale from flask_login import current_user from markupsafe import escape -from . import logger - +from . import constants, logger jinjia = Blueprint('jinjia', __name__) log = logger.create() @@ -128,12 +127,55 @@ def formatseriesindex_filter(series_index): return series_index return 0 + @jinjia.app_template_filter('escapedlink') def escapedlink_filter(url, text): return "{}".format(url, escape(text)) + @jinjia.app_template_filter('uuidfilter') def uuidfilter(var): return uuid4() +@jinjia.app_template_filter('cache_timestamp') +def cache_timestamp(rolling_period='month'): + if rolling_period == 'day': + return str(int(datetime.datetime.today().replace(hour=1, minute=1).timestamp())) + elif rolling_period == 'year': + return str(int(datetime.datetime.today().replace(day=1).timestamp())) + else: + return str(int(datetime.datetime.today().replace(month=1, day=1).timestamp())) + + +@jinjia.app_template_filter('last_modified') +def book_last_modified(book): + return str(int(book.last_modified.timestamp())) + + +@jinjia.app_template_filter('get_cover_srcset') +def get_cover_srcset(book): + srcset = list() + resolutions = { + constants.COVER_THUMBNAIL_SMALL: 'sm', + constants.COVER_THUMBNAIL_MEDIUM: 'md', + constants.COVER_THUMBNAIL_LARGE: 'lg' + } + for resolution, shortname in resolutions.items(): + url = url_for('web.get_cover', book_id=book.id, resolution=shortname, c=book_last_modified(book)) + srcset.append(f'{url} {resolution}x') + return ', '.join(srcset) + + +@jinjia.app_template_filter('get_series_srcset') +def get_cover_srcset(series): + srcset = list() + resolutions = { + constants.COVER_THUMBNAIL_SMALL: 'sm', + constants.COVER_THUMBNAIL_MEDIUM: 'md', + constants.COVER_THUMBNAIL_LARGE: 'lg' + } + for resolution, shortname in resolutions.items(): + url = url_for('web.get_series_cover', series_id=series.id, resolution=shortname, c=cache_timestamp()) + srcset.append(f'{url} {resolution}x') + return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py new file mode 100644 index 00000000..5b97cecb --- /dev/null +++ b/cps/schedule.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import datetime + +from . import config, constants +from .services.background_scheduler import BackgroundScheduler +from .tasks.database import TaskReconnectDatabase +from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails +from .services.worker import WorkerThread + + +def get_scheduled_tasks(reconnect=True): + tasks = list() + + # Reconnect Calibre database (metadata.db) + if reconnect: + tasks.append(lambda: TaskReconnectDatabase()) + + # Generate all missing book cover thumbnails + if config.schedule_generate_book_covers: + tasks.append(lambda: TaskGenerateCoverThumbnails()) + + # Generate all missing series thumbnails + if config.schedule_generate_series_covers: + tasks.append(lambda: TaskGenerateSeriesThumbnails()) + + return tasks + + +def end_scheduled_tasks(): + worker = WorkerThread.get_instance() + for __, __, __, task in worker.tasks: + if task.scheduled and task.is_cancellable: + worker.end_task(task.id) + + +def register_scheduled_tasks(): + scheduler = BackgroundScheduler() + + if scheduler: + # Remove all existing jobs + scheduler.remove_all_jobs() + + start = config.schedule_start_time + end = config.schedule_end_time + + # Register scheduled tasks + if start != end: + scheduler.schedule_tasks(tasks=get_scheduled_tasks(), trigger='cron', hour=start) + scheduler.schedule(func=end_scheduled_tasks, trigger='cron', hour=end) + + # Kick-off tasks, if they should currently be running + if should_task_be_running(start, end): + scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) + + +def register_startup_tasks(): + scheduler = BackgroundScheduler() + + if scheduler: + start = config.schedule_start_time + end = config.schedule_end_time + + # Run scheduled tasks immediately for development and testing + # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks + if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, end): + scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) + + +def should_task_be_running(start, end): + now = datetime.datetime.now().hour + return (start < end and start <= now < end) or (end < start <= now or now < end) diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py new file mode 100644 index 00000000..415b2962 --- /dev/null +++ b/cps/services/background_scheduler.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import atexit + +from .. import logger +from .worker import WorkerThread + +try: + from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + use_APScheduler = True +except (ImportError, RuntimeError) as e: + use_APScheduler = False + log = logger.create() + log.info('APScheduler not found. Unable to schedule tasks.') + + +class BackgroundScheduler: + _instance = None + + def __new__(cls): + if not use_APScheduler: + return False + + if cls._instance is None: + cls._instance = super(BackgroundScheduler, cls).__new__(cls) + cls.log = logger.create() + cls.scheduler = BScheduler() + cls.scheduler.start() + + atexit.register(lambda: cls.scheduler.shutdown()) + + return cls._instance + + def schedule(self, func, trigger, **trigger_args): + if use_APScheduler: + return self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + + # Expects a lambda expression for the task + def schedule_task(self, task, user=None, trigger='cron', **trigger_args): + if use_APScheduler: + def scheduled_task(): + worker_task = task() + worker_task.scheduled = True + WorkerThread.add(user, worker_task) + return self.schedule(func=scheduled_task, trigger=trigger, **trigger_args) + + # Expects a list of lambda expressions for the tasks + def schedule_tasks(self, tasks, user=None, trigger='cron', **trigger_args): + if use_APScheduler: + for task in tasks: + self.schedule_task(task, user=user, trigger=trigger, **trigger_args) + + # Expects a lambda expression for the task + def schedule_task_immediately(self, task, user=None): + if use_APScheduler: + def immediate_task(): + WorkerThread.add(user, task()) + return self.schedule(func=immediate_task, trigger='date') + + # Expects a list of lambda expressions for the tasks + def schedule_tasks_immediately(self, tasks, user=None): + if use_APScheduler: + for task in tasks: + self.schedule_task_immediately(task, user) + + # Remove all jobs + def remove_all_jobs(self): + self.scheduler.remove_all_jobs() diff --git a/cps/services/worker.py b/cps/services/worker.py index 076c9104..7e0f8a5c 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -37,6 +37,8 @@ STAT_WAITING = 0 STAT_FAIL = 1 STAT_STARTED = 2 STAT_FINISH_SUCCESS = 3 +STAT_ENDED = 4 +STAT_CANCELLED = 5 # Only retain this many tasks in dequeued list TASK_CLEANUP_TRIGGER = 20 @@ -51,7 +53,6 @@ def _get_main_thread(): raise Exception("main thread not found?!") - class ImprovedQueue(queue.Queue): def to_list(self): """ @@ -61,12 +62,13 @@ class ImprovedQueue(queue.Queue): with self.mutex: return list(self.queue) + # Class for all worker tasks in the background class WorkerThread(threading.Thread): _instance = None @classmethod - def getInstance(cls): + def get_instance(cls): if cls._instance is None: cls._instance = WorkerThread() return cls._instance @@ -83,12 +85,13 @@ class WorkerThread(threading.Thread): @classmethod def add(cls, user, task): - ins = cls.getInstance() + ins = cls.get_instance() ins.num += 1 - log.debug("Add Task for user: {} - {}".format(user, task)) + username = user if user is not None else 'System' + log.debug("Add Task for user: {} - {}".format(username, task)) ins.queue.put(QueuedTask( num=ins.num, - user=user, + user=username, added=datetime.now(), task=task, )) @@ -144,8 +147,18 @@ class WorkerThread(threading.Thread): # CalibreTask.start() should wrap all exceptions in it's own error handling item.task.start(self) + # remove self_cleanup tasks from list + if item.task.self_cleanup: + self.dequeued.remove(item) + self.queue.task_done() + def end_task(self, task_id): + ins = self.get_instance() + for __, __, __, task in ins.tasks: + if str(task.id) == str(task_id) and task.is_cancellable: + task.stat = STAT_CANCELLED if task.stat == STAT_WAITING else STAT_ENDED + class CalibreTask: __metaclass__ = abc.ABCMeta @@ -158,10 +171,12 @@ class CalibreTask: self.end_time = None self.message = message self.id = uuid.uuid4() + self.self_cleanup = False + self._scheduled = False @abc.abstractmethod def run(self, worker_thread): - """Provides the caller some human-readable name for this class""" + """The main entry-point for this task""" raise NotImplementedError @abc.abstractmethod @@ -169,6 +184,11 @@ class CalibreTask: """Provides the caller some human-readable name for this class""" raise NotImplementedError + @abc.abstractmethod + def is_cancellable(self): + """Does this task gracefully handle being cancelled (STAT_ENDED, STAT_CANCELLED)?""" + raise NotImplementedError + def start(self, *args): self.start_time = datetime.now() self.stat = STAT_STARTED @@ -219,7 +239,7 @@ class CalibreTask: We have a separate dictating this because there may be certain tasks that want to override this """ # By default, we're good to clean a task if it's "Done" - return self.stat in (STAT_FINISH_SUCCESS, STAT_FAIL) + return self.stat in (STAT_FINISH_SUCCESS, STAT_FAIL, STAT_ENDED, STAT_CANCELLED) '''@progress.setter def progress(self, x): @@ -229,6 +249,22 @@ class CalibreTask: x = 0 self._progress = x''' + @property + def self_cleanup(self): + return self._self_cleanup + + @self_cleanup.setter + def self_cleanup(self, is_self_cleanup): + self._self_cleanup = is_self_cleanup + + @property + def scheduled(self): + return self._scheduled + + @scheduled.setter + def scheduled(self, is_scheduled): + self._scheduled = is_scheduled + def _handleError(self, error_message): self.stat = STAT_FAIL self.progress = 1 diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index 3a980180..b2b35423 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -5150,7 +5150,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head pointer-events: none } -#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { +#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, #cancelTaskModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { cursor: pointer } @@ -5237,7 +5237,11 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d margin-bottom: 20px } -body.admin:not(.modal-open) .btn-default { +body.admin > div.container-fluid div.scheduled_tasks_details { + margin-bottom: 20px +} + +body.admin .btn-default { margin-bottom: 10px } @@ -5468,7 +5472,7 @@ body.admin.modal-open .navbar { z-index: 0 !important } -#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal { +#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal, #cancelTaskModal { top: 0; overflow: hidden; padding-top: 70px; @@ -5478,7 +5482,7 @@ body.admin.modal-open .navbar { background: rgba(0, 0, 0, .5) } -#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before { +#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before, #cancelTaskModal:before { content: "\E208"; padding-right: 10px; display: block; @@ -5500,18 +5504,18 @@ body.admin.modal-open .navbar { z-index: 99 } -#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { +#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before, #cancelTaskModal.in:before { -webkit-transform: translate(0, 0); -ms-transform: translate(0, 0); transform: translate(0, 0) } -#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { +#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog, #cancelTaskModal > .modal-dialog { width: 450px; margin: auto } -#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { +#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content, #cancelTaskModal > .modal-dialog > .modal-content { max-height: calc(100% - 90px); -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); box-shadow: 0 5px 15px rgba(0, 0, 0, .5); @@ -5522,7 +5526,7 @@ body.admin.modal-open .navbar { width: 450px } -#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { +#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header, #cancelTaskModal > .modal-dialog > .modal-content > .modal-header { padding: 15px 20px; border-radius: 3px 3px 0 0; line-height: 1.71428571; @@ -5535,7 +5539,7 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { +#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before, #cancelTaskModal > .modal-dialog > .modal-content > .modal-header:before { padding-right: 10px; font-size: 18px; color: #999; @@ -5564,6 +5568,11 @@ body.admin.modal-open .navbar { font-family: plex-icons-new, serif } +#cancelTaskModal > .modal-dialog > .modal-content > .modal-header:before { + content: "\EA6D"; + font-family: plex-icons-new, serif +} + #RestartDialog > .modal-dialog > .modal-content > .modal-header:after { content: "Restart Calibre-Web"; display: inline-block; @@ -5588,7 +5597,13 @@ body.admin.modal-open .navbar { font-size: 20px } -#StatusDialog > .modal-dialog > .modal-content > .modal-header > span, #deleteModal > .modal-dialog > .modal-content > .modal-header > span, #loader > center > img, .rating-mobile { +#cancelTaskModal > .modal-dialog > .modal-content > .modal-header:after { + content: "Delete Book"; + display: inline-block; + font-size: 20px +} + +#StatusDialog > .modal-dialog > .modal-content > .modal-header > span, #deleteModal > .modal-dialog > .modal-content > .modal-header > span, #cancelTaskModal > .modal-dialog > .modal-content > .modal-header > span, #loader > center > img, .rating-mobile { display: none } @@ -5602,7 +5617,7 @@ body.admin.modal-open .navbar { text-align: left } -#ShutdownDialog > .modal-dialog > .modal-content > .modal-body, #StatusDialog > .modal-dialog > .modal-content > .modal-body, #deleteModal > .modal-dialog > .modal-content > .modal-body { +#ShutdownDialog > .modal-dialog > .modal-content > .modal-body, #StatusDialog > .modal-dialog > .modal-content > .modal-body, #deleteModal > .modal-dialog > .modal-content > .modal-body, #cancelTaskModal > .modal-dialog > .modal-content > .modal-body { padding: 20px 20px 40px; font-size: 16px; line-height: 1.6em; @@ -5612,7 +5627,7 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p, #cancelTaskModal > .modal-dialog > .modal-content > .modal-body > p { padding: 20px 20px 0 0; font-size: 16px; line-height: 1.6em; @@ -5621,7 +5636,7 @@ body.admin.modal-open .navbar { background: #282828 } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default, #cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { float: right; z-index: 9; position: relative; @@ -5669,6 +5684,18 @@ body.admin.modal-open .navbar { border-radius: 3px } +#cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { + float: right; + z-index: 9; + position: relative; + margin: 0 0 0 10px; + min-width: 80px; + padding: 10px 18px; + font-size: 16px; + line-height: 1.33; + border-radius: 3px +} + #RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart) { margin: 25px 0 0 10px } @@ -5681,7 +5708,11 @@ body.admin.modal-open .navbar { margin: 0 0 0 10px } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { +#cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { + margin: 0 0 0 10px +} + +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover, #cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { background-color: hsla(0, 0%, 100%, .3) } @@ -7303,11 +7334,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. background-color: transparent !important } - #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { + #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog, #cancelTaskModal > .modal-dialog { max-width: calc(100vw - 40px) } - #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { + #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content, #cancelTaskModal > .modal-dialog > .modal-content { max-width: calc(100vw - 40px); left: 0 } @@ -7457,7 +7488,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. padding: 30px 15px } - #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { + #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before, #cancelTaskModal.in:before { left: auto; right: 34px } diff --git a/cps/static/js/table.js b/cps/static/js/table.js index dce1d06c..4cafe05d 100644 --- a/cps/static/js/table.js +++ b/cps/static/js/table.js @@ -15,7 +15,7 @@ * along with this program. If not, see . */ -/* exported TableActions, RestrictionActions, EbookActions, responseHandler */ +/* exported TableActions, RestrictionActions, EbookActions, TaskActions, responseHandler */ /* global getPath, confirmDialog */ var selections = []; @@ -42,6 +42,24 @@ $(function() { }, 1000); } + $("#cancel_task_confirm").click(function() { + //get data-id attribute of the clicked element + var taskId = $(this).data("task-id"); + $.ajax({ + method: "post", + contentType: "application/json; charset=utf-8", + dataType: "json", + url: window.location.pathname + "/../ajax/canceltask", + data: JSON.stringify({"task_id": taskId}), + }); + }); + //triggered when modal is about to be shown + $("#cancelTaskModal").on("show.bs.modal", function(e) { + //get data-id attribute of the clicked element and store in button + var taskId = $(e.relatedTarget).data("task-id"); + $(e.currentTarget).find("#cancel_task_confirm").data("task-id", taskId); + }); + $("#books-table").on("check.bs.table check-all.bs.table uncheck.bs.table uncheck-all.bs.table", function (e, rowsAfter, rowsBefore) { var rows = rowsAfter; @@ -581,6 +599,7 @@ function handle_header_buttons () { $(".header_select").removeAttr("disabled"); } } + /* Function for deleting domain restrictions */ function TableActions (value, row) { return [ @@ -618,6 +637,19 @@ function UserActions (value, row) { ].join(""); } +/* Function for cancelling tasks */ +function TaskActions (value, row) { + var cancellableStats = [0, 1, 2]; + if (row.id && row.is_cancellable && cancellableStats.includes(row.stat)) { + return [ + "
", + "", + "
" + ].join(""); + } + return ''; +} + /* Function for keeping checked rows */ function responseHandler(res) { $.each(res.rows, function (i, row) { diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index e610b14b..28c64c58 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -273,3 +273,7 @@ class TaskConvert(CalibreTask): def __str__(self): return "Convert {} {}".format(self.bookid, self.kindle_mail) + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/database.py b/cps/tasks/database.py new file mode 100644 index 00000000..0441d564 --- /dev/null +++ b/cps/tasks/database.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from cps import config, logger +from cps.services.worker import CalibreTask + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + + +class TaskReconnectDatabase(CalibreTask): + def __init__(self, task_message=u'Reconnecting Calibre database'): + super(TaskReconnectDatabase, self).__init__(task_message) + self.log = logger.create() + self.listen_address = config.get_config_ipaddress() + self.listen_port = config.config_port + + def run(self, worker_thread): + address = self.listen_address if self.listen_address else 'localhost' + port = self.listen_port if self.listen_port else 8083 + + try: + urlopen('http://' + address + ':' + str(port) + '/reconnect') + self._handleSuccess() + except Exception as ex: + self._handleError(u'Unable to reconnect Calibre database: ' + str(ex)) + + @property + def name(self): + return "Reconnect Database" + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py index 03526c8b..4cfc742a 100644 --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -266,5 +266,9 @@ class TaskEmail(CalibreTask): def name(self): return "E-mail" + @property + def is_cancellable(self): + return False + def __str__(self): return "E-mail {}, {}".format(self.name, self.subject) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py new file mode 100644 index 00000000..d147f10d --- /dev/null +++ b/cps/tasks/thumbnail.py @@ -0,0 +1,472 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 monkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from .. import constants +from cps import config, db, fs, gdriveutils, logger, ub +from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED +from datetime import datetime +from sqlalchemy import func, text, or_ + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + +try: + from wand.image import Image + use_IM = True +except (ImportError, RuntimeError) as e: + use_IM = False + + +def get_resize_height(resolution): + return int(225 * resolution) + + +def get_resize_width(resolution, original_width, original_height): + height = get_resize_height(resolution) + percent = (height / float(original_height)) + width = int((float(original_width) * float(percent))) + return width if width % 2 == 0 else width + 1 + + +def get_best_fit(width, height, image_width, image_height): + resize_width = int(width / 2.0) + resize_height = int(height / 2.0) + aspect_ratio = image_width / image_height + + # If this image's aspect ratio is different than the first image, then resize this image + # to fill the width and height of the first image + if aspect_ratio < width / height: + resize_width = int(width / 2.0) + resize_height = image_height * int(width / 2.0) / image_width + + elif aspect_ratio > width / height: + resize_width = image_width * int(height / 2.0) / image_height + resize_height = int(height / 2.0) + + return {'width': resize_width, 'height': resize_height} + + +class TaskGenerateCoverThumbnails(CalibreTask): + def __init__(self, task_message=''): + super(TaskGenerateCoverThumbnails, self).__init__(task_message) + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + self.resolutions = [ + constants.COVER_THUMBNAIL_SMALL, + constants.COVER_THUMBNAIL_MEDIUM + ] + + def run(self, worker_thread): + if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: + self.message = 'Scanning Books' + books_with_covers = self.get_books_with_covers() + count = len(books_with_covers) + + total_generated = 0 + for i, book in enumerate(books_with_covers): + generated = 0 + book_cover_thumbnails = self.get_book_cover_thumbnails(book.id) + + # Generate new thumbnails for missing covers + resolutions = list(map(lambda t: t.resolution, book_cover_thumbnails)) + missing_resolutions = list(set(self.resolutions).difference(resolutions)) + for resolution in missing_resolutions: + generated += 1 + self.create_book_cover_thumbnail(book, resolution) + + # Replace outdated or missing thumbnails + for thumbnail in book_cover_thumbnails: + if book.last_modified > thumbnail.generated_at: + generated += 1 + self.update_book_cover_thumbnail(book, thumbnail) + + elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): + generated += 1 + self.update_book_cover_thumbnail(book, thumbnail) + + # Increment the progress + self.progress = (1.0 / count) * i + + if generated > 0: + total_generated += generated + self.message = u'Generated {0} cover thumbnails'.format(total_generated) + + # Check if job has been cancelled or ended + if self.stat == STAT_CANCELLED: + self.log.info(f'GenerateCoverThumbnails task has been cancelled.') + return + + if self.stat == STAT_ENDED: + self.log.info(f'GenerateCoverThumbnails task has been ended.') + return + + if total_generated == 0: + self.self_cleanup = True + + self._handleSuccess() + self.app_db_session.remove() + + def get_books_with_covers(self): + return self.calibre_db.session \ + .query(db.Books) \ + .filter(db.Books.has_cover == 1) \ + .all() + + def get_book_cover_thumbnails(self, book_id): + return self.app_db_session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == constants.THUMBNAIL_TYPE_COVER) \ + .filter(ub.Thumbnail.entity_id == book_id) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ + .all() + + def create_book_cover_thumbnail(self, book, resolution): + thumbnail = ub.Thumbnail() + thumbnail.type = constants.THUMBNAIL_TYPE_COVER + thumbnail.entity_id = book.id + thumbnail.format = 'jpeg' + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error creating book thumbnail: ' + str(ex)) + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def update_book_cover_thumbnail(self, book, thumbnail): + thumbnail.generated_at = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_book_thumbnail(self, book, thumbnail): + if book and thumbnail: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: + height = get_resize_height(thumbnail.resolution) + if img.height > height: + width = get_resize_width(thumbnail.resolution, img.width, img.height) + img.resize(width=width, height=height, filter='lanczos') + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, + constants.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) + except Exception as ex: + # Bubble exception to calling function + self.log.info(u'Error generating thumbnail file: ' + str(ex)) + raise ex + finally: + if stream is not None: + stream.close() + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + height = get_resize_height(thumbnail.resolution) + if img.height > height: + width = get_resize_width(thumbnail.resolution, img.width, img.height) + img.resize(width=width, height=height, filter='lanczos') + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) + + @property + def name(self): + return 'GenerateCoverThumbnails' + + @property + def is_cancellable(self): + return True + + +class TaskGenerateSeriesThumbnails(CalibreTask): + def __init__(self, task_message=''): + super(TaskGenerateSeriesThumbnails, self).__init__(task_message) + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + self.resolutions = [ + constants.COVER_THUMBNAIL_SMALL, + constants.COVER_THUMBNAIL_MEDIUM, + ] + + def run(self, worker_thread): + if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: + self.message = 'Scanning Series' + all_series = self.get_series_with_four_plus_books() + count = len(all_series) + + total_generated = 0 + for i, series in enumerate(all_series): + generated = 0 + series_thumbnails = self.get_series_thumbnails(series.id) + series_books = self.get_series_books(series.id) + + # Generate new thumbnails for missing covers + resolutions = list(map(lambda t: t.resolution, series_thumbnails)) + missing_resolutions = list(set(self.resolutions).difference(resolutions)) + for resolution in missing_resolutions: + generated += 1 + self.create_series_thumbnail(series, series_books, resolution) + + # Replace outdated or missing thumbnails + for thumbnail in series_thumbnails: + if any(book.last_modified > thumbnail.generated_at for book in series_books): + generated += 1 + self.update_series_thumbnail(series_books, thumbnail) + + elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): + generated += 1 + self.update_series_thumbnail(series_books, thumbnail) + + # Increment the progress + self.progress = (1.0 / count) * i + + if generated > 0: + total_generated += generated + self.message = u'Generated {0} series thumbnails'.format(total_generated) + + # Check if job has been cancelled or ended + if self.stat == STAT_CANCELLED: + self.log.info(f'GenerateSeriesThumbnails task has been cancelled.') + return + + if self.stat == STAT_ENDED: + self.log.info(f'GenerateSeriesThumbnails task has been ended.') + return + + if total_generated == 0: + self.self_cleanup = True + + self._handleSuccess() + self.app_db_session.remove() + + def get_series_with_four_plus_books(self): + return self.calibre_db.session \ + .query(db.Series) \ + .join(db.books_series_link) \ + .join(db.Books) \ + .filter(db.Books.has_cover == 1) \ + .group_by(text('books_series_link.series')) \ + .having(func.count('book_series_link') > 3) \ + .all() + + def get_series_books(self, series_id): + return self.calibre_db.session \ + .query(db.Books) \ + .join(db.books_series_link) \ + .join(db.Series) \ + .filter(db.Books.has_cover == 1) \ + .filter(db.Series.id == series_id) \ + .all() + + def get_series_thumbnails(self, series_id): + return self.app_db_session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == constants.THUMBNAIL_TYPE_SERIES) \ + .filter(ub.Thumbnail.entity_id == series_id) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ + .all() + + def create_series_thumbnail(self, series, series_books, resolution): + thumbnail = ub.Thumbnail() + thumbnail.type = constants.THUMBNAIL_TYPE_SERIES + thumbnail.entity_id = series.id + thumbnail.format = 'jpeg' + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_series_thumbnail(series_books, thumbnail) + except Exception as ex: + self.log.info(u'Error creating book thumbnail: ' + str(ex)) + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def update_series_thumbnail(self, series_books, thumbnail): + thumbnail.generated_at = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + self.generate_series_thumbnail(series_books, thumbnail) + except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_series_thumbnail(self, series_books, thumbnail): + # Get the last four books in the series based on series_index + books = sorted(series_books, key=lambda b: float(b.series_index), reverse=True)[:4] + + top = 0 + left = 0 + width = 0 + height = 0 + with Image() as canvas: + for book in books: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: + # Use the first image in this set to determine the width and height to scale the + # other images in this set + if width == 0 or height == 0: + width = get_resize_width(thumbnail.resolution, img.width, img.height) + height = get_resize_height(thumbnail.resolution) + canvas.blank(width, height) + + dimensions = get_best_fit(width, height, img.width, img.height) + + # resize and crop the image + img.resize(width=int(dimensions['width']), height=int(dimensions['height']), filter='lanczos') + img.crop(width=int(width / 2.0), height=int(height / 2.0), gravity='center') + + # add the image to the canvas + canvas.composite(img, left, top) + + except Exception as ex: + self.log.info(u'Error generating thumbnail file: ' + str(ex)) + raise ex + finally: + if stream is not None: + stream.close() + + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + # Use the first image in this set to determine the width and height to scale the + # other images in this set + if width == 0 or height == 0: + width = get_resize_width(thumbnail.resolution, img.width, img.height) + height = get_resize_height(thumbnail.resolution) + canvas.blank(width, height) + + dimensions = get_best_fit(width, height, img.width, img.height) + + # resize and crop the image + img.resize(width=int(dimensions['width']), height=int(dimensions['height']), filter='lanczos') + img.crop(width=int(width / 2.0), height=int(height / 2.0), gravity='center') + + # add the image to the canvas + canvas.composite(img, left, top) + + # set the coordinates for the next iteration + if left == 0 and top == 0: + left = int(width / 2.0) + elif left == int(width / 2.0) and top == 0: + left = 0 + top = int(height / 2.0) + else: + left = int(width / 2.0) + + canvas.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + canvas.save(filename=filename) + + @property + def name(self): + return 'GenerateSeriesThumbnails' + + @property + def is_cancellable(self): + return True + + +class TaskClearCoverThumbnailCache(CalibreTask): + def __init__(self, book_id, task_message=u'Clearing cover thumbnail cache'): + super(TaskClearCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.book_id = book_id + self.app_db_session = ub.get_new_session_instance() + self.cache = fs.FileSystem() + + def run(self, worker_thread): + if self.app_db_session: + if self.book_id: + thumbnails = self.get_thumbnails_for_book(self.book_id) + for thumbnail in thumbnails: + self.delete_thumbnail(thumbnail) + + self._handleSuccess() + self.app_db_session.remove() + + def get_thumbnails_for_book(self, book_id): + return self.app_db_session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == constants.THUMBNAIL_TYPE_COVER) \ + .filter(ub.Thumbnail.entity_id == book_id) \ + .all() + + def delete_thumbnail(self, thumbnail): + thumbnail.expiration = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error deleting book thumbnail: ' + str(ex)) + self._handleError(u'Error deleting book thumbnail: ' + str(ex)) + + @property + def name(self): + return 'ThumbnailsClear' + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/upload.py b/cps/tasks/upload.py index e0bb0094..cf5a64ac 100644 --- a/cps/tasks/upload.py +++ b/cps/tasks/upload.py @@ -36,3 +36,7 @@ class TaskUpload(CalibreTask): def __str__(self): return "Upload {}".format(self.book_title) + + @property + def is_cancellable(self): + return False diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 6cd7815e..8f749e0b 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -160,15 +160,42 @@ -
-

{{_('Administration')}}

- {{_('Download Debug Package')}} - {{_('View Logs')}} +
+
+

{{_('Scheduled Tasks')}}

+
+
+
{{_('Time at which tasks start to run')}}
+
{{config.schedule_start_time}}:00
+
+
+
{{_('Time at which tasks stop running')}}
+
{{config.schedule_end_time}}:00
+
+
+
{{_('Generate book cover thumbnails')}}
+
{{ display_bool_setting(config.schedule_generate_book_covers) }}
+
+
+
{{_('Generate series cover thumbnails')}}
+
{{ display_bool_setting(config.schedule_generate_series_covers) }}
+
+
+ {{_('Edit Scheduled Tasks Settings')}}
-
-
{{_('Reconnect Calibre Database')}}
-
{{_('Restart')}}
-
{{_('Shutdown')}}
+
+ +
+

{{_('Administration')}}

+ {{_('Download Debug Package')}} + {{_('View Logs')}} +
+
+
{{_('Reconnect Calibre Database')}}
+
+
+
{{_('Restart')}}
+
{{_('Shutdown')}}
diff --git a/cps/templates/author.html b/cps/templates/author.html index d832ad5c..9f124876 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -37,7 +37,7 @@
- + {{ image.book_cover(entry, alt=author.name|safe) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 57470858..52b1e7fb 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -3,7 +3,8 @@ {% if book %}
- {{ book.title }} + +
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 06357ce8..7e720e55 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,8 @@
- {{ entry.title }} + +
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 74448b98..4842a7ea 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -1,3 +1,4 @@ +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}
@@ -9,7 +10,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html index 1421ea6a..f2e94fb2 100644 --- a/cps/templates/fragment.html +++ b/cps/templates/fragment.html @@ -1,3 +1,4 @@ +{% import 'image.html' as image %}
{% block body %}{% endblock %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html index b9d40961..638b7245 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -1,3 +1,4 @@ +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}

{{_(title)}}

@@ -27,7 +28,7 @@
- {{ entry[0].series[0].name }} + {{ image.series(entry[0].series[0], alt=entry[0].series[0].name|shortentitle) }} {{entry.count}} diff --git a/cps/templates/image.html b/cps/templates/image.html new file mode 100644 index 00000000..0bdba9a5 --- /dev/null +++ b/cps/templates/image.html @@ -0,0 +1,20 @@ +{% macro book_cover(book, alt=None) -%} + {%- set image_title = book.title if book.title else book.name -%} + {%- set image_alt = alt if alt else image_title -%} + {% set srcset = book|get_cover_srcset %} + {{ image_alt }} +{%- endmacro %} + +{% macro series(series, alt=None) -%} + {%- set image_alt = alt if alt else image_title -%} + {% set srcset = series|get_series_srcset %} + {{ book_title }} +{%- endmacro %} diff --git a/cps/templates/index.html b/cps/templates/index.html index 162adc7d..ff7ff115 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,3 +1,4 @@ +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %} {% if g.user.show_detail_random() %} @@ -9,7 +10,7 @@
- {{ entry.title }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} @@ -91,7 +92,7 @@
- {{ entry.title }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index ec69c91b..c60f63d4 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -1,4 +1,5 @@ {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal %} +{% import 'image.html' as image %} diff --git a/cps/templates/schedule_edit.html b/cps/templates/schedule_edit.html new file mode 100644 index 00000000..f989baf9 --- /dev/null +++ b/cps/templates/schedule_edit.html @@ -0,0 +1,39 @@ +{% extends "layout.html" %} +{% block header %} + + +{% endblock %} +{% block body %} +
+

{{title}}

+
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + {{_('Cancel')}} +
+
+{% endblock %} diff --git a/cps/templates/search.html b/cps/templates/search.html index a7ba5e68..12966348 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,3 +1,4 @@ +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}
@@ -45,7 +46,7 @@ {% if entry.Books.has_cover is defined %} - {{ entry.Books.title }} + {{ image.book_cover(entry.Books) }} {% if entry.Books.id in read_book_ids %}{% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index adfead60..693f6b4a 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -1,3 +1,4 @@ +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}
@@ -34,7 +35,7 @@
- {{ entry.title }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/tasks.html b/cps/templates/tasks.html index 3ef50474..0e5a352b 100644 --- a/cps/templates/tasks.html +++ b/cps/templates/tasks.html @@ -16,6 +16,9 @@ {{_('Progress')}} {{_('Run Time')}} {{_('Start Time')}} + {% if g.user.role_admin() %} + {{_('Actions')}} + {% endif %} @@ -23,6 +26,30 @@
{% endblock %} +{% block modal %} +{{ delete_book() }} +{% if g.user.role_admin() %} + +{% endif %} +{% endblock %} {% block js %} diff --git a/cps/ub.py b/cps/ub.py index 786dc909..bb456d17 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import atexit import os import sys import datetime @@ -510,6 +511,28 @@ class RemoteAuthToken(Base): return '' % self.id +def filename(context): + file_format = context.get_current_parameters()['format'] + if file_format == 'jpeg': + return context.get_current_parameters()['uuid'] + '.jpg' + else: + return context.get_current_parameters()['uuid'] + '.' + file_format + + +class Thumbnail(Base): + __tablename__ = 'thumbnail' + + id = Column(Integer, primary_key=True) + entity_id = Column(Integer) + uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) + format = Column(String, default='jpeg') + type = Column(SmallInteger, default=constants.THUMBNAIL_TYPE_COVER) + resolution = Column(SmallInteger, default=constants.COVER_THUMBNAIL_SMALL) + filename = Column(String, default=filename) + generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) + expiration = Column(DateTime, nullable=True) + + # Add missing tables during migration of database def add_missing_tables(engine, _session): if not engine.dialect.has_table(engine.connect(), "book_read_link"): @@ -524,6 +547,8 @@ def add_missing_tables(engine, _session): KoboStatistics.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "archived_book"): ArchivedBook.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "thumbnail"): + Thumbnail.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): Registration.__table__.create(bind=engine) with engine.connect() as conn: @@ -829,6 +854,16 @@ def init_db(app_db_path): sys.exit(3) +def get_new_session_instance(): + new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) + new_session = scoped_session(sessionmaker()) + new_session.configure(bind=new_engine) + + atexit.register(lambda: new_session.remove() if new_session else True) + + return new_session + + def dispose(): global session diff --git a/cps/web.py b/cps/web.py index 6d33f809..e73b9f27 100644 --- a/cps/web.py +++ b/cps/web.py @@ -48,8 +48,8 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db, kobo_sync_status from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, check_email, check_username, \ - get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ +from .helper import check_valid_domain, render_task_status, check_email, check_username, get_cc_columns, \ + get_book_cover, get_series_cover_thumbnail, get_download_link, send_mail, generate_random_password, \ send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password, valid_email, \ edit_book_read_status from .pagination import Pagination @@ -128,7 +128,7 @@ def viewer_required(f): @web.route("/ajax/emailstat") @login_required def get_email_status_json(): - tasks = WorkerThread.getInstance().tasks + tasks = WorkerThread.get_instance().tasks return jsonify(render_task_status(tasks)) @@ -761,6 +761,7 @@ def books_table(): return render_title_template('book_table.html', title=_(u"Books List"), cc=cc, page="book_table", visiblility=visibility) + @web.route("/ajax/listbooks") @login_required def list_books(): @@ -858,6 +859,7 @@ def list_books(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @web.route("/ajax/table_settings", methods=['POST']) @login_required def update_table_settings(): @@ -937,6 +939,7 @@ def publisher_list(): charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Publishers"), page="publisherlist", data="publisher", order=order_no) else: @@ -1066,7 +1069,7 @@ def category_list(): @login_required def get_tasks_status(): # if current user admin, show all email, otherwise only own emails - tasks = WorkerThread.getInstance().tasks + tasks = WorkerThread.get_instance().tasks answer = render_task_status(tasks) return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") @@ -1395,7 +1398,8 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): pagination=pagination, entries=entries, result_count=result_count, - title=_(u"Advanced Search"), page="advsearch", + title=_(u"Advanced Search"), + page="advsearch", order=order[1]) @@ -1411,14 +1415,38 @@ def advanced_search_form(): @web.route("/cover/") +@web.route("/cover//") @login_required_if_no_ano -def get_cover(book_id): - return get_book_cover(book_id) +def get_cover(book_id, resolution=None): + resolutions = { + 'og': constants.COVER_THUMBNAIL_ORIGINAL, + 'sm': constants.COVER_THUMBNAIL_SMALL, + 'md': constants.COVER_THUMBNAIL_MEDIUM, + 'lg': constants.COVER_THUMBNAIL_LARGE, + } + cover_resolution = resolutions.get(resolution, None) + return get_book_cover(book_id, cover_resolution) + + +@web.route("/series_cover/") +@web.route("/series_cover//") +@login_required_if_no_ano +def get_series_cover(series_id, resolution=None): + resolutions = { + 'og': constants.COVER_THUMBNAIL_ORIGINAL, + 'sm': constants.COVER_THUMBNAIL_SMALL, + 'md': constants.COVER_THUMBNAIL_MEDIUM, + 'lg': constants.COVER_THUMBNAIL_LARGE, + } + cover_resolution = resolutions.get(resolution, None) + return get_series_cover_thumbnail(series_id, cover_resolution) + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") + @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") @login_required_if_no_ano diff --git a/requirements.txt b/requirements.txt index 0b95dfa3..ef79d551 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +APScheduler>=3.6.3,<3.8.0 Babel>=1.3,<3.0 Flask-Babel>=0.11.1,<2.1.0 Flask-Login>=0.3.2,<0.5.1 diff --git a/test/Calibre-Web TestSummary_Windows.html b/test/Calibre-Web TestSummary_Windows.html index 6668632d..21a41612 100644 --- a/test/Calibre-Web TestSummary_Windows.html +++ b/test/Calibre-Web TestSummary_Windows.html @@ -37,20 +37,20 @@
-

Start Time: 2021-05-18 17:33:17

+

Start Time: 2021-10-06 00:29:44

-

Stop Time: 2021-05-18 20:37:38

+

Stop Time: 2021-10-06 00:37:52

-

Duration: 2h 35 min

+

Duration: 7:28 min

@@ -102,15 +102,15 @@ - - TestAnonymous + + TestShelf + 24 + 1 + 9 13 - 13 - 0 - 0 - 0 + 1 - Detail + Detail @@ -118,1439 +118,33 @@ -
TestAnonymous - test_check_locale_guest
+
TestShelf - test_add_shelf_from_search
PASS - + -
TestAnonymous - test_guest_about
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_category
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_format
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_hot
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_language
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_publisher
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_rated
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_rating
- - PASS - - - - - - -
TestAnonymous - test_guest_change_visibility_series
- - PASS - - - - - - -
TestAnonymous - test_guest_random_books_available
- - PASS - - - - - - -
TestAnonymous - test_guest_restricted_settings_visibility
- - PASS - - - - - - -
TestAnonymous - test_guest_visibility_sidebar
- - PASS - - - - - - - TestCli - 8 - 8 - 0 - 0 - 0 - - Detail - - - - - - - -
TestCli - test_already_started
- - PASS - - - - - - -
TestCli - test_bind_to_single_interface
- - PASS - - - - - - -
TestCli - test_change_password
- - PASS - - - - - - -
TestCli - test_cli_SSL_files
- - PASS - - - - - - -
TestCli - test_cli_different_folder
- - PASS - - - - - - -
TestCli - test_cli_different_settings_database
- - PASS - - - - - - -
TestCli - test_environ_port_setting
- - PASS - - - - - - -
TestCli - test_settingsdb_not_writeable
- - PASS - - - - - - - TestCliGdrivedb - 2 - 2 - 0 - 0 - 0 - - Detail - - - - - - - -
TestCliGdrivedb - test_cli_gdrive_location
- - PASS - - - - - - -
TestCliGdrivedb - test_gdrive_db_nonwrite
- - PASS - - - - - - - TestCoverEditBooks - 1 - 1 - 0 - 0 - 0 - - Detail - - - - - - - -
TestCoverEditBooks - test_upload_jpg
- - PASS - - - - - - - TestDeleteDatabase - 1 - 1 - 0 - 0 - 0 - - Detail - - - - - - - -
TestDeleteDatabase - test_delete_books_in_database
- - PASS - - - - - - - TestEbookConvertCalibre - 11 - 0 - 0 - 0 - 11 - - Detail - - - - - - - -
TestEbookConvertCalibre - test_convert_deactivate
+
TestShelf - test_adv_search_shelf
- SKIP + ERROR
-