From 21fce9a5b5b004c1fcf16aa74e96d27f7e4bde0f Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sat, 19 Dec 2020 02:58:40 -0600 Subject: [PATCH] Added background scheduler and scheduled thumbnail generation job --- cps.py | 4 --- cps/__init__.py | 9 ++++- cps/services/background_scheduler.py | 52 ++++++++++++++++++++++++++++ cps/tasks/thumbnail.py | 12 +++---- cps/thumbnails.py | 4 --- requirements.txt | 1 + 6 files changed, 66 insertions(+), 16 deletions(-) create mode 100644 cps/services/background_scheduler.py diff --git a/cps.py b/cps.py index e90a38d9..50ab0076 100755 --- a/cps.py +++ b/cps.py @@ -43,7 +43,6 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler -from cps.thumbnails import generate_thumbnails try: from cps.kobo import kobo, get_kobo_activated @@ -79,9 +78,6 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) - - generate_thumbnails() - success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index 1a7dc868..fa85e15c 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -36,6 +36,8 @@ from flask_principal import Principal from . import config_sql, logger, cache_buster, cli, ub, db from .reverseproxy import ReverseProxied from .server import WebServer +from .services.background_scheduler import BackgroundScheduler +from .tasks.thumbnail import TaskThumbnail mimetypes.init() @@ -95,7 +97,7 @@ def create_app(): app.instance_path = app.instance_path.decode('utf-8') if os.environ.get('FLASK_DEBUG'): - cache_buster.init_cache_busting(app) + cache_buster.init_cache_busting(app) log.info('Starting Calibre Web...') Principal(app) @@ -115,8 +117,13 @@ def create_app(): config.config_goodreads_api_secret, config.config_use_goodreads) + scheduler = BackgroundScheduler() + # Generate 100 book cover thumbnails every 5 minutes + scheduler.add_task(user=None, task=lambda: TaskThumbnail(config=config, limit=100), trigger='interval', minutes=5) + return app + @babel.localeselector def get_locale(): # if a user is logged in, use the locale from the user settings diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py new file mode 100644 index 00000000..efa57379 --- /dev/null +++ b/cps/services/background_scheduler.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import atexit + +from .. import logger +from .worker import WorkerThread +from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + + +class BackgroundScheduler: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(BackgroundScheduler, cls).__new__(cls) + + scheduler = BScheduler() + atexit.register(lambda: scheduler.shutdown()) + + cls.log = logger.create() + cls.scheduler = scheduler + cls.scheduler.start() + + return cls._instance + + def add(self, func, trigger, **trigger_args): + self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + + def add_task(self, user, task, trigger, **trigger_args): + def scheduled_task(): + worker_task = task() + self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) + WorkerThread.add(user, worker_task) + + self.add(func=scheduled_task, trigger=trigger, **trigger_args) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index c452ab41..378b688e 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,7 +19,7 @@ from __future__ import division, print_function, unicode_literals import os -from cps import config, db, gdriveutils, logger, ub +from cps import db, logger, ub from cps.constants import CACHE_DIR as _CACHE_DIR from cps.services.worker import CalibreTask from datetime import datetime, timedelta @@ -36,8 +36,9 @@ THUMBNAIL_RESOLUTION_2X = 2.0 class TaskThumbnail(CalibreTask): - def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + def __init__(self, config, limit=100, task_message=u'Generating cover thumbnails'): super(TaskThumbnail, self).__init__(task_message) + self.config = config self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -47,9 +48,6 @@ class TaskThumbnail(CalibreTask): if self.worker_db.session and use_IM: thumbnails = self.get_thumbnail_book_ids() thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails)) - self.log.info(','.join([str(elem) for elem in thumbnail_book_ids])) - self.log.info(len(thumbnail_book_ids)) - books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) count = len(books_without_thumbnails) @@ -116,10 +114,10 @@ class TaskThumbnail(CalibreTask): def generate_book_thumbnail(self, book, thumbnail): if book and thumbnail: - if config.config_use_google_drive: + if self.config.config_use_google_drive: self.log.info('google drive thumbnail') else: - book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + book_cover_filepath = os.path.join(self.config.config_calibre_dir, book.path, 'cover.jpg') if os.path.isfile(book_cover_filepath): with Image(filename=book_cover_filepath) as img: height = self.get_thumbnail_height(thumbnail) diff --git a/cps/thumbnails.py b/cps/thumbnails.py index 6ccff56f..89e68f50 100644 --- a/cps/thumbnails.py +++ b/cps/thumbnails.py @@ -57,7 +57,3 @@ def cover_thumbnail_exists_for_book(book): return thumbnail_path and os.path.isfile(thumbnail_path) return False - - -def generate_thumbnails(): - WorkerThread.add(None, TaskThumbnail()) diff --git a/requirements.txt b/requirements.txt index f154dd5b..9d0a8654 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +APScheduler==3.6.3 Babel>=1.3, <2.9 Flask-Babel>=0.11.1,<2.1.0 Flask-Login>=0.3.2,<0.5.1