Added thumbnail task and database table

.gitignore

@@ -21,6 +21,7 @@ vendor/
 # calibre-web
 *.db
 *.log
+cps/cache
 
 .idea/
 *.bak

cps.py

@@ -43,6 +43,7 @@ from cps.gdrive import gdrive
 from cps.editbooks import editbook
 from cps.remotelogin import remotelogin
 from cps.error_handler import init_errorhandler
+from cps.thumbnails import generate_thumbnails
 
 try:
     from cps.kobo import kobo, get_kobo_activated
@@ -78,6 +79,9 @@ def main():
         app.register_blueprint(kobo_auth)
     if oauth_available:
         app.register_blueprint(oauth)
+
+    generate_thumbnails()
+
     success = web_server.start()
     sys.exit(0 if success else 1)
 

cps/constants.py

@@ -33,6 +33,7 @@ else:
 STATIC_DIR          = os.path.join(BASE_DIR, 'cps', 'static')
 TEMPLATES_DIR       = os.path.join(BASE_DIR, 'cps', 'templates')
 TRANSLATIONS_DIR    = os.path.join(BASE_DIR, 'cps', 'translations')
+CACHE_DIR           = os.path.join(BASE_DIR, 'cps', 'cache')
 
 if HOME_CONFIG:
     home_dir = os.path.join(os.path.expanduser("~"),".calibre-web")

cps/helper.py

@@ -551,6 +551,11 @@ def get_book_cover_with_uuid(book_uuid,
 
 def get_book_cover_internal(book, use_generic_cover_on_failure):
     if book and book.has_cover:
+        # if thumbnails.cover_thumbnail_exists_for_book(book):
+        #     thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
+        #     return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename)
+        # else:
+            # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title)))
         if config.config_use_google_drive:
             try:
                 if not gd.is_gdrive_ready():
@@ -561,8 +566,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
                 else:
                     log.error('%s/cover.jpg not found on Google Drive', book.path)
                     return get_cover_on_failure(use_generic_cover_on_failure)
-            except Exception as e:
+            except Exception as ex:
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
                 return get_cover_on_failure(use_generic_cover_on_failure)
         else:
             cover_file_path = os.path.join(config.config_calibre_dir, book.path)

cps/tasks/thumbnail.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import os
+
+from cps import config, db, gdriveutils, logger, ub
+from cps.constants import CACHE_DIR as _CACHE_DIR
+from cps.services.worker import CalibreTask
+from datetime import datetime, timedelta
+from sqlalchemy import func
+
+try:
+    from wand.image import Image
+    use_IM = True
+except (ImportError, RuntimeError) as e:
+    use_IM = False
+
+THUMBNAIL_RESOLUTION_1X = 1.0
+THUMBNAIL_RESOLUTION_2X = 2.0
+
+
+class TaskThumbnail(CalibreTask):
+    def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
+        super(TaskThumbnail, self).__init__(task_message)
+        self.limit = limit
+        self.log = logger.create()
+        self.app_db_session = ub.get_new_session_instance()
+        self.worker_db = db.CalibreDB(expire_on_commit=False)
+
+    def run(self, worker_thread):
+        if self.worker_db.session and use_IM:
+            thumbnails = self.get_thumbnail_book_ids()
+            thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails))
+            self.log.info(','.join([str(elem) for elem in thumbnail_book_ids]))
+            self.log.info(len(thumbnail_book_ids))
+
+            books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
+
+            count = len(books_without_thumbnails)
+            for i, book in enumerate(books_without_thumbnails):
+                thumbnails = self.get_thumbnails_for_book(thumbnails, book)
+                if thumbnails:
+                    for thumbnail in thumbnails:
+                        self.update_book_thumbnail(book, thumbnail)
+
+                else:
+                    self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X)
+                    self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X)
+
+                self.progress = (1.0 / count) * i
+
+        self._handleSuccess()
+        self.app_db_session.close()
+
+    def get_thumbnail_book_ids(self):
+        return self.app_db_session\
+            .query(ub.Thumbnail)\
+            .group_by(ub.Thumbnail.book_id)\
+            .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\
+            .all()
+
+    def get_books_without_thumbnails(self, thumbnail_book_ids):
+        return self.worker_db.session\
+            .query(db.Books)\
+            .filter(db.Books.has_cover == 1)\
+            .filter(db.Books.id.notin_(thumbnail_book_ids))\
+            .limit(self.limit)\
+            .all()
+
+    def get_thumbnails_for_book(self, thumbnails, book):
+        results = list()
+        for thumbnail in thumbnails:
+            if thumbnail.book_id == book.id:
+                results.append(thumbnail)
+
+        return results
+
+    def update_book_thumbnail(self, book, thumbnail):
+        thumbnail.expiration = datetime.utcnow() + timedelta(days=30)
+
+        try:
+            self.app_db_session.commit()
+            self.generate_book_thumbnail(book, thumbnail)
+        except Exception as ex:
+            self._handleError(u'Error updating book thumbnail: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def create_book_thumbnail(self, book, resolution):
+        thumbnail = ub.Thumbnail()
+        thumbnail.book_id = book.id
+        thumbnail.resolution = resolution
+
+        self.app_db_session.add(thumbnail)
+        try:
+            self.app_db_session.commit()
+            self.generate_book_thumbnail(book, thumbnail)
+        except Exception as ex:
+            self._handleError(u'Error creating book thumbnail: ' + str(ex))
+            self.app_db_session.rollback()
+
+    def generate_book_thumbnail(self, book, thumbnail):
+        if book and thumbnail:
+            if config.config_use_google_drive:
+                self.log.info('google drive thumbnail')
+            else:
+                book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg')
+                if os.path.isfile(book_cover_filepath):
+                    with Image(filename=book_cover_filepath) as img:
+                        height = self.get_thumbnail_height(thumbnail)
+                        if img.height > height:
+                            width = self.get_thumbnail_width(height, img)
+                            img.resize(width=width, height=height, filter='lanczos')
+                            img.save(filename=self.get_thumbnail_cache_path(thumbnail))
+
+    def get_thumbnail_height(self, thumbnail):
+        return int(225 * thumbnail.resolution)
+
+    def get_thumbnail_width(self, height, img):
+        percent = (height / float(img.height))
+        return int((float(img.width) * float(percent)))
+
+    def get_thumbnail_cache_dir(self):
+        if not os.path.isdir(_CACHE_DIR):
+            os.makedirs(_CACHE_DIR)
+
+        if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
+            os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
+
+        return os.path.join(_CACHE_DIR, 'thumbnails')
+
+    def get_thumbnail_cache_path(self, thumbnail):
+        if thumbnail:
+            return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename)
+        return None
+
+    @property
+    def name(self):
+        return "Thumbnail"

cps/thumbnails.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2020 mmonkey
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import os
+
+from . import logger, ub
+from .constants import CACHE_DIR as _CACHE_DIR
+from .services.worker import WorkerThread
+from .tasks.thumbnail import TaskThumbnail
+
+from datetime import datetime
+
+THUMBNAIL_RESOLUTION_1X = 1.0
+THUMBNAIL_RESOLUTION_2X = 2.0
+
+log = logger.create()
+
+
+def get_thumbnail_cache_dir():
+    if not os.path.isdir(_CACHE_DIR):
+        os.makedirs(_CACHE_DIR)
+
+    if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
+        os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
+
+    return os.path.join(_CACHE_DIR, 'thumbnails')
+
+
+def get_thumbnail_cache_path(thumbnail):
+    if thumbnail:
+        return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename)
+
+    return None
+
+
+def cover_thumbnail_exists_for_book(book):
+    if book and book.has_cover:
+        thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
+        if thumbnail and thumbnail.expiration > datetime.utcnow():
+            thumbnail_path = get_thumbnail_cache_path(thumbnail)
+            return thumbnail_path and os.path.isfile(thumbnail_path)
+
+    return False
+
+
+def generate_thumbnails():
+    WorkerThread.add(None, TaskThumbnail())

cps/ub.py

@@ -40,13 +40,14 @@ except ImportError:
         oauth_support = False
 from sqlalchemy import create_engine, exc, exists, event
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
+from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session
 from werkzeug.security import generate_password_hash
 
-from . import constants
+from . import cli, constants
 
 
 session = None
@@ -434,6 +435,28 @@ class RemoteAuthToken(Base):
         return '<Token %r>' % self.id
 
 
+class Thumbnail(Base):
+    __tablename__ = 'thumbnail'
+
+    id = Column(Integer, primary_key=True)
+    book_id = Column(Integer)
+    uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True)
+    format = Column(String, default='jpeg')
+    resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0)
+    expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30))
+
+    @hybrid_property
+    def extension(self):
+        if self.format == 'jpeg':
+            return 'jpg'
+        else:
+            return self.format
+
+    @hybrid_property
+    def filename(self):
+        return self.uuid + '.' + self.extension
+
+
 # Migrate database to current version, has to be updated after every database change. Currently migration from
 # everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding
 # rows with SQL commands
@@ -451,6 +474,8 @@ def migrate_Database(session):
         KoboStatistics.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "archived_book"):
         ArchivedBook.__table__.create(bind=engine)
+    if not engine.dialect.has_table(engine.connect(), "thumbnail"):
+        Thumbnail.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "registration"):
         ReadBook.__table__.create(bind=engine)
         with engine.connect() as conn:
@@ -676,6 +701,13 @@ def init_db(app_db_path):
         create_anonymous_user(session)
 
 
+def get_new_session_instance():
+    new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False)
+    new_session = scoped_session(sessionmaker())
+    new_session.configure(bind=new_engine)
+    return new_session
+
+
 def dispose():
     global session