Added thumbnail task and database table

This commit is contained in:
mmonkey 2020-12-19 00:49:36 -06:00
parent 9a20faf640
commit 774b9ae12d
7 changed files with 264 additions and 4 deletions

1
.gitignore vendored
View File

@ -21,6 +21,7 @@ vendor/
# calibre-web
*.db
*.log
cps/cache
.idea/
*.bak

4
cps.py
View File

@ -43,6 +43,7 @@ from cps.gdrive import gdrive
from cps.editbooks import editbook
from cps.remotelogin import remotelogin
from cps.error_handler import init_errorhandler
from cps.thumbnails import generate_thumbnails
try:
from cps.kobo import kobo, get_kobo_activated
@ -78,6 +79,9 @@ def main():
app.register_blueprint(kobo_auth)
if oauth_available:
app.register_blueprint(oauth)
generate_thumbnails()
success = web_server.start()
sys.exit(0 if success else 1)

View File

@ -33,6 +33,7 @@ else:
STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static')
TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates')
TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations')
CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache')
if HOME_CONFIG:
home_dir = os.path.join(os.path.expanduser("~"),".calibre-web")

View File

@ -551,6 +551,11 @@ def get_book_cover_with_uuid(book_uuid,
def get_book_cover_internal(book, use_generic_cover_on_failure):
if book and book.has_cover:
# if thumbnails.cover_thumbnail_exists_for_book(book):
# thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
# return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename)
# else:
# WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title)))
if config.config_use_google_drive:
try:
if not gd.is_gdrive_ready():
@ -561,8 +566,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
else:
log.error('%s/cover.jpg not found on Google Drive', book.path)
return get_cover_on_failure(use_generic_cover_on_failure)
except Exception as e:
log.debug_or_exception(e)
except Exception as ex:
log.debug_or_exception(ex)
return get_cover_on_failure(use_generic_cover_on_failure)
else:
cover_file_path = os.path.join(config.config_calibre_dir, book.path)

154
cps/tasks/thumbnail.py Normal file
View File

@ -0,0 +1,154 @@
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2020 mmonkey
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
import os
from cps import config, db, gdriveutils, logger, ub
from cps.constants import CACHE_DIR as _CACHE_DIR
from cps.services.worker import CalibreTask
from datetime import datetime, timedelta
from sqlalchemy import func
try:
from wand.image import Image
use_IM = True
except (ImportError, RuntimeError) as e:
use_IM = False
THUMBNAIL_RESOLUTION_1X = 1.0
THUMBNAIL_RESOLUTION_2X = 2.0
class TaskThumbnail(CalibreTask):
def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
super(TaskThumbnail, self).__init__(task_message)
self.limit = limit
self.log = logger.create()
self.app_db_session = ub.get_new_session_instance()
self.worker_db = db.CalibreDB(expire_on_commit=False)
def run(self, worker_thread):
if self.worker_db.session and use_IM:
thumbnails = self.get_thumbnail_book_ids()
thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails))
self.log.info(','.join([str(elem) for elem in thumbnail_book_ids]))
self.log.info(len(thumbnail_book_ids))
books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
count = len(books_without_thumbnails)
for i, book in enumerate(books_without_thumbnails):
thumbnails = self.get_thumbnails_for_book(thumbnails, book)
if thumbnails:
for thumbnail in thumbnails:
self.update_book_thumbnail(book, thumbnail)
else:
self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X)
self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X)
self.progress = (1.0 / count) * i
self._handleSuccess()
self.app_db_session.close()
def get_thumbnail_book_ids(self):
return self.app_db_session\
.query(ub.Thumbnail)\
.group_by(ub.Thumbnail.book_id)\
.having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\
.all()
def get_books_without_thumbnails(self, thumbnail_book_ids):
return self.worker_db.session\
.query(db.Books)\
.filter(db.Books.has_cover == 1)\
.filter(db.Books.id.notin_(thumbnail_book_ids))\
.limit(self.limit)\
.all()
def get_thumbnails_for_book(self, thumbnails, book):
results = list()
for thumbnail in thumbnails:
if thumbnail.book_id == book.id:
results.append(thumbnail)
return results
def update_book_thumbnail(self, book, thumbnail):
thumbnail.expiration = datetime.utcnow() + timedelta(days=30)
try:
self.app_db_session.commit()
self.generate_book_thumbnail(book, thumbnail)
except Exception as ex:
self._handleError(u'Error updating book thumbnail: ' + str(ex))
self.app_db_session.rollback()
def create_book_thumbnail(self, book, resolution):
thumbnail = ub.Thumbnail()
thumbnail.book_id = book.id
thumbnail.resolution = resolution
self.app_db_session.add(thumbnail)
try:
self.app_db_session.commit()
self.generate_book_thumbnail(book, thumbnail)
except Exception as ex:
self._handleError(u'Error creating book thumbnail: ' + str(ex))
self.app_db_session.rollback()
def generate_book_thumbnail(self, book, thumbnail):
if book and thumbnail:
if config.config_use_google_drive:
self.log.info('google drive thumbnail')
else:
book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg')
if os.path.isfile(book_cover_filepath):
with Image(filename=book_cover_filepath) as img:
height = self.get_thumbnail_height(thumbnail)
if img.height > height:
width = self.get_thumbnail_width(height, img)
img.resize(width=width, height=height, filter='lanczos')
img.save(filename=self.get_thumbnail_cache_path(thumbnail))
def get_thumbnail_height(self, thumbnail):
return int(225 * thumbnail.resolution)
def get_thumbnail_width(self, height, img):
percent = (height / float(img.height))
return int((float(img.width) * float(percent)))
def get_thumbnail_cache_dir(self):
if not os.path.isdir(_CACHE_DIR):
os.makedirs(_CACHE_DIR)
if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
return os.path.join(_CACHE_DIR, 'thumbnails')
def get_thumbnail_cache_path(self, thumbnail):
if thumbnail:
return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename)
return None
@property
def name(self):
return "Thumbnail"

63
cps/thumbnails.py Normal file
View File

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2020 mmonkey
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, print_function, unicode_literals
import os
from . import logger, ub
from .constants import CACHE_DIR as _CACHE_DIR
from .services.worker import WorkerThread
from .tasks.thumbnail import TaskThumbnail
from datetime import datetime
THUMBNAIL_RESOLUTION_1X = 1.0
THUMBNAIL_RESOLUTION_2X = 2.0
log = logger.create()
def get_thumbnail_cache_dir():
if not os.path.isdir(_CACHE_DIR):
os.makedirs(_CACHE_DIR)
if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
return os.path.join(_CACHE_DIR, 'thumbnails')
def get_thumbnail_cache_path(thumbnail):
if thumbnail:
return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename)
return None
def cover_thumbnail_exists_for_book(book):
if book and book.has_cover:
thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first()
if thumbnail and thumbnail.expiration > datetime.utcnow():
thumbnail_path = get_thumbnail_cache_path(thumbnail)
return thumbnail_path and os.path.isfile(thumbnail_path)
return False
def generate_thumbnails():
WorkerThread.add(None, TaskThumbnail())

View File

@ -40,13 +40,14 @@ except ImportError:
oauth_support = False
from sqlalchemy import create_engine, exc, exists, event
from sqlalchemy import Column, ForeignKey
from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session
from werkzeug.security import generate_password_hash
from . import constants
from . import cli, constants
session = None
@ -434,6 +435,28 @@ class RemoteAuthToken(Base):
return '<Token %r>' % self.id
class Thumbnail(Base):
__tablename__ = 'thumbnail'
id = Column(Integer, primary_key=True)
book_id = Column(Integer)
uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True)
format = Column(String, default='jpeg')
resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0)
expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30))
@hybrid_property
def extension(self):
if self.format == 'jpeg':
return 'jpg'
else:
return self.format
@hybrid_property
def filename(self):
return self.uuid + '.' + self.extension
# Migrate database to current version, has to be updated after every database change. Currently migration from
# everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding
# rows with SQL commands
@ -451,6 +474,8 @@ def migrate_Database(session):
KoboStatistics.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "archived_book"):
ArchivedBook.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "thumbnail"):
Thumbnail.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "registration"):
ReadBook.__table__.create(bind=engine)
with engine.connect() as conn:
@ -676,6 +701,13 @@ def init_db(app_db_path):
create_anonymous_user(session)
def get_new_session_instance():
new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False)
new_session = scoped_session(sessionmaker())
new_session.configure(bind=new_engine)
return new_session
def dispose():
global session