Merge remote-tracking branch 'upstream/master'

.github/ISSUE_TEMPLATE/bug_report.md

@@ -31,7 +31,7 @@ If applicable, add screenshots to help explain your problem.
  - OS: [e.g. Windows 10/Raspberry Pi OS]
  - Python version: [e.g. python2.7]
  - Calibre-Web version: [e.g. 0.6.8 or 087c4c59 (git rev-parse --short HEAD)]:
- - Docker container: [None/Technosoft2000/Linuxuser]:
+ - Docker container: [None/Technosoft2000/LinuxServer]:
  - Special Hardware: [e.g. Rasperry Pi Zero]
  - Browser: [e.g. Chrome 83.0.4103.97, Safari 13.3.7, Firefox 68.0.1 ESR]
 

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.gitignore

@@ -10,6 +10,7 @@ env/
 venv/
 eggs/
 dist/
+executable/
 build/
 vendor/
 .eggs/
@@ -29,4 +30,4 @@ vendor/
 settings.yaml
 gdrive_credentials
 client_secrets.json
-
+gmail.json

README.md

@@ -12,7 +12,7 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d
 - full graphical setup
 - User management with fine-grained per-user permissions
 - Admin interface
-- User Interface in czech, dutch, english, finnish, french, german, greek, hungarian, italian, japanese, khmer, polish, russian, simplified chinese, spanish, swedish, turkish, ukrainian
+- User Interface in brazilian, czech, dutch, english, finnish, french, german, greek, hungarian, italian, japanese, khmer, polish, russian, simplified chinese, spanish, swedish, turkish, ukrainian
 - OPDS feed for eBook reader apps 
 - Filter and search by titles, authors, tags, series and language
 - Create a custom book collection (shelves)
@@ -22,7 +22,7 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d
 - Support for public user registration
 - Send eBooks to Kindle devices with the click of a button
 - Sync your Kobo devices through Calibre-Web with your Calibre library
-- Support for reading eBooks directly in the browser (.txt, .epub, .pdf, .cbr, .cbt, .cbz)
+- Support for reading eBooks directly in the browser (.txt, .epub, .pdf, .cbr, .cbt, .cbz, .djvu)
 - Upload new books in many formats, including audio formats (.mp3, .m4a, .m4b)
 - Support for Calibre Custom Columns
 - Ability to hide content based on categories and Custom Column content per user
@@ -32,8 +32,8 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d
 
 ## Quick start
 
-1. Install dependencies by running `pip3 install --target vendor -r requirements.txt` (python3.x) or `pip install --target vendor -r requirements.txt` (python2.7).
-2. Execute the command: `python cps.py` (or `nohup python cps.py` - recommended if you want to exit the terminal window)
+1. Install dependencies by running `pip3 install --target vendor -r requirements.txt` (python3.x). Alternativly set up a python virtual environment.
+2. Execute the command: `python3 cps.py` (or `nohup python3 cps.py` - recommended if you want to exit the terminal window)
 3. Point your browser to `http://localhost:8083` or `http://localhost:8083/opds` for the OPDS catalog
 4. Set `Location of Calibre database` to the path of the folder where your Calibre library (metadata.db) lives, push "submit" button\
    Optionally a Google Drive can be used to host the calibre library [-> Using Google Drive integration](https://github.com/janeczku/calibre-web/wiki/Configuration#using-google-drive-integration)
@@ -48,7 +48,7 @@ Please note that running the above install command can fail on some versions of
 
 ## Requirements
 
-python 3.x+, (Python 2.7+)
+python 3.x+
 
 Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata:
 

cps.py

@@ -31,7 +31,7 @@ else:
     sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'vendor'))
 
 
-from cps import create_app, config
+from cps import create_app
 from cps import web_server
 from cps.opds import opds
 from cps.web import web
@@ -41,6 +41,8 @@ from cps.shelf import shelf
 from cps.admin import admi
 from cps.gdrive import gdrive
 from cps.editbooks import editbook
+from cps.remotelogin import remotelogin
+from cps.error_handler import init_errorhandler
 
 try:
     from cps.kobo import kobo, get_kobo_activated
@@ -58,13 +60,17 @@ except ImportError:
 
 def main():
     app = create_app()
+
+    init_errorhandler()
+
     app.register_blueprint(web)
     app.register_blueprint(opds)
     app.register_blueprint(jinjia)
     app.register_blueprint(about)
     app.register_blueprint(shelf)
     app.register_blueprint(admi)
-    if config.config_use_google_drive:
+    app.register_blueprint(remotelogin)
+    # if config.config_use_google_drive:
     app.register_blueprint(gdrive)
     app.register_blueprint(editbook)
     if kobo_available:

cps/__init__.py

@@ -45,6 +45,7 @@ mimetypes.add_type('application/fb2+zip', '.fb2')
 mimetypes.add_type('application/x-mobipocket-ebook', '.mobi')
 mimetypes.add_type('application/x-mobipocket-ebook', '.prc')
 mimetypes.add_type('application/vnd.amazon.ebook', '.azw')
+mimetypes.add_type('application/x-mobi8-ebook', '.azw3')
 mimetypes.add_type('application/x-cbr', '.cbr')
 mimetypes.add_type('application/x-cbz', '.cbz')
 mimetypes.add_type('application/x-cbt', '.cbt')
@@ -94,9 +95,13 @@ def create_app():
         app.root_path = app.root_path.decode('utf-8')
         app.instance_path = app.instance_path.decode('utf-8')
 
+    if os.environ.get('FLASK_DEBUG'):
         cache_buster.init_cache_busting(app)
 
     log.info('Starting Calibre Web...')
+    if sys.version_info < (3, 0):
+        log.info('Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2 please consider upgrading to Python3')
+        print('Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2 please consider upgrading to Python3')
     Principal(app)
     lm.init_app(app)
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
@@ -122,7 +127,7 @@ def get_locale():
     user = getattr(g, 'user', None)
     # user = None
     if user is not None and hasattr(user, "locale"):
-        if user.nickname != 'Guest':   # if the account is the guest account bypass the config lang settings
+        if user.name != 'Guest':   # if the account is the guest account bypass the config lang settings
             return user.locale
 
     preferred = list()

cps/about.py

@@ -31,12 +31,13 @@ import werkzeug, flask, flask_login, flask_principal, jinja2
 from flask_babel import gettext as _
 
 from . import db, calibre_db, converter, uploader, server, isoLanguages, constants
-from .web import render_title_template
+from .render_template import render_title_template
 try:
     from flask_login import __version__ as flask_loginVersion
 except ImportError:
     from flask_login.__about__ import __version__ as flask_loginVersion
 try:
+    # pylint: disable=unused-import
     import unidecode
     # _() necessary to make babel aware of string for translation
     unidecode_version = _(u'installed')
@@ -48,6 +49,11 @@ try:
 except ImportError:
     flask_danceVersion = None
 
+try:
+    from greenlet import __version__ as greenlet_Version
+except ImportError:
+    greenlet_Version = None
+
 from . import services
 
 about = flask.Blueprint('about', __name__)
@@ -77,7 +83,8 @@ _VERSIONS = OrderedDict(
     python_LDAP = services.ldapVersion if bool(services.ldapVersion) else None,
     Goodreads = u'installed' if bool(services.goodreads_support) else None,
     jsonschema = services.SyncToken.__version__  if bool(services.SyncToken) else None,
-    flask_dance = flask_danceVersion
+    flask_dance = flask_danceVersion,
+    greenlet = greenlet_Version
 )
 _VERSIONS.update(uploader.get_versions())
 

cps/cache_buster.py

@@ -49,7 +49,7 @@ def init_cache_busting(app):
             # compute version component
             rooted_filename = os.path.join(dirpath, filename)
             with open(rooted_filename, 'rb') as f:
-                file_hash = hashlib.md5(f.read()).hexdigest()[:7]
+                file_hash = hashlib.md5(f.read()).hexdigest()[:7] # nosec
 
             # save version to tables
             file_path = rooted_filename.replace(static_folder, "")
@@ -64,6 +64,7 @@ def init_cache_busting(app):
         return filename.split("?", 1)[0]
 
     @app.url_defaults
+    # pylint: disable=unused-variable
     def reverse_to_cache_busted_url(endpoint, values):
         """
         Make `url_for` produce busted filenames when using the 'static' endpoint.

cps/cli.py

@@ -45,6 +45,7 @@ parser.add_argument('-v', '--version', action='version', help='Shows version num
                     version=version_info())
 parser.add_argument('-i', metavar='ip-address', help='Server IP-Address to listen')
 parser.add_argument('-s', metavar='user:pass', help='Sets specific username to new password')
+parser.add_argument('-f', action='store_true', help='Enables filepicker in unconfigured mode')
 args = parser.parse_args()
 
 if sys.version_info < (3, 0):
@@ -70,7 +71,7 @@ if args.c:
     if os.path.isfile(args.c):
         certfilepath = args.c
     else:
-        print("Certfilepath is invalid. Exiting...")
+        print("Certfile path is invalid. Exiting...")
         sys.exit(1)
 
 if args.c == "":
@@ -80,7 +81,7 @@ if args.k:
     if os.path.isfile(args.k):
         keyfilepath = args.k
     else:
-        print("Keyfilepath is invalid. Exiting...")
+        print("Keyfile path is invalid. Exiting...")
         sys.exit(1)
 
 if (args.k and not args.c) or (not args.k and args.c):
@@ -90,23 +91,29 @@ if (args.k and not args.c) or (not args.k and args.c):
 if args.k == "":
     keyfilepath = ""
 
-# handle and check ipadress argument
-ipadress = args.i or None
-if ipadress:
+# handle and check ip address argument
+ip_address = args.i or None
+if ip_address:
     try:
         # try to parse the given ip address with socket
         if hasattr(socket, 'inet_pton'):
-            if ':' in ipadress:
-                socket.inet_pton(socket.AF_INET6, ipadress)
+            if ':' in ip_address:
+                socket.inet_pton(socket.AF_INET6, ip_address)
             else:
-                socket.inet_pton(socket.AF_INET, ipadress)
+                socket.inet_pton(socket.AF_INET, ip_address)
         else:
             # on windows python < 3.4, inet_pton is not available
             # inet_atom only handles IPv4 addresses
-            socket.inet_aton(ipadress)
+            socket.inet_aton(ip_address)
     except socket.error as err:
-        print(ipadress, ':', err)
+        print(ip_address, ':', err)
         sys.exit(1)
 
 # handle and check user password argument
-user_password = args.s or None
+user_credentials = args.s or None
+if user_credentials and ":" not in user_credentials:
+    print("No valid 'username:password' format")
+    sys.exit(3)
+
+# Handles enabling of filepicker
+filepicker = args.f or None

cps/comic.py

@@ -18,21 +18,21 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
-import io
 
 from . import logger, isoLanguages
 from .constants import BookMeta
 
-try:
-    from PIL import Image as PILImage
-    use_PIL = True
-except ImportError as e:
-    use_PIL = False
-
 
 log = logger.create()
 
 
+try:
+    from wand.image import Image
+    use_IM = True
+except (ImportError, RuntimeError) as e:
+    use_IM = False
+
+
 try:
     from comicapi.comicarchive import ComicArchive, MetaDataStyle
     use_comic_meta = True
@@ -52,45 +52,37 @@ except (ImportError, LookupError) as e:
         use_rarfile = False
     use_comic_meta = False
 
+NO_JPEG_EXTENSIONS = ['.png', '.webp', '.bmp']
+COVER_EXTENSIONS = ['.png', '.webp', '.bmp', '.jpg', '.jpeg']
+
 def _cover_processing(tmp_file_name, img, extension):
-    if use_PIL:
+    tmp_cover_name = os.path.join(os.path.dirname(tmp_file_name), 'cover.jpg')
+    if use_IM:
         # convert to jpg because calibre only supports jpg
-        if extension in ('.png',  '.webp'):
-            imgc = PILImage.open(io.BytesIO(img))
-            im = imgc.convert('RGB')
-            tmp_bytesio = io.BytesIO()
-            im.save(tmp_bytesio, format='JPEG')
-            img = tmp_bytesio.getvalue()
+        if extension in NO_JPEG_EXTENSIONS:
+            with Image(filename=tmp_file_name) as imgc:
+                imgc.format = 'jpeg'
+                imgc.transform_colorspace('rgb')
+                imgc.save(tmp_cover_name)
+                return tmp_cover_name
 
     if not img:
         return None
 
-    tmp_cover_name = os.path.join(os.path.dirname(tmp_file_name), 'cover.jpg')
     with open(tmp_cover_name, 'wb') as f:
         f.write(img)
     return tmp_cover_name
 
 
-
-def _extractCover(tmp_file_name, original_file_extension, rarExecutable):
-    cover_data = extension = None
-    if use_comic_meta:
-        archive = ComicArchive(tmp_file_name, rar_exe_path=rarExecutable)
-        for index, name in enumerate(archive.getPageNameList()):
-            ext = os.path.splitext(name)
-            if len(ext) > 1:
-                extension = ext[1].lower()
-                if extension in ('.jpg', '.jpeg', '.png', '.webp'):
-                    cover_data = archive.getPage(index)
-                    break
-    else:
+def _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecutable):
+    cover_data = None
     if original_file_extension.upper() == '.CBZ':
         cf = zipfile.ZipFile(tmp_file_name)
         for name in cf.namelist():
             ext = os.path.splitext(name)
             if len(ext) > 1:
                 extension = ext[1].lower()
-                    if extension in ('.jpg', '.jpeg', '.png', '.webp'):
+                if extension in COVER_EXTENSIONS:
                     cover_data = cf.read(name)
                     break
     elif original_file_extension.upper() == '.CBT':
@@ -99,7 +91,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExecutable):
             ext = os.path.splitext(name)
             if len(ext) > 1:
                 extension = ext[1].lower()
-                    if extension in ('.jpg', '.jpeg', '.png', '.webp'):
+                if extension in COVER_EXTENSIONS:
                     cover_data = cf.extractfile(name).read()
                     break
     elif original_file_extension.upper() == '.CBR' and use_rarfile:
@@ -110,11 +102,27 @@ def _extractCover(tmp_file_name, original_file_extension, rarExecutable):
                 ext = os.path.splitext(name)
                 if len(ext) > 1:
                     extension = ext[1].lower()
-                        if extension in ('.jpg', '.jpeg', '.png', '.webp'):
+                    if extension in COVER_EXTENSIONS:
                         cover_data = cf.read(name)
                         break
-            except Exception as e:
-                log.debug('Rarfile failed with error: %s', e)
+        except Exception as ex:
+            log.debug('Rarfile failed with error: %s', ex)
+    return cover_data
+
+
+def _extractCover(tmp_file_name, original_file_extension, rarExecutable):
+    cover_data = extension = None
+    if use_comic_meta:
+        archive = ComicArchive(tmp_file_name, rar_exe_path=rarExecutable)
+        for index, name in enumerate(archive.getPageNameList()):
+            ext = os.path.splitext(name)
+            if len(ext) > 1:
+                extension = ext[1].lower()
+                if extension in COVER_EXTENSIONS:
+                    cover_data = archive.getPage(index)
+                    break
+    else:
+        cover_data = _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecutable)
     return _cover_processing(tmp_file_name, cover_data, extension)
 
 
@@ -139,13 +147,15 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
                 file_path=tmp_file_path,
                 extension=original_file_extension,
                 title=loadedMetadata.title or original_file_name,
-                author=" & ".join([credit["person"] for credit in loadedMetadata.credits if credit["role"] == "Writer"]) or u'Unknown',
+                author=" & ".join([credit["person"]
+                                   for credit in loadedMetadata.credits if credit["role"] == "Writer"]) or u'Unknown',
                 cover=_extractCover(tmp_file_path, original_file_extension, rarExecutable),
                 description=loadedMetadata.comments or "",
                 tags="",
                 series=loadedMetadata.series or "",
                 series_id=loadedMetadata.issue or "",
-                languages=loadedMetadata.language)
+                languages=loadedMetadata.language,
+                publisher="")
 
     return BookMeta(
         file_path=tmp_file_path,
@@ -157,4 +167,5 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
         tags="",
         series="",
         series_id="",
-        languages="")
+        languages="",
+        publisher="")

cps/config_sql.py

@@ -20,11 +20,18 @@
 from __future__ import division, print_function, unicode_literals
 import os
 import sys
+import json
 
-from sqlalchemy import exc, Column, String, Integer, SmallInteger, Boolean, BLOB, JSON
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, String, Integer, SmallInteger, Boolean, BLOB, JSON
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.sql.expression import text
+try:
+    # Compatibility with sqlalchemy 2.0
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 
-from . import constants, cli, logger, ub
+from . import constants, cli, logger
 
 
 log = logger.create()
@@ -34,7 +41,7 @@ class _Flask_Settings(_Base):
     __tablename__ = 'flask_settings'
 
     id = Column(Integer, primary_key=True)
-    flask_session_key = Column(BLOB, default="")
+    flask_session_key = Column(BLOB, default=b"")
 
     def __init__(self, key):
         self.flask_session_key = key
@@ -53,6 +60,8 @@ class _Settings(_Base):
     mail_password = Column(String, default='mypassword')
     mail_from = Column(String, default='automailer <mail@example.com>')
     mail_size = Column(Integer, default=25*1024*1024)
+    mail_server_type = Column(SmallInteger, default=0)
+    mail_gmail_token = Column(JSON, default={})
 
     config_calibre_dir = Column(String)
     config_port = Column(Integer, default=constants.DEFAULT_PORT)
@@ -65,7 +74,7 @@ class _Settings(_Base):
     config_random_books = Column(Integer, default=4)
     config_authors_max = Column(Integer, default=0)
     config_read_column = Column(Integer, default=0)
-    config_title_regex = Column(String, default=u'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+')
+    config_title_regex = Column(String, default=r'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+')
     config_mature_content_tags = Column(String, default='')
     config_theme = Column(Integer, default=0)
 
@@ -145,15 +154,16 @@ class _ConfigSQL(object):
         self.load()
 
         change = False
-        if self.config_converterpath == None:
+        if self.config_converterpath == None:  # pylint: disable=access-member-before-definition
             change = True
             self.config_converterpath = autodetect_calibre_binary()
 
-        if self.config_kepubifypath == None:
+        if self.config_kepubifypath == None:  # pylint: disable=access-member-before-definition
+
             change = True
             self.config_kepubifypath = autodetect_kepubify_binary()
 
-        if self.config_rarfile_location == None:
+        if self.config_rarfile_location == None:  # pylint: disable=access-member-before-definition
             change = True
             self.config_rarfile_location = autodetect_unrar_binary()
         if change:
@@ -180,8 +190,9 @@ class _ConfigSQL(object):
             return None
         return self.config_keyfile
 
-    def get_config_ipaddress(self):
-        return cli.ipadress or ""
+    @staticmethod
+    def get_config_ipaddress():
+        return cli.ip_address or ""
 
     def _has_role(self, role_flag):
         return constants.has_flag(self.config_default_role, role_flag)
@@ -239,18 +250,18 @@ class _ConfigSQL(object):
         return {k:v for k, v in self.__dict__.items() if k.startswith('mail_')}
 
     def get_mail_server_configured(self):
-        return not bool(self.mail_server == constants.DEFAULT_MAIL_SERVER)
+        return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0)
+                    or (self.mail_gmail_token != {} and self.mail_server_type == 1))
 
 
     def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None):
-        '''Possibly updates a field of this object.
+        """Possibly updates a field of this object.
         The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor.
 
         :returns: `True` if the field has changed value
-        '''
+        """
         new_value = dictionary.get(field, default)
         if new_value is None:
-            # log.debug("_ConfigSQL set_from_dictionary field '%s' not found", field)
             return False
 
         if field not in self.__dict__:
@@ -267,10 +278,17 @@ class _ConfigSQL(object):
         if current_value == new_value:
             return False
 
-        # log.debug("_ConfigSQL set_from_dictionary '%s' = %r (was %r)", field, new_value, current_value)
         setattr(self, field, new_value)
         return True
 
+    def toDict(self):
+        storage = {}
+        for k, v in self.__dict__.items():
+            if k[0] != '_' and not k.endswith("password") and not k.endswith("secret"):
+                storage[k] = v
+        return storage
+
+
     def load(self):
         '''Load all configuration values from the underlying storage.'''
         s = self._read_from_storage()  # type: _Settings
@@ -290,12 +308,20 @@ class _ConfigSQL(object):
                 have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
         constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')]
+        if os.environ.get('FLASK_DEBUG'):
+            logfile = logger.setup(logger.LOG_TO_STDOUT, logger.logging.DEBUG)
+        else:
+            # pylint: disable=access-member-before-definition
             logfile = logger.setup(self.config_logfile, self.config_log_level)
         if logfile != self.config_logfile:
             log.warning("Log path %s not valid, falling back to default", self.config_logfile)
             self.config_logfile = logfile
             self._session.merge(s)
+            try:
                 self._session.commit()
+            except OperationalError as e:
+                log.error('Database error: %s', e)
+                self._session.rollback()
 
     def save(self):
         '''Apply all configuration values to the underlying storage.'''
@@ -309,7 +335,11 @@ class _ConfigSQL(object):
 
         log.debug("_ConfigSQL updating storage")
         self._session.merge(s)
+        try:
             self._session.commit()
+        except OperationalError as e:
+            log.error('Database error: %s', e)
+            self._session.rollback()
         self.load()
 
     def invalidate(self, error=None):
@@ -328,7 +358,7 @@ def _migrate_table(session, orm_class):
         if column_name[0] != '_':
             try:
                 session.query(column).first()
-            except exc.OperationalError as err:
+            except OperationalError as err:
                 log.debug("%s: %s", column_name, err.args[0])
                 if column.default is not None:
                     if sys.version_info < (3, 0):
@@ -338,19 +368,29 @@ def _migrate_table(session, orm_class):
                     column_default = ""
                 else:
                     if isinstance(column.default.arg, bool):
-                        column_default = ("DEFAULT %r" % int(column.default.arg))
+                        column_default = "DEFAULT {}".format(int(column.default.arg))
                     else:
-                        column_default = ("DEFAULT %r" % column.default.arg)
-                alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__,
+                        column_default = "DEFAULT `{}`".format(column.default.arg)
+                if isinstance(column.type, JSON):
+                    column_type = "JSON"
+                else:
+                    column_type = column.type
+                alter_table = text("ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__,
                                                                         column_name,
-                                                                        column.type,
-                                                                        column_default)
+                                                                        column_type,
+                                                                        column_default))
                 log.debug(alter_table)
                 session.execute(alter_table)
                 changed = True
+            except json.decoder.JSONDecodeError as e:
+                log.error("Database corrupt column: {}".format(column_name))
+                log.debug(e)
 
     if changed:
+        try:
             session.commit()
+        except OperationalError:
+            session.rollback()
 
 
 def autodetect_calibre_binary():
@@ -403,12 +443,12 @@ def load_configuration(session):
         session.commit()
     conf = _ConfigSQL(session)
     # Migrate from global restrictions to user based restrictions
-    if bool(conf.config_default_show & constants.MATURE_CONTENT) and conf.config_denied_tags == "":
-        conf.config_denied_tags = conf.config_mature_content_tags
-        conf.save()
-        session.query(ub.User).filter(ub.User.mature_content != True). \
-            update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False)
-        session.commit()
+    #if bool(conf.config_default_show & constants.MATURE_CONTENT) and conf.config_denied_tags == "":
+    #    conf.config_denied_tags = conf.config_mature_content_tags
+    #    conf.save()
+    #    session.query(ub.User).filter(ub.User.mature_content != True). \
+    #        update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False)
+    #    session.commit()
     return conf
 
 def get_flask_session_key(session):

cps/constants.py

@@ -21,7 +21,11 @@ import sys
 import os
 from collections import namedtuple
 
-HOME_CONFIG = False
+# if installed via pip this variable is set to true (empty file with name .HOMEDIR present)
+HOME_CONFIG = os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), '.HOMEDIR'))
+
+#In executables updater is not available, so variable is set to False there
+UPDATER_AVAILABLE = True
 
 # Base dir is parent of current file, necessary if called from different folder
 if sys.version_info < (3, 0):
@@ -84,6 +88,26 @@ SIDEBAR_ARCHIVED        = 1 << 15
 SIDEBAR_DOWNLOAD        = 1 << 16
 SIDEBAR_LIST            = 1 << 17
 
+sidebar_settings = {
+                "detail_random": DETAIL_RANDOM,
+                "sidebar_language": SIDEBAR_LANGUAGE,
+                "sidebar_series": SIDEBAR_SERIES,
+                "sidebar_category": SIDEBAR_CATEGORY,
+                "sidebar_random": SIDEBAR_RANDOM,
+                "sidebar_author": SIDEBAR_AUTHOR,
+                "sidebar_best_rated": SIDEBAR_BEST_RATED,
+                "sidebar_read_and_unread": SIDEBAR_READ_AND_UNREAD,
+                "sidebar_recent": SIDEBAR_RECENT,
+                "sidebar_sorted": SIDEBAR_SORTED,
+                "sidebar_publisher": SIDEBAR_PUBLISHER,
+                "sidebar_rating": SIDEBAR_RATING,
+                "sidebar_format": SIDEBAR_FORMAT,
+                "sidebar_archived": SIDEBAR_ARCHIVED,
+                "sidebar_download": SIDEBAR_DOWNLOAD,
+                "sidebar_list": SIDEBAR_LIST,
+            }
+
+
 ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_ANONYMOUS
 ADMIN_USER_SIDEBAR      = (SIDEBAR_LIST << 1) - 1
 
@@ -102,7 +126,7 @@ LDAP_AUTH_SIMPLE         = 0
 
 DEFAULT_MAIL_SERVER = "mail.example.org"
 
-DEFAULT_PASSWORD    = "admin123"
+DEFAULT_PASSWORD    = "admin123"  # nosec
 DEFAULT_PORT        = 8083
 env_CALIBRE_PORT = os.environ.get("CALIBRE_PORT", DEFAULT_PORT)
 try:
@@ -128,9 +152,9 @@ def selected_roles(dictionary):
 
 # :rtype: BookMeta
 BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
-                                  'series_id, languages')
+                                  'series_id, languages, publisher')
 
-STABLE_VERSION = {'version': '0.6.10 Beta'}
+STABLE_VERSION = {'version': '0.6.12 Beta'}
 
 NIGHTLY_VERSION = {}
 NIGHTLY_VERSION[0] = '$Format:%H$'

cps/db.py

@@ -30,11 +30,17 @@ from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
 from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.orm.collections import InstrumentedList
-from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
+from sqlalchemy.ext.declarative import DeclarativeMeta
+from sqlalchemy.exc import OperationalError
+try:
+    # Compatibility with sqlalchemy 2.0
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.pool import StaticPool
-from flask_login import current_user
 from sqlalchemy.sql.expression import and_, true, false, text, func, or_
 from sqlalchemy.ext.associationproxy import association_proxy
+from flask_login import current_user
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
 from flask_babel import gettext as _
@@ -50,6 +56,8 @@ try:
 except ImportError:
     use_unidecode = False
 
+log = logger.create()
+
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
 
@@ -113,6 +121,8 @@ class Identifiers(Base):
             return u"Douban"
         elif format_type == "goodreads":
             return u"Goodreads"
+        elif format_type == "babelio":
+            return u"Babelio"        
         elif format_type == "google":
             return u"Google Books"
         elif format_type == "kobo":
@@ -140,6 +150,8 @@ class Identifiers(Base):
             return u"https://dx.doi.org/{0}".format(self.val)
         elif format_type == "goodreads":
             return u"https://www.goodreads.com/book/show/{0}".format(self.val)
+        elif format_type == "babelio":
+            return u"https://www.babelio.com/livres/titre/{0}".format(self.val)        
         elif format_type == "douban":
             return u"https://book.douban.com/subject/{0}".format(self.val)
         elif format_type == "google":
@@ -154,10 +166,8 @@ class Identifiers(Base):
             return u"https://portal.issn.org/resource/ISSN/{0}".format(self.val)
         elif format_type == "isfdb":
             return u"http://www.isfdb.org/cgi-bin/pl.cgi?{0}".format(self.val)
-        elif format_type == "url":
-            return u"{0}".format(self.val)
         else:
-            return u""
+            return u"{0}".format(self.val)
 
 
 class Comments(Base):
@@ -326,7 +336,6 @@ class Books(Base):
     has_cover = Column(Integer, default=0)
     uuid = Column(String)
     isbn = Column(String(collation='NOCASE'), default="")
-    # Iccn = Column(String(collation='NOCASE'), default="")
     flags = Column(Integer, nullable=False, default=1)
 
     authors = relationship('Authors', secondary=books_authors_link, backref='books')
@@ -384,14 +393,14 @@ class Custom_Columns(Base):
 
 class AlchemyEncoder(json.JSONEncoder):
 
-    def default(self, obj):
-        if isinstance(obj.__class__, DeclarativeMeta):
+    def default(self, o):
+        if isinstance(o.__class__, DeclarativeMeta):
             # an SQLAlchemy class
             fields = {}
-            for field in [x for x in dir(obj) if not x.startswith('_') and x != 'metadata']:
+            for field in [x for x in dir(o) if not x.startswith('_') and x != 'metadata' and x!="password"]:
                 if field == 'books':
                     continue
-                data = obj.__getattribute__(field)
+                data = o.__getattribute__(field)
                 try:
                     if isinstance(data, str):
                         data = data.replace("'", "\'")
@@ -402,18 +411,21 @@ class AlchemyEncoder(json.JSONEncoder):
                                 el.append(ele.get())
                             else:
                                 el.append(json.dumps(ele, cls=AlchemyEncoder))
+                        if field == 'authors':
+                            data = " & ".join(el)
+                        else:
                             data = ",".join(el)
                         if data == '[]':
                             data = ""
                     else:
                         json.dumps(data)
                     fields[field] = data
-                except:
+                except Exception:
                     fields[field] = ""
             # a json-encodable dict
             return fields
 
-        return json.JSONEncoder.default(self, obj)
+        return json.JSONEncoder.default(self, o)
 
 
 class CalibreDB():
@@ -425,54 +437,22 @@ class CalibreDB():
     # instances alive once they reach the end of their respective scopes
     instances = WeakSet()
 
-    def __init__(self):
+    def __init__(self, expire_on_commit=True):
         """ Initialize a new CalibreDB session
         """
         self.session = None
         if self._init:
-            self.initSession()
+            self.initSession(expire_on_commit)
 
         self.instances.add(self)
 
-
-    def initSession(self):
+    def initSession(self, expire_on_commit=True):
         self.session = self.session_factory()
+        self.session.expire_on_commit = expire_on_commit
         self.update_title_sort(self.config)
 
     @classmethod
-    def setup_db(cls, config, app_db_path):
-        cls.config = config
-        cls.dispose()
-
-        if not config.config_calibre_dir:
-            config.invalidate()
-            return False
-
-        dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
-        if not os.path.exists(dbpath):
-            config.invalidate()
-            return False
-
-        try:
-            cls.engine = create_engine('sqlite://',
-                                       echo=False,
-                                       isolation_level="SERIALIZABLE",
-                                       connect_args={'check_same_thread': False},
-                                       poolclass=StaticPool)
-            cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
-            cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
-
-            conn = cls.engine.connect()
-            # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
-        except Exception as e:
-            config.invalidate(e)
-            return False
-
-        config.db_configured = True
-
-        if not cc_classes:
-            cc = conn.execute("SELECT id, datatype FROM custom_columns")
-
+    def setup_db_cc_classes(self, cc):
         cc_ids = []
         books_custom_column_links = {}
         for row in cc:
@@ -538,6 +518,49 @@ class CalibreDB():
                                      secondary=books_custom_column_links[cc_id[0]],
                                      backref='books'))
 
+        return cc_classes
+
+    @classmethod
+    def setup_db(cls, config, app_db_path):
+        cls.config = config
+        cls.dispose()
+
+        # toDo: if db changed -> delete shelfs, delete download books, delete read boks, kobo sync??
+
+        if not config.config_calibre_dir:
+            config.invalidate()
+            return False
+
+        dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
+        if not os.path.exists(dbpath):
+            config.invalidate()
+            return False
+
+        try:
+            cls.engine = create_engine('sqlite://',
+                                       echo=False,
+                                       isolation_level="SERIALIZABLE",
+                                       connect_args={'check_same_thread': False},
+                                       poolclass=StaticPool)
+            with cls.engine.begin() as connection:
+                connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
+                connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
+
+            conn = cls.engine.connect()
+            # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
+        except Exception as ex:
+            config.invalidate(ex)
+            return False
+
+        config.db_configured = True
+
+        if not cc_classes:
+            try:
+                cc = conn.execute("SELECT id, datatype FROM custom_columns")
+                cls.setup_db_cc_classes(cc)
+            except OperationalError as e:
+                log.debug_or_exception(e)
+
         cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
                                                           bind=cls.engine))
@@ -557,8 +580,8 @@ class CalibreDB():
     def get_book_by_uuid(self, book_uuid):
         return self.session.query(Books).filter(Books.uuid == book_uuid).first()
 
-    def get_book_format(self, book_id, format):
-        return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == format).first()
+    def get_book_format(self, book_id, file_format):
+        return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == file_format).first()
 
     # Language and content filters for displaying in the UI
     def common_filters(self, allow_show_archived=False):
@@ -597,6 +620,22 @@ class CalibreDB():
         return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
                     pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
 
+    @staticmethod
+    def get_checkbox_sorted(inputlist, state, offset, limit, order):
+        outcome = list()
+        elementlist = {ele.id: ele for ele in inputlist}
+        for entry in state:
+            try:
+                outcome.append(elementlist[entry])
+            except KeyError:
+                pass
+            del elementlist[entry]
+        for entry in elementlist:
+            outcome.append(elementlist[entry])
+        if order == "asc":
+            outcome.reverse()
+        return outcome[offset:offset + limit]
+
     # Fill indexpage with all requested data from database
     def fill_indexpage(self, page, pagesize, database, db_filter, order, *join):
         return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join)
@@ -608,19 +647,29 @@ class CalibreDB():
             randm = self.session.query(Books) \
                 .filter(self.common_filters(allow_show_archived)) \
                 .order_by(func.random()) \
-                .limit(self.config.config_random_books)
+                .limit(self.config.config_random_books).all()
         else:
             randm = false()
         off = int(int(pagesize) * (page - 1))
-        query = self.session.query(database) \
-            .join(*join, isouter=True) \
-            .filter(db_filter) \
+        query = self.session.query(database)
+        if len(join) == 3:
+            query = query.outerjoin(join[0], join[1]).outerjoin(join[2])
+        elif len(join) == 2:
+            query = query.outerjoin(join[0], join[1])
+        elif len(join) == 1:
+            query = query.outerjoin(join[0])
+        query = query.filter(db_filter)\
             .filter(self.common_filters(allow_show_archived))
+        entries = list()
+        pagination = list()
+        try:
             pagination = Pagination(page, pagesize,
                                     len(query.all()))
             entries = query.order_by(*order).offset(off).limit(pagesize).all()
-        for book in entries:
-            book = self.order_authors(book)
+        except Exception as ex:
+            log.debug_or_exception(ex)
+        #for book in entries:
+        #    book = self.order_authors(book)
         return entries, randm, pagination
 
     # Orders all Authors in the list according to authors sort
@@ -660,23 +709,33 @@ class CalibreDB():
         return self.session.query(Books) \
             .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
 
-    # read search results from calibre-database and return it (function is used for feed and simple search
-    def get_search_results(self, term, offset=None, order=None, limit=None):
-        order = order or [Books.sort]
-        pagination = None
+    def search_query(self, term, *join):
         term.strip().lower()
         self.session.connection().connection.connection.create_function("lower", 1, lcase)
         q = list()
         authorterms = re.split("[, ]+", term)
         for authorterm in authorterms:
             q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
-        result = self.session.query(Books).filter(self.common_filters(True)).filter(
+        query = self.session.query(Books)
+        if len(join) == 3:
+            query = query.outerjoin(join[0], join[1]).outerjoin(join[2])
+        elif len(join) == 2:
+            query = query.outerjoin(join[0], join[1])
+        elif len(join) == 1:
+            query = query.outerjoin(join[0])
+        return query.filter(self.common_filters(True)).filter(
             or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")),
                 Books.series.any(func.lower(Series.name).ilike("%" + term + "%")),
                 Books.authors.any(and_(*q)),
                 Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")),
                 func.lower(Books.title).ilike("%" + term + "%")
-                )).order_by(*order).all()
+                ))
+
+    # read search results from calibre-database and return it (function is used for feed and simple search
+    def get_search_results(self, term, offset=None, order=None, limit=None, *join):
+        order = order or [Books.sort]
+        pagination = None
+        result = self.search_query(term, *join).order_by(*order).all()
         result_count = len(result)
         if offset != None and limit != None:
             offset = int(offset)
@@ -731,7 +790,7 @@ class CalibreDB():
             if old_session:
                 try:
                     old_session.close()
-                except:
+                except Exception:
                     pass
                 if old_session.bind:
                     try:
@@ -762,7 +821,7 @@ class CalibreDB():
 def lcase(s):
     try:
         return unidecode.unidecode(s.lower())
-    except Exception as e:
+    except Exception as ex:
         log = logger.create()
-        log.exception(e)
+        log.debug_or_exception(ex)
         return s.lower()

cps/debug_info.py

@@ -21,7 +21,12 @@ import shutil
 import glob
 import zipfile
 import json
-import io
+from io import BytesIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 import os
 
 from flask import send_file
@@ -32,11 +37,12 @@ from .about import collect_stats
 log = logger.create()
 
 def assemble_logfiles(file_name):
-    log_list = glob.glob(file_name + '*')
-    wfd = io.StringIO()
+    log_list = sorted(glob.glob(file_name + '*'), reverse=True)
+    wfd = StringIO()
     for f in log_list:
         with open(f, 'r') as fd:
             shutil.copyfileobj(fd, wfd)
+    wfd.seek(0)
     return send_file(wfd,
                      as_attachment=True,
                      attachment_filename=os.path.basename(file_name))
@@ -44,8 +50,12 @@ def assemble_logfiles(file_name):
 def send_debug():
     file_list = glob.glob(logger.get_logfile(config.config_logfile) + '*')
     file_list.extend(glob.glob(logger.get_accesslogfile(config.config_access_logfile) + '*'))
-    memory_zip = io.BytesIO()
+    for element in [logger.LOG_TO_STDOUT, logger.LOG_TO_STDERR]:
+        if element in file_list:
+            file_list.remove(element)
+    memory_zip = BytesIO()
     with zipfile.ZipFile(memory_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr('settings.txt', json.dumps(config.toDict()))
         zf.writestr('libs.txt', json.dumps(collect_stats()))
         for fp in file_list:
             zf.write(fp, os.path.basename(fp))

cps/editbooks.py

@@ -27,33 +27,50 @@ import json
 from shutil import copyfile
 from uuid import uuid4
 
+from babel import Locale as LC
+from babel.core import UnknownLocaleError
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
 from flask_babel import gettext as _
 from flask_login import current_user, login_required
-from sqlalchemy.exc import OperationalError
-
+from sqlalchemy.exc import OperationalError, IntegrityError
+from sqlite3 import OperationalError as sqliteOperationalError
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
 from . import config, get_locale, ub, db
 from . import calibre_db
 from .services.worker import WorkerThread
 from .tasks.upload import TaskUpload
-from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
+from .render_template import render_title_template
+from .usermanagement import login_required_if_no_ano
+
+try:
+    from functools import wraps
+except ImportError:
+    pass  # We're not using Python 3
 
 
 editbook = Blueprint('editbook', __name__)
 log = logger.create()
 
 
-# Modifies different Database objects, first check if elements have to be added to database, than check
-# if elements have to be deleted, because they are no longer used
-def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type):
-    # passing input_elements not as a list may lead to undesired results
-    if not isinstance(input_elements, list):
-        raise TypeError(str(input_elements) + " should be passed as a list")
-    changed = False
-    input_elements = [x for x in input_elements if x != '']
-    # we have all input element (authors, series, tags) names now
-    # 1. search for elements to remove
+def upload_required(f):
+    @wraps(f)
+    def inner(*args, **kwargs):
+        if current_user.role_upload() or current_user.role_admin():
+            return f(*args, **kwargs)
+        abort(403)
+
+    return inner
+
+def edit_required(f):
+    @wraps(f)
+    def inner(*args, **kwargs):
+        if current_user.role_edit() or current_user.role_admin():
+            return f(*args, **kwargs)
+        abort(403)
+
+    return inner
+
+def search_objects_remove(db_book_object, db_type, input_elements):
     del_elements = []
     for c_elements in db_book_object:
         found = False
@@ -71,7 +88,10 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
         # if the element was not found in the new list, add it to remove list
         if not found:
             del_elements.append(c_elements)
-    # 2. search for elements that need to be added
+    return del_elements
+
+
+def search_objects_add(db_book_object, db_type, input_elements):
     add_elements = []
     for inp_element in input_elements:
         found = False
@@ -87,15 +107,21 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
                 break
         if not found:
             add_elements.append(inp_element)
-    # if there are elements to remove, we remove them now
+    return add_elements
+
+
+def remove_objects(db_book_object, db_session, del_elements):
+    changed = False
     if len(del_elements) > 0:
         for del_element in del_elements:
             db_book_object.remove(del_element)
             changed = True
             if len(del_element.books) == 0:
                 db_session.delete(del_element)
-    # if there are elements to add, we add them now!
-    if len(add_elements) > 0:
+    return changed
+
+def add_objects(db_book_object, db_object, db_session, db_type, add_elements):
+    changed = False
     if db_type == 'languages':
         db_filter = db_object.lang_code
     elif db_type == 'custom':
@@ -122,9 +148,18 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
             db_session.add(new_element)
             db_book_object.append(new_element)
         else:
+            db_element = create_objects_for_addition(db_element, add_element, db_type)
+            changed = True
+            # add element to book
+            changed = True
+            db_book_object.append(db_element)
+    return changed
+
+
+def create_objects_for_addition(db_element, add_element, db_type):
     if db_type == 'custom':
         if db_element.value != add_element:
-                        new_element.value = add_element
+            db_element.value = add_element  # ToDo: Before new_element, but this is not plausible
     elif db_type == 'languages':
         if db_element.lang_code != add_element:
             db_element.lang_code = add_element
@@ -142,9 +177,26 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
             db_element.sort = None
     elif db_element.name != add_element:
         db_element.name = add_element
-                # add element to book
-                changed = True
-                db_book_object.append(db_element)
+    return db_element
+
+
+# Modifies different Database objects, first check if elements if elements have to be deleted,
+# because they are no longer used, than check if elements have to be added to database
+def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type):
+    # passing input_elements not as a list may lead to undesired results
+    if not isinstance(input_elements, list):
+        raise TypeError(str(input_elements) + " should be passed as a list")
+    input_elements = [x for x in input_elements if x != '']
+    # we have all input element (authors, series, tags) names now
+    # 1. search for elements to remove
+    del_elements = search_objects_remove(db_book_object, db_type, input_elements)
+    # 2. search for elements that need to be added
+    add_elements = search_objects_add(db_book_object, db_type, input_elements)
+    # if there are elements to remove, we remove them now
+    changed = remove_objects(db_book_object, db_session, del_elements)
+    # if there are elements to add, we add them now!
+    if len(add_elements) > 0:
+        changed |= add_objects(db_book_object, db_object, db_session, db_type, add_elements)
     return changed
 
 
@@ -177,45 +229,22 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session):
 @editbook.route("/ajax/delete/<int:book_id>")
 @login_required
 def delete_book_from_details(book_id):
-    return Response(delete_book(book_id,"", True), mimetype='application/json')
+    return Response(delete_book(book_id, "", True), mimetype='application/json')
 
 
 @editbook.route("/delete/<int:book_id>", defaults={'book_format': ""})
 @editbook.route("/delete/<int:book_id>/<string:book_format>")
 @login_required
 def delete_book_ajax(book_id, book_format):
-    return delete_book(book_id,book_format, False)
+    return delete_book(book_id, book_format, False)
 
-def delete_book(book_id, book_format, jsonResponse):
-    warning = {}
-    if current_user.role_delete_books():
-        book = calibre_db.get_book(book_id)
-        if book:
-            try:
-                result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
-                if not result:
-                    if jsonResponse:
-                        return json.dumps({"location": url_for("editbook.edit_book"),
-                                           "type": "alert",
-                                           "format": "",
-                                           "error": error}),
-                    else:
-                        flash(error, category="error")
-                        return redirect(url_for('editbook.edit_book', book_id=book_id))
-                if error:
-                    if jsonResponse:
-                        warning = {"location": url_for("editbook.edit_book"),
-                                                "type": "warning",
-                                                "format": "",
-                                                "error": error}
-                    else:
-                        flash(error, category="warning")
-                if not book_format:
+
+def delete_whole_book(book_id, book):
     # delete book from Shelfs, Downloads, Read list
     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
     ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete()
     ub.delete_download(book_id)
-                    ub.session.commit()
+    ub.session_commit()
 
     # check if only this book links to:
     # author, language, series, tags, custom columns
@@ -225,7 +254,7 @@ def delete_book(book_id, book_format, jsonResponse):
     modify_database_object([u''], book.languages, db.Languages, calibre_db.session, 'languages')
     modify_database_object([u''], book.publishers, db.Publishers, calibre_db.session, 'publishers')
 
-                    cc = calibre_db.session.query(db.Custom_Columns).\
+    cc = calibre_db.session.query(db.Custom_Columns). \
         filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     for c in cc:
         cc_string = "custom_column_" + str(c.id)
@@ -254,16 +283,9 @@ def delete_book(book_id, book_format, jsonResponse):
             modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id],
                                    calibre_db.session, 'custom')
     calibre_db.session.query(db.Books).filter(db.Books.id == book_id).delete()
-                else:
-                    calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\
-                        filter(db.Data.format == book_format).delete()
-                calibre_db.session.commit()
-            except Exception as e:
-                log.exception(e)
-                calibre_db.session.rollback()
-        else:
-            # book not found
-            log.error('Book with id "%s" could not be deleted: not found', book_id)
+
+
+def render_delete_book_result(book_format, jsonResponse, warning, book_id):
     if book_format:
         if jsonResponse:
             return json.dumps([warning, {"location": url_for("editbook.edit_book", book_id=book_id),
@@ -284,10 +306,57 @@ def delete_book(book_id, book_format, jsonResponse):
             return redirect(url_for('web.index'))
 
 
+def delete_book(book_id, book_format, jsonResponse):
+    warning = {}
+    if current_user.role_delete_books():
+        book = calibre_db.get_book(book_id)
+        if book:
+            try:
+                result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
+                if not result:
+                    if jsonResponse:
+                        return json.dumps([{"location": url_for("editbook.edit_book", book_id=book_id),
+                                           "type": "danger",
+                                           "format": "",
+                                           "message": error}])
+                    else:
+                        flash(error, category="error")
+                        return redirect(url_for('editbook.edit_book', book_id=book_id))
+                if error:
+                    if jsonResponse:
+                        warning = {"location": url_for("editbook.edit_book", book_id=book_id),
+                                                "type": "warning",
+                                                "format": "",
+                                                "message": error}
+                    else:
+                        flash(error, category="warning")
+                if not book_format:
+                    delete_whole_book(book_id, book)
+                else:
+                    calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\
+                        filter(db.Data.format == book_format).delete()
+                calibre_db.session.commit()
+            except Exception as ex:
+                log.debug_or_exception(ex)
+                calibre_db.session.rollback()
+                if jsonResponse:
+                    return json.dumps([{"location": url_for("editbook.edit_book", book_id=book_id),
+                                        "type": "danger",
+                                        "format": "",
+                                        "message": ex}])
+                else:
+                    flash(str(ex), category="error")
+                    return redirect(url_for('editbook.edit_book', book_id=book_id))
+
+        else:
+            # book not found
+            log.error('Book with id "%s" could not be deleted: not found', book_id)
+    return render_delete_book_result(book_format, jsonResponse, warning, book_id)
+
+
 def render_edit_book(book_id):
-    calibre_db.update_title_sort(config)
     cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
-    book = calibre_db.get_filtered_book(book_id)
+    book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     if not book:
         flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))
@@ -389,7 +458,7 @@ def edit_book_comments(comments, book):
     return modif_date
 
 
-def edit_book_languages(languages, book, upload=False):
+def edit_book_languages(languages, book, upload=False, invalid=None):
     input_languages = languages.split(',')
     unknown_languages = []
     if not upload:
@@ -398,6 +467,9 @@ def edit_book_languages(languages, book, upload=False):
         input_l = isoLanguages.get_valid_language_codes(get_locale(), input_languages, unknown_languages)
     for l in unknown_languages:
         log.error('%s is not a valid language', l)
+        if isinstance(invalid, list):
+            invalid.append(l)
+        else:
             flash(_(u"%(langname)s is not a valid language", langname=l), category="warning")
     # ToDo: Not working correct
     if upload and len(input_l) == 1:
@@ -411,10 +483,10 @@ def edit_book_languages(languages, book, upload=False):
     return modify_database_object(input_l, book.languages, db.Languages, calibre_db.session, 'languages')
 
 
-def edit_book_publisher(to_save, book):
+def edit_book_publisher(publishers, book):
     changed = False
-    if to_save["publisher"]:
-        publisher = to_save["publisher"].rstrip().strip()
+    if publishers:        
+        publisher = publishers.rstrip().strip()
         if len(book.publishers) == 0 or (len(book.publishers) > 0 and publisher != book.publishers[0].name):
             changed |= modify_database_object([publisher], book.publishers, db.Publishers, calibre_db.session,
                                               'publisher')
@@ -423,18 +495,8 @@ def edit_book_publisher(to_save, book):
     return changed
 
 
-def edit_cc_data(book_id, book, to_save):
+def edit_cc_data_number(book_id, book, c, to_save, cc_db_value, cc_string):
     changed = False
-    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
-    for c in cc:
-        cc_string = "custom_column_" + str(c.id)
-        if not c.is_multiple:
-            if len(getattr(book, cc_string)) > 0:
-                cc_db_value = getattr(book, cc_string)[0].value
-            else:
-                cc_db_value = None
-            if to_save[cc_string].strip():
-                if c.datatype == 'int' or c.datatype == 'bool' or c.datatype == 'float':
     if to_save[cc_string] == 'None':
         to_save[cc_string] = None
     elif c.datatype == 'bool':
@@ -455,8 +517,11 @@ def edit_cc_data(book_id, book, to_save):
             new_cc = cc_class(value=to_save[cc_string], book=book_id)
             calibre_db.session.add(new_cc)
             changed = True
+    return changed, to_save
 
-                else:
+
+def edit_cc_data_string(book, c, to_save, cc_db_value, cc_string):
+    changed = False
     if c.datatype == 'rating':
         to_save[cc_string] = str(int(float(to_save[cc_string]) * 2))
     if to_save[cc_string].strip() != cc_db_value:
@@ -480,6 +545,24 @@ def edit_cc_data(book_id, book, to_save):
                 cc_class.value == to_save[cc_string].strip()).first()
         # add cc value to book
         getattr(book, cc_string).append(new_cc)
+    return changed, to_save
+
+
+def edit_cc_data(book_id, book, to_save):
+    changed = False
+    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    for c in cc:
+        cc_string = "custom_column_" + str(c.id)
+        if not c.is_multiple:
+            if len(getattr(book, cc_string)) > 0:
+                cc_db_value = getattr(book, cc_string)[0].value
+            else:
+                cc_db_value = None
+            if to_save[cc_string].strip():
+                if c.datatype == 'int' or c.datatype == 'bool' or c.datatype == 'float':
+                    changed, to_save = edit_cc_data_number(book_id, book, c, to_save, cc_db_value, cc_string)
+                else:
+                    changed, to_save = edit_cc_data_string(book, c, to_save, cc_db_value, cc_string)
             else:
                 if cc_db_value is not None:
                     # remove old cc_val
@@ -545,7 +628,7 @@ def upload_single_file(request, book, book_id):
                     calibre_db.session.add(db_format)
                     calibre_db.session.commit()
                     calibre_db.update_title_sort(config)
-                except OperationalError as e:
+                except (OperationalError, IntegrityError) as e:
                     calibre_db.session.rollback()
                     log.error('Database error: %s', e)
                     flash(_(u"Database error: %(error)s.", error=e), category="error")
@@ -553,7 +636,7 @@ def upload_single_file(request, book, book_id):
 
             # Queue uploader info
             uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
-            WorkerThread.add(current_user.nickname, TaskUpload(
+            WorkerThread.add(current_user.name, TaskUpload(
                 "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>"))
 
             return uploader.process(
@@ -577,17 +660,63 @@ def upload_cover(request, book):
     return None
 
 
+def handle_title_on_edit(book, book_title):
+    # handle book title
+    book_title = book_title.rstrip().strip()
+    if book.title != book_title:
+        if book_title == '':
+            book_title = _(u'Unknown')
+        book.title = book_title
+        return True
+    return False
+
+
+def handle_author_on_edit(book, author_name, update_stored=True):
+    # handle author(s)
+    input_authors = author_name.split('&')
+    input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
+    # Remove duplicates in authors list
+    input_authors = helper.uniq(input_authors)
+    # we have all author names now
+    if input_authors == ['']:
+        input_authors = [_(u'Unknown')]  # prevent empty Author
+
+    change = modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
+
+    # Search for each author if author is in database, if not, author name and sorted author name is generated new
+    # everything then is assembled for sorted author field in database
+    sort_authors_list = list()
+    for inp in input_authors:
+        stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+        if not stored_author:
+            stored_author = helper.get_sorted_author(inp)
+        else:
+            stored_author = stored_author.sort
+        sort_authors_list.append(helper.get_sorted_author(stored_author))
+    sort_authors = ' & '.join(sort_authors_list)
+    if book.author_sort != sort_authors and update_stored:
+        book.author_sort = sort_authors
+        change = True
+    return input_authors, change
+
+
 @editbook.route("/admin/book/<int:book_id>", methods=['GET', 'POST'])
 @login_required_if_no_ano
 @edit_required
 def edit_book(book_id):
     modif_date = False
+
+    # create the function for sorting...
+    try:
+        calibre_db.update_title_sort(config)
+    except sqliteOperationalError as e:
+        log.debug_or_exception(e)
+        calibre_db.session.rollback()
+
     # Show form
     if request.method != 'POST':
         return render_edit_book(book_id)
 
-    # create the function for sorting...
-    calibre_db.update_title_sort(config)
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
 
     # Book not found
@@ -605,41 +734,14 @@ def edit_book(book_id):
         # Update book
         edited_books_id = None
 
-        #handle book title
-        if book.title != to_save["book_title"].rstrip().strip():
-            if to_save["book_title"] == '':
-                to_save["book_title"] = _(u'Unknown')
-            book.title = to_save["book_title"].rstrip().strip()
+        # handle book title
+        title_change = handle_title_on_edit(book, to_save["book_title"])
+
+        input_authors, authorchange = handle_author_on_edit(book, to_save["author_name"])
+        if authorchange or title_change:
             edited_books_id = book.id
             modif_date = True
 
-        # handle author(s)
-        input_authors = to_save["author_name"].split('&')
-        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
-        # Remove duplicates in authors list
-        input_authors = helper.uniq(input_authors)
-        # we have all author names now
-        if input_authors == ['']:
-            input_authors = [_(u'Unknown')]  # prevent empty Author
-
-        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
-
-        # Search for each author if author is in database, if not, authorname and sorted authorname is generated new
-        # everything then is assembled for sorted author field in database
-        sort_authors_list = list()
-        for inp in input_authors:
-            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
-            if not stored_author:
-                stored_author = helper.get_sorted_author(inp)
-            else:
-                stored_author = stored_author.sort
-            sort_authors_list.append(helper.get_sorted_author(stored_author))
-        sort_authors = ' & '.join(sort_authors_list)
-        if book.author_sort != sort_authors:
-            edited_books_id = book.id
-            book.author_sort = sort_authors
-            modif_date = True
-
         if config.config_use_google_drive:
             gdriveutils.updateGdriveCalibreFromLocal()
 
@@ -664,10 +766,8 @@ def edit_book(book_id):
 
             # Add default series_index to book
             modif_date |= edit_book_series_index(to_save["series_index"], book)
-
             # Handle book comments/description
             modif_date |= edit_book_comments(to_save["description"], book)
-
             # Handle identifiers
             input_identifiers = identifier_list(to_save, book)
             modification, warning = modify_identifiers(input_identifiers, book.identifiers, calibre_db.session)
@@ -676,9 +776,16 @@ def edit_book(book_id):
             modif_date |= modification
             # Handle book tags
             modif_date |= edit_book_tags(to_save['tags'], book)
-
             # Handle book series
             modif_date |= edit_book_series(to_save["series"], book)
+            # handle book publisher
+            modif_date |= edit_book_publisher(to_save['publisher'], book)
+            # handle book languages
+            modif_date |= edit_book_languages(to_save['languages'], book)
+            # handle book ratings
+            modif_date |= edit_book_ratings(to_save, book)
+            # handle cc data
+            modif_date |= edit_cc_data(book_id, book, to_save)
 
             if to_save["pubdate"]:
                 try:
@@ -688,18 +795,6 @@ def edit_book(book_id):
             else:
                 book.pubdate = db.Books.DEFAULT_PUBDATE
 
-            # handle book publisher
-            modif_date |= edit_book_publisher(to_save, book)
-
-            # handle book languages
-            modif_date |= edit_book_languages(to_save['languages'], book)
-
-            # handle book ratings
-            modif_date |= edit_book_ratings(to_save, book)
-
-            # handle cc data
-            modif_date |= edit_cc_data(book_id, book, to_save)
-
             if modif_date:
                 book.last_modified = datetime.utcnow()
             calibre_db.session.merge(book)
@@ -715,8 +810,8 @@ def edit_book(book_id):
             calibre_db.session.rollback()
             flash(error, category="error")
             return render_edit_book(book_id)
-    except Exception as e:
-        log.exception(e)
+    except Exception as ex:
+        log.debug_or_exception(ex)
         calibre_db.session.rollback()
         flash(_("Error editing book, please check logfile for details"), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
@@ -735,6 +830,7 @@ def merge_metadata(to_save, meta):
     to_save["description"] = to_save["description"] or Markup(
         getattr(meta, 'description', '')).unescape()
 
+
 def identifier_list(to_save, book):
     """Generate a list of Identifiers from form information"""
     id_type_prefix = 'identifier-type-'
@@ -749,43 +845,8 @@ def identifier_list(to_save, book):
         result.append(db.Identifiers(to_save[val_key], type_value, book.id))
     return result
 
-@editbook.route("/upload", methods=["GET", "POST"])
-@login_required_if_no_ano
-@upload_required
-def upload():
-    if not config.config_uploading:
-        abort(404)
-    if request.method == 'POST' and 'btn-upload' in request.files:
-        for requested_file in request.files.getlist("btn-upload"):
-            try:
-                modif_date = False
-                # create the function for sorting...
-                calibre_db.update_title_sort(config)
-                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
-
-                # check if file extension is correct
-                if '.' in requested_file.filename:
-                    file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
-                    if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
-                        flash(
-                            _("File extension '%(ext)s' is not allowed to be uploaded to this server",
-                              ext=file_ext), category="error")
-                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-                else:
-                    flash(_('File to be uploaded must have an extension'), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-
-                # extract metadata from file
-                try:
-                    meta = uploader.upload(requested_file, config.config_rarfile_location)
-                except (IOError, OSError):
-                    log.error("File %s could not saved to temp dir", requested_file.filename)
-                    flash(_(u"File %(filename)s could not saved to temp dir",
-                            filename= requested_file.filename), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-                title = meta.title
-                authr = meta.author
 
+def prepare_authors_on_upload(title, authr):
     if title != _(u'Unknown') and authr != _(u'Unknown'):
         entry = calibre_db.check_exists_book(authr, title)
         if entry:
@@ -804,7 +865,7 @@ def upload():
     if input_authors == ['']:
         input_authors = [_(u'Unknown')]  # prevent empty Author
 
-                sort_authors_list=list()
+    sort_authors_list = list()
     db_author = None
     for inp in input_authors:
         stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
@@ -820,12 +881,20 @@ def upload():
             sort_author = stored_author.sort
         sort_authors_list.append(sort_author)
     sort_authors = ' & '.join(sort_authors_list)
+    return sort_authors, input_authors, db_author
+
+
+def create_book_on_upload(modif_date, meta):
+    title = meta.title
+    authr = meta.author
+    sort_authors, input_authors, db_author = prepare_authors_on_upload(title, authr)
 
     title_dir = helper.get_valid_filename(title)
     author_dir = helper.get_valid_filename(db_author.name)
 
     # combine path and normalize path from windows systems
     path = os.path.join(author_dir, title_dir).replace('\\', '/')
+
     # Calibre adds books with utc as timezone
     db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
                        '1', datetime.utcnow(), path, meta.cover, db_author, [], "")
@@ -842,6 +911,9 @@ def upload():
     # handle tags
     modif_date |= edit_book_tags(meta.tags, db_book)
 
+    # handle publisher
+    modif_date |= edit_book_publisher(meta.publisher, db_book)
+
     # handle series
     modif_date |= edit_book_series(meta.series, db_book)
 
@@ -853,19 +925,33 @@ def upload():
 
     # flush content, get db_book.id available
     calibre_db.session.flush()
+    return db_book, input_authors, title_dir
 
-                # Comments needs book id therfore only possible after flush
-                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
+def file_handling_on_upload(requested_file):
+    # check if file extension is correct
+    if '.' in requested_file.filename:
+        file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
+        if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
+            flash(
+                _("File extension '%(ext)s' is not allowed to be uploaded to this server",
+                  ext=file_ext), category="error")
+            return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+    else:
+        flash(_('File to be uploaded must have an extension'), category="error")
+        return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
-                book_id = db_book.id
-                title = db_book.title
+    # extract metadata from file
+    try:
+        meta = uploader.upload(requested_file, config.config_rarfile_location)
+    except (IOError, OSError):
+        log.error("File %s could not saved to temp dir", requested_file.filename)
+        flash(_(u"File %(filename)s could not saved to temp dir",
+                filename=requested_file.filename), category="error")
+        return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+    return meta, None
 
-                error = helper.update_dir_structure_file(book_id,
-                                                   config.config_calibre_dir,
-                                                   input_authors[0],
-                                                   meta.file_path,
-                                                   title_dir + meta.extension)
 
+def move_coverfile(meta, db_book):
     # move cover to final directory, including book id
     if meta.cover:
         coverfile = meta.cover
@@ -882,6 +968,41 @@ def upload():
                 error=e),
               category="error")
 
+
+@editbook.route("/upload", methods=["GET", "POST"])
+@login_required_if_no_ano
+@upload_required
+def upload():
+    if not config.config_uploading:
+        abort(404)
+    if request.method == 'POST' and 'btn-upload' in request.files:
+        for requested_file in request.files.getlist("btn-upload"):
+            try:
+                modif_date = False
+                # create the function for sorting...
+                calibre_db.update_title_sort(config)
+                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
+
+                meta, error = file_handling_on_upload(requested_file)
+                if error:
+                    return error
+
+                db_book, input_authors, title_dir = create_book_on_upload(modif_date, meta)
+
+                # Comments needs book id therefore only possible after flush
+                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
+
+                book_id = db_book.id
+                title = db_book.title
+
+                error = helper.update_dir_structure_file(book_id,
+                                                   config.config_calibre_dir,
+                                                   input_authors[0],
+                                                   meta.file_path,
+                                                   title_dir + meta.extension)
+
+                move_coverfile(meta, db_book)
+
                 # save data to database, reread data
                 calibre_db.session.commit()
 
@@ -890,7 +1011,7 @@ def upload():
                 if error:
                     flash(error, category="error")
                 uploadText=_(u"File %(file)s uploaded", file=title)
-                WorkerThread.add(current_user.nickname, TaskUpload(
+                WorkerThread.add(current_user.name, TaskUpload(
                     "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>"))
 
                 if len(request.files.getlist("btn-upload")) < 2:
@@ -900,7 +1021,7 @@ def upload():
                     else:
                         resp = {"location": url_for('web.show_book', book_id=book_id)}
                         return Response(json.dumps(resp), mimetype='application/json')
-            except OperationalError as e:
+            except (OperationalError, IntegrityError) as e:
                 calibre_db.session.rollback()
                 log.error("Database error: %s", e)
                 flash(_(u"Database error: %(error)s.", error=e), category="error")
@@ -920,7 +1041,7 @@ def convert_bookformat(book_id):
 
     log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to)
     rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(),
-                                     book_format_to.upper(), current_user.nickname)
+                                     book_format_to.upper(), current_user.name)
 
     if rtn is None:
         flash(_(u"Book successfully queued for converting to %(book_format)s",
@@ -930,61 +1051,89 @@ def convert_bookformat(book_id):
         flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
     return redirect(url_for('editbook.edit_book', book_id=book_id))
 
+
 @editbook.route("/ajax/editbooks/<param>", methods=['POST'])
 @login_required_if_no_ano
 @edit_required
 def edit_list_book(param):
     vals = request.form.to_dict()
     book = calibre_db.get_book(vals['pk'])
+    ret = ""
     if param =='series_index':
         edit_book_series_index(vals['value'], book)
+        ret = Response(json.dumps({'success': True, 'newValue': book.series_index}), mimetype='application/json')
     elif param =='tags':
         edit_book_tags(vals['value'], book)
+        ret = Response(json.dumps({'success': True, 'newValue': ', '.join([tag.name for tag in book.tags])}),
+                       mimetype='application/json')
     elif param =='series':
         edit_book_series(vals['value'], book)
+        ret = Response(json.dumps({'success': True, 'newValue':  ', '.join([serie.name for serie in book.series])}),
+                       mimetype='application/json')
     elif param =='publishers':
-        vals['publisher'] = vals['value']
-        edit_book_publisher(vals, book)
+        edit_book_publisher(vals['value'], book)
+        ret =  Response(json.dumps({'success': True,
+                                    'newValue': ', '.join([publisher.name for publisher in book.publishers])}),
+                       mimetype='application/json')
     elif param =='languages':
-        edit_book_languages(vals['value'], book)
+        invalid = list()
+        edit_book_languages(vals['value'], book, invalid=invalid)
+        if invalid:
+            ret = Response(json.dumps({'success': False,
+                                       'msg': 'Invalid languages in request: {}'.format(','.join(invalid))}),
+                           mimetype='application/json')
+        else:
+            lang_names = list()
+            for lang in book.languages:
+                try:
+                    lang_names.append(LC.parse(lang.lang_code).get_language_name(get_locale()))
+                except UnknownLocaleError:
+                    lang_names.append(_(isoLanguages.get(part3=lang.lang_code).name))
+            ret =  Response(json.dumps({'success': True, 'newValue':  ', '.join(lang_names)}),
+                            mimetype='application/json')
     elif param =='author_sort':
         book.author_sort = vals['value']
-    elif param =='title':
-        book.title = vals['value']
+        ret = Response(json.dumps({'success': True, 'newValue':  book.author_sort}),
+                       mimetype='application/json')
+    elif param == 'title':
+        sort = book.sort
+        handle_title_on_edit(book, vals.get('value', ""))
         helper.update_dir_stucture(book.id, config.config_calibre_dir)
+        ret = Response(json.dumps({'success': True, 'newValue':  book.title}),
+                       mimetype='application/json')
     elif param =='sort':
         book.sort = vals['value']
-    # ToDo: edit books
+        ret = Response(json.dumps({'success': True, 'newValue':  book.sort}),
+                       mimetype='application/json')
     elif param =='authors':
-        input_authors = vals['value'].split('&')
-        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
-        modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
-        sort_authors_list = list()
-        for inp in input_authors:
-            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
-            if not stored_author:
-                stored_author = helper.get_sorted_author(inp)
-            else:
-                stored_author = stored_author.sort
-            sort_authors_list.append(helper.get_sorted_author(stored_author))
-        sort_authors = ' & '.join(sort_authors_list)
-        if book.author_sort != sort_authors:
-            book.author_sort = sort_authors
+        input_authors, __ = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
         helper.update_dir_stucture(book.id, config.config_calibre_dir, input_authors[0])
+        ret = Response(json.dumps({'success': True,
+                                   'newValue':  ' & '.join([author.replace('|',',') for author in input_authors])}),
+                       mimetype='application/json')
     book.last_modified = datetime.utcnow()
     calibre_db.session.commit()
-    return ""
+    # revert change for sort if automatic fields link is deactivated
+    if param == 'title' and vals.get('checkT') == "false":
+        book.sort = sort
+        calibre_db.session.commit()
+    return ret
+
 
 @editbook.route("/ajax/sort_value/<field>/<int:bookid>")
 @login_required
 def get_sorted_entry(field, bookid):
-    if field == 'title' or field == 'authors':
+    if field in ['title', 'authors', 'sort', 'author_sort']:
         book = calibre_db.get_filtered_book(bookid)
         if book:
             if field == 'title':
                 return json.dumps({'sort': book.sort})
             elif field == 'authors':
                 return json.dumps({'author_sort': book.author_sort})
+            if field == 'sort':
+                return json.dumps({'sort': book.title})
+            if field == 'author_sort':
+                return json.dumps({'author_sort': book.author})
     return ""
 
 
@@ -1036,6 +1185,6 @@ def merge_list_book():
                                                         element.format,
                                                         element.uncompressed_size,
                                                         to_name))
-                    delete_book(from_book.id,"", True) # json_resp =
+                    delete_book(from_book.id,"", True)
                     return json.dumps({'success': True})
     return ""

cps/epub.py

@@ -87,18 +87,29 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     lang = epub_metadata['language'].split('-', 1)[0].lower()
     epub_metadata['language'] = isoLanguages.get_lang3(lang)
 
-    series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns)
-    if len(series) > 0:
-        epub_metadata['series'] = series[0]
-    else:
-        epub_metadata['series'] = ''
+    epub_metadata = parse_epbub_series(ns, tree, epub_metadata)
 
-    series_id = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series_index']/@content", namespaces=ns)
-    if len(series_id) > 0:
-        epub_metadata['series_id'] = series_id[0]
-    else:
-        epub_metadata['series_id'] = '1'
+    coverfile = parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path)
 
+    if not epub_metadata['title']:
+        title = original_file_name
+    else:
+        title = epub_metadata['title']
+
+    return BookMeta(
+        file_path=tmp_file_path,
+        extension=original_file_extension,
+        title=title.encode('utf-8').decode('utf-8'),
+        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
+        cover=coverfile,
+        description=epub_metadata['description'],
+        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
+        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
+        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
+        languages=epub_metadata['language'],
+        publisher="")
+
+def parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path):
     coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
     coverfile = None
     if len(coversection) > 0:
@@ -126,20 +137,18 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
                     coverfile = extractCover(epubZip, filename, "", tmp_file_path)
             else:
                 coverfile = extractCover(epubZip, coversection[0], coverpath, tmp_file_path)
+    return coverfile
 
-    if not epub_metadata['title']:
-        title = original_file_name
+def parse_epbub_series(ns, tree, epub_metadata):
+    series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns)
+    if len(series) > 0:
+        epub_metadata['series'] = series[0]
     else:
-        title = epub_metadata['title']
+        epub_metadata['series'] = ''
 
-    return BookMeta(
-        file_path=tmp_file_path,
-        extension=original_file_extension,
-        title=title.encode('utf-8').decode('utf-8'),
-        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
-        cover=coverfile,
-        description=epub_metadata['description'],
-        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
-        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
-        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
-        languages=epub_metadata['language'])
+    series_id = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series_index']/@content", namespaces=ns)
+    if len(series_id) > 0:
+        epub_metadata['series_id'] = series_id[0]
+    else:
+        epub_metadata['series_id'] = '1'
+    return epub_metadata

cps/error_handler.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2020 OzzieIsaacs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import traceback
+from flask import render_template
+from werkzeug.exceptions import default_exceptions
+try:
+    from werkzeug.exceptions import FailedDependency
+except ImportError:
+    from werkzeug.exceptions import UnprocessableEntity as FailedDependency
+
+from . import config, app, logger, services
+
+
+log = logger.create()
+
+# custom error page
+def error_http(error):
+    return render_template('http_error.html',
+                           error_code="Error {0}".format(error.code),
+                           error_name=error.name,
+                           issue=False,
+                           instance=config.config_calibre_web_title
+                           ), error.code
+
+
+def internal_error(error):
+    return render_template('http_error.html',
+                           error_code="Internal Server Error",
+                           error_name=str(error),
+                           issue=True,
+                           error_stack=traceback.format_exc().split("\n"),
+                           instance=config.config_calibre_web_title
+                           ), 500
+
+def init_errorhandler():
+    # http error handling
+    for ex in default_exceptions:
+        if ex < 500:
+            app.register_error_handler(ex, error_http)
+        elif ex == 500:
+            app.register_error_handler(ex, internal_error)
+
+
+    if services.ldap:
+        # Only way of catching the LDAPException upon logging in with LDAP server down
+        @app.errorhandler(services.ldap.LDAPException)
+        # pylint: disable=unused-variable
+        def handle_exception(e):
+            log.debug('LDAP server not accessible while trying to login to opds feed')
+            return error_http(FailedDependency())
+

cps/fb2.py

@@ -30,51 +30,52 @@ def get_fb2_info(tmp_file_path, original_file_extension):
     }
 
     fb2_file = open(tmp_file_path)
-    tree = etree.fromstring(fb2_file.read())
+    tree = etree.fromstring(fb2_file.read().encode())
 
     authors = tree.xpath('/fb:FictionBook/fb:description/fb:title-info/fb:author', namespaces=ns)
 
     def get_author(element):
         last_name = element.xpath('fb:last-name/text()', namespaces=ns)
         if len(last_name):
-            last_name = last_name[0].encode('utf-8')
+            last_name = last_name[0]
         else:
             last_name = u''
         middle_name = element.xpath('fb:middle-name/text()', namespaces=ns)
         if len(middle_name):
-            middle_name = middle_name[0].encode('utf-8')
+            middle_name = middle_name[0]
         else:
             middle_name = u''
         first_name = element.xpath('fb:first-name/text()', namespaces=ns)
         if len(first_name):
-            first_name = first_name[0].encode('utf-8')
+            first_name = first_name[0]
         else:
             first_name = u''
-        return (first_name.decode('utf-8') + u' '
-                + middle_name.decode('utf-8') + u' '
-                + last_name.decode('utf-8')).encode('utf-8')
+        return (first_name + u' '
+                + middle_name + u' '
+                + last_name)
 
     author = str(", ".join(map(get_author, authors)))
 
     title = tree.xpath('/fb:FictionBook/fb:description/fb:title-info/fb:book-title/text()', namespaces=ns)
     if len(title):
-        title = str(title[0].encode('utf-8'))
+        title = str(title[0])
     else:
         title = u''
     description = tree.xpath('/fb:FictionBook/fb:description/fb:publish-info/fb:book-name/text()', namespaces=ns)
     if len(description):
-        description = str(description[0].encode('utf-8'))
+        description = str(description[0])
     else:
         description = u''
 
     return BookMeta(
         file_path=tmp_file_path,
         extension=original_file_extension,
-        title=title.decode('utf-8'),
-        author=author.decode('utf-8'),
+        title=title,
+        author=author,
         cover=None,
-        description=description.decode('utf-8'),
+        description=description,
         tags="",
         series="",
         series_id="",
-        languages="")
+        languages="",
+        publisher="")

cps/gdrive.py

@@ -35,9 +35,9 @@ from flask_babel import gettext as _
 from flask_login import login_required
 
 from . import logger, gdriveutils, config, ub, calibre_db
-from .web import admin_required
+from .admin import admin_required
 
-gdrive = Blueprint('gdrive', __name__)
+gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive')
 log = logger.create()
 
 try:
@@ -47,10 +47,10 @@ except ImportError as err:
 
 current_milli_time = lambda: int(round(time() * 1000))
 
-gdrive_watch_callback_token = 'target=calibreweb-watch_files'
+gdrive_watch_callback_token = 'target=calibreweb-watch_files'  #nosec
 
 
-@gdrive.route("/gdrive/authenticate")
+@gdrive.route("/authenticate")
 @login_required
 @admin_required
 def authenticate_google_drive():
@@ -63,7 +63,7 @@ def authenticate_google_drive():
     return redirect(authUrl)
 
 
-@gdrive.route("/gdrive/callback")
+@gdrive.route("/callback")
 def google_drive_callback():
     auth_code = request.args.get('code')
     if not auth_code:
@@ -77,18 +77,14 @@ def google_drive_callback():
     return redirect(url_for('admin.configuration'))
 
 
-@gdrive.route("/gdrive/watch/subscribe")
+@gdrive.route("/watch/subscribe")
 @login_required
 @admin_required
 def watch_gdrive():
     if not config.config_google_drive_watch_changes_response:
         with open(gdriveutils.CLIENT_SECRETS, 'r') as settings:
             filedata = json.load(settings)
-        if filedata['web']['redirect_uris'][0].endswith('/'):
-            filedata['web']['redirect_uris'][0] = filedata['web']['redirect_uris'][0][:-((len('/gdrive/callback')+1))]
-        else:
-            filedata['web']['redirect_uris'][0] = filedata['web']['redirect_uris'][0][:-(len('/gdrive/callback'))]
-        address = '%s/gdrive/watch/callback' % filedata['web']['redirect_uris'][0]
+        address = filedata['web']['redirect_uris'][0].rstrip('/').replace('/gdrive/callback', '/gdrive/watch/callback')
         notification_id = str(uuid4())
         try:
             result = gdriveutils.watchChange(gdriveutils.Gdrive.Instance().drive, notification_id,
@@ -98,14 +94,15 @@ def watch_gdrive():
         except HttpError as e:
             reason=json.loads(e.content)['error']['errors'][0]
             if reason['reason'] == u'push.webhookUrlUnauthorized':
-                flash(_(u'Callback domain is not verified, please follow steps to verify domain in google developer console'), category="error")
+                flash(_(u'Callback domain is not verified, '
+                        u'please follow steps to verify domain in google developer console'), category="error")
             else:
                 flash(reason['message'], category="error")
 
     return redirect(url_for('admin.configuration'))
 
 
-@gdrive.route("/gdrive/watch/revoke")
+@gdrive.route("/watch/revoke")
 @login_required
 @admin_required
 def revoke_watch_gdrive():
@@ -121,14 +118,14 @@ def revoke_watch_gdrive():
     return redirect(url_for('admin.configuration'))
 
 
-@gdrive.route("/gdrive/watch/callback", methods=['GET', 'POST'])
+@gdrive.route("/watch/callback", methods=['GET', 'POST'])
 def on_received_watch_confirmation():
     if not config.config_google_drive_watch_changes_response:
         return ''
     if request.headers.get('X-Goog-Channel-Token') != gdrive_watch_callback_token \
             or request.headers.get('X-Goog-Resource-State') != 'change' \
             or not request.data:
-        return '' # redirect(url_for('admin.configuration'))
+        return ''
 
     log.debug('%r', request.headers)
     log.debug('%r', request.data)
@@ -145,16 +142,19 @@ def on_received_watch_confirmation():
             else:
                 dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode()
             if not response['deleted'] and response['file']['title'] == 'metadata.db' \
-                and response['file']['md5Checksum'] != hashlib.md5(dbpath):
-                tmpDir = tempfile.gettempdir()
+                and response['file']['md5Checksum'] != hashlib.md5(dbpath):  # nosec
+                tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web')
+                if not os.path.isdir(tmp_dir):
+                    os.mkdir(tmp_dir)
+
                 log.info('Database file updated')
-                copyfile(dbpath, os.path.join(tmpDir, "metadata.db_" + str(current_milli_time())))
+                copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time())))
                 log.info('Backing up existing and downloading updated metadata.db')
-                gdriveutils.downloadFile(None, "metadata.db", os.path.join(tmpDir, "tmp_metadata.db"))
+                gdriveutils.downloadFile(None, "metadata.db", os.path.join(tmp_dir, "tmp_metadata.db"))
                 log.info('Setting up new DB')
-                # prevent error on windows, as os.rename does on exisiting files
-                move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath)
+                # prevent error on windows, as os.rename does on existing files, also allow cross hdd move
+                move(os.path.join(tmp_dir, "tmp_metadata.db"), dbpath)
                 calibre_db.reconnect_db(config, ub.app_DB_path)
-    except Exception as e:
-        log.exception(e)
+    except Exception as ex:
+        log.debug_or_exception(ex)
     return ''

cps/gdriveutils.py

@@ -28,18 +28,31 @@ from sqlalchemy import create_engine
 from sqlalchemy import Column, UniqueConstraint
 from sqlalchemy import String, Integer
 from sqlalchemy.orm import sessionmaker, scoped_session
-from sqlalchemy.ext.declarative import declarative_base
+try:
+    # Compatibility with sqlalchemy 2.0
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.exc import OperationalError, InvalidRequestError
 
 try:
+    from apiclient import errors
+    from httplib2 import ServerNotFoundError
+    importError = None
+    gdrive_support = True
+except ImportError as e:
+    importError = e
+    gdrive_support = False
+try:
+    from pydrive2.auth import GoogleAuth
+    from pydrive2.drive import GoogleDrive
+    from pydrive2.auth import RefreshError
+except ImportError as err:
+    try:
         from pydrive.auth import GoogleAuth
         from pydrive.drive import GoogleDrive
         from pydrive.auth import RefreshError
-    from apiclient import errors
-    from httplib2 import ServerNotFoundError
-    gdrive_support = True
-    importError = None
-except ImportError as err:
+    except ImportError as err:
         importError = err
         gdrive_support = False
 
@@ -91,7 +104,7 @@ class Singleton:
         except AttributeError:
             self._instance = self._decorated()
             return self._instance
-        except ImportError as e:
+        except (ImportError, NameError) as e:
             log.debug(e)
             return None
 
@@ -189,8 +202,8 @@ def getDrive(drive=None, gauth=None):
                 gauth.Refresh()
             except RefreshError as e:
                 log.error("Google Drive error: %s", e)
-            except Exception as e:
-                log.exception(e)
+            except Exception as ex:
+                log.debug_or_exception(ex)
         else:
             # Initialize the saved creds
             gauth.Authorize()
@@ -208,7 +221,7 @@ def listRootFolders():
         drive = getDrive(Gdrive.Instance().drive)
         folder = "'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false"
         fileList = drive.ListFile({'q': folder}).GetList()
-    except ServerNotFoundError as e:
+    except (ServerNotFoundError, ssl.SSLError) as e:
         log.info("GDrive Error %s" % e)
         fileList = []
     return fileList
@@ -244,7 +257,12 @@ def getEbooksFolderId(drive=None):
             log.error('Error gDrive, root ID not found')
         gDriveId.path = '/'
         session.merge(gDriveId)
+        try:
             session.commit()
+        except OperationalError as ex:
+            log.error("gdrive.db DB is not Writeable")
+            log.debug('Database error: %s', ex)
+            session.rollback()
         return gDriveId.gdrive_id
 
 
@@ -259,6 +277,7 @@ def getFile(pathId, fileName, drive):
 
 def getFolderId(path, drive):
     # drive = getDrive(drive)
+    try:
         currentFolderId = getEbooksFolderId(drive)
         sqlCheckPath = path if path[-1] == '/' else path + '/'
         storedPathName = session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first()
@@ -290,6 +309,10 @@ def getFolderId(path, drive):
                 session.commit()
         else:
             currentFolderId = storedPathName.gdrive_id
+    except OperationalError as ex:
+        log.error("gdrive.db DB is not Writeable")
+        log.debug('Database error: %s', ex)
+        session.rollback()
     return currentFolderId
 
 
@@ -333,7 +356,7 @@ def moveGdriveFolderRemote(origin_file, target_folder):
                                       addParents=gFileTargetDir['id'],
                                       removeParents=previous_parents,
                                       fields='id, parents').execute()
-    # if previous_parents has no childs anymore, delete original fileparent
+    # if previous_parents has no children anymore, delete original fileparent
     if len(children['items']) == 1:
         deleteDatabaseEntry(previous_parents)
         drive.auth.service.files().delete(fileId=previous_parents).execute()
@@ -385,7 +408,8 @@ def uploadFileToEbooksFolder(destFile, f):
             if len(existingFiles) > 0:
                 driveFile = existingFiles[0]
             else:
-                driveFile = drive.CreateFile({'title': x, 'parents': [{"kind": "drive#fileLink", 'id': parent['id']}],})
+                driveFile = drive.CreateFile({'title': x,
+                                              'parents': [{"kind": "drive#fileLink", 'id': parent['id']}], })
             driveFile.SetContentFile(f)
             driveFile.Upload()
         else:
@@ -483,8 +507,8 @@ def getChangeById (drive, change_id):
     except (errors.HttpError) as error:
         log.error(error)
         return None
-    except Exception as e:
-        log.error(e)
+    except Exception as ex:
+        log.error(ex)
         return None
 
 
@@ -493,9 +517,10 @@ def deleteDatabaseOnChange():
     try:
         session.query(GdriveId).delete()
         session.commit()
-    except (OperationalError, InvalidRequestError):
+    except (OperationalError, InvalidRequestError) as ex:
         session.rollback()
-        log.info(u"GDrive DB is not Writeable")
+        log.debug('Database error: %s', ex)
+        log.error(u"GDrive DB is not Writeable")
 
 
 def updateGdriveCalibreFromLocal():
@@ -510,13 +535,23 @@ def updateDatabaseOnEdit(ID,newPath):
     storedPathName = session.query(GdriveId).filter(GdriveId.gdrive_id == ID).first()
     if storedPathName:
         storedPathName.path = sqlCheckPath
+        try:
             session.commit()
+        except OperationalError as ex:
+            log.error("gdrive.db DB is not Writeable")
+            log.debug('Database error: %s', ex)
+            session.rollback()
 
 
 # Deletes the hashes in database of deleted book
 def deleteDatabaseEntry(ID):
     session.query(GdriveId).filter(GdriveId.gdrive_id == ID).delete()
+    try:
         session.commit()
+    except OperationalError as ex:
+        log.error("gdrive.db DB is not Writeable")
+        log.debug('Database error: %s', ex)
+        session.rollback()
 
 
 # Gets cover file from gdrive
@@ -533,7 +568,12 @@ def get_cover_via_gdrive(cover_path):
             permissionAdded = PermissionAdded()
             permissionAdded.gdrive_id = df['id']
             session.add(permissionAdded)
+            try:
                 session.commit()
+            except OperationalError as ex:
+                log.error("gdrive.db DB is not Writeable")
+                log.debug('Database error: %s', ex)
+                session.rollback()
         return df.metadata.get('webContentLink')
     else:
         return None
@@ -547,21 +587,24 @@ def partial(total_byte_len, part_size_limit):
     return s
 
 # downloads files in chunks from gdrive
-def do_gdrive_download(df, headers):
+def do_gdrive_download(df, headers, convert_encoding=False):
     total_size = int(df.metadata.get('fileSize'))
     download_url = df.metadata.get('downloadUrl')
     s = partial(total_size, 1024 * 1024)  # I'm downloading BIG files, so 100M chunk size is fine for me
 
-    def stream():
+    def stream(convert_encoding):
         for byte in s:
             headers = {"Range": 'bytes=%s-%s' % (byte[0], byte[1])}
             resp, content = df.auth.Get_Http_Object().request(download_url, headers=headers)
             if resp.status == 206:
+                if convert_encoding:
+                    result = chardet.detect(content)
+                    content = content.decode(result['encoding']).encode('utf-8')
                 yield content
             else:
                 log.warning('An error occurred: %s', resp)
                 return
-    return Response(stream_with_context(stream()), headers=headers)
+    return Response(stream_with_context(stream(convert_encoding)), headers=headers)
 
 
 _SETTINGS_YAML_TEMPLATE = """

cps/helper.py

@@ -24,10 +24,7 @@ import io
 import mimetypes
 import re
 import shutil
-import glob
 import time
-import zipfile
-import json
 import unicodedata
 from datetime import datetime, timedelta
 from tempfile import gettempdir
@@ -35,10 +32,10 @@ from tempfile import gettempdir
 import requests
 from babel.dates import format_datetime
 from babel.units import format_unit
-from flask import send_from_directory, make_response, redirect, abort, url_for, send_file
+from flask import send_from_directory, make_response, redirect, abort, url_for
 from flask_babel import gettext as _
 from flask_login import current_user
-from sqlalchemy.sql.expression import true, false, and_, text
+from sqlalchemy.sql.expression import true, false, and_, text, func
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash
 
@@ -53,13 +50,6 @@ try:
 except ImportError:
     use_unidecode = False
 
-try:
-    from PIL import Image as PILImage
-    from PIL import UnidentifiedImageError
-    use_PIL = True
-except ImportError:
-    use_PIL = False
-
 from . import calibre_db
 from .tasks.convert import TaskConvert
 from . import logger, config, get_locale, db, ub
@@ -69,9 +59,17 @@ from .subproc_wrapper import process_wait
 from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS
 from .tasks.mail import TaskEmail
 
-
 log = logger.create()
 
+try:
+    from wand.image import Image
+    from wand.exceptions import MissingDelegateError
+    use_IM = True
+except (ImportError, RuntimeError) as e:
+    log.debug('Cannot import Image, generating covers from non jpg files will not work: %s', e)
+    use_IM = False
+    MissingDelegateError = BaseException
+
 
 # Convert existing book entry to new format
 def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
@@ -118,15 +116,15 @@ def send_test_mail(kindle_mail, user_name):
 
 # Send registration email or password reset email, depending on parameter resend (False means welcome email)
 def send_registration_mail(e_mail, user_name, default_password, resend=False):
-    text = "Hello %s!\r\n" % user_name
+    txt = "Hello %s!\r\n" % user_name
     if not resend:
-        text += "Your new account at Calibre-Web has been created. Thanks for joining us!\r\n"
-    text += "Please log in to your account using the following informations:\r\n"
-    text += "User name: %s\r\n" % user_name
-    text += "Password: %s\r\n" % default_password
-    text += "Don't forget to change your password after first login.\r\n"
-    text += "Sincerely\r\n\r\n"
-    text += "Your Calibre-Web team"
+        txt += "Your new account at Calibre-Web has been created. Thanks for joining us!\r\n"
+    txt += "Please log in to your account using the following informations:\r\n"
+    txt += "User name: %s\r\n" % user_name
+    txt += "Password: %s\r\n" % default_password
+    txt += "Don't forget to change your password after first login.\r\n"
+    txt += "Sincerely\r\n\r\n"
+    txt += "Your Calibre-Web team"
     WorkerThread.add(None, TaskEmail(
         subject=_(u'Get Started with Calibre-Web'),
         filepath=None,
@@ -134,55 +132,16 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
         settings=config.get_mail_settings(),
         recipient=e_mail,
         taskMessage=_(u"Registration e-mail for user: %(name)s", name=user_name),
-        text=text
+        text=txt
     ))
-
     return
 
 
-def check_send_to_kindle(entry):
-    """
-        returns all available book formats for sending to Kindle
-    """
-    if len(entry.data):
+def check_send_to_kindle_with_converter(formats):
     bookformats = list()
-        if not config.config_converterpath:
-            # no converter - only for mobi and pdf formats
-            for ele in iter(entry.data):
-                if ele.uncompressed_size < config.mail_size:
-                    if 'MOBI' in ele.format:
+    if 'EPUB' in formats and 'MOBI' not in formats:
         bookformats.append({'format': 'Mobi',
-                                            'convert': 0,
-                                            'text': _('Send %(format)s to Kindle', format='Mobi')})
-                    if 'PDF' in ele.format:
-                        bookformats.append({'format': 'Pdf',
-                                            'convert': 0,
-                                            'text': _('Send %(format)s to Kindle', format='Pdf')})
-                    if 'AZW' in ele.format:
-                        bookformats.append({'format': 'Azw',
-                                            'convert': 0,
-                                            'text': _('Send %(format)s to Kindle', format='Azw')})
-        else:
-            formats = list()
-            for ele in iter(entry.data):
-                if ele.uncompressed_size < config.mail_size:
-                    formats.append(ele.format)
-            if 'MOBI' in formats:
-                bookformats.append({'format': 'Mobi',
-                                    'convert': 0,
-                                    'text': _('Send %(format)s to Kindle', format='Mobi')})
-            if 'AZW' in formats:
-                bookformats.append({'format': 'Azw',
-                                    'convert': 0,
-                                    'text': _('Send %(format)s to Kindle', format='Azw')})
-            if 'PDF' in formats:
-                bookformats.append({'format': 'Pdf',
-                                    'convert': 0,
-                                    'text': _('Send %(format)s to Kindle', format='Pdf')})
-            if config.config_converterpath:
-                if 'EPUB' in formats and not 'MOBI' in formats:
-                    bookformats.append({'format': 'Mobi',
-                                        'convert':1,
+                            'convert': 1,
                             'text': _('Convert %(orig)s to %(format)s and send to Kindle',
                                       orig='Epub',
                                       format='Mobi')})
@@ -193,6 +152,33 @@ def check_send_to_kindle(entry):
                                       orig='Azw3',
                                       format='Mobi')})
     return bookformats
+
+
+def check_send_to_kindle(entry):
+    """
+        returns all available book formats for sending to Kindle
+    """
+    formats = list()
+    bookformats = list()
+    if len(entry.data):
+        for ele in iter(entry.data):
+            if ele.uncompressed_size < config.mail_size:
+                formats.append(ele.format)
+        if 'MOBI' in formats:
+            bookformats.append({'format': 'Mobi',
+                                'convert': 0,
+                                'text': _('Send %(format)s to Kindle', format='Mobi')})
+        if 'PDF' in formats:
+            bookformats.append({'format': 'Pdf',
+                                'convert': 0,
+                                'text': _('Send %(format)s to Kindle', format='Pdf')})
+        if 'AZW' in formats:
+            bookformats.append({'format': 'Azw',
+                                'convert': 0,
+                                'text': _('Send %(format)s to Kindle', format='Azw')})
+        if config.config_converterpath:
+            bookformats.extend(check_send_to_kindle_with_converter(formats))
+        return bookformats
     else:
         log.error(u'Cannot find book entry %d', entry.id)
         return None
@@ -201,7 +187,7 @@ def check_send_to_kindle(entry):
 # Check if a reader is existing for any of the book formats, if not, return empty list, otherwise return
 # list with supported formats
 def check_read_formats(entry):
-    EXTENSIONS_READER = {'TXT', 'PDF', 'EPUB', 'CBZ', 'CBT', 'CBR'}
+    EXTENSIONS_READER = {'TXT', 'PDF', 'EPUB', 'CBZ', 'CBT', 'CBR', 'DJVU'}
     bookformats = list()
     if len(entry.data):
         for ele in iter(entry.data):
@@ -494,8 +480,8 @@ def reset_password(user_id):
         password = generate_random_password()
         existing_user.password = generate_password_hash(password)
         ub.session.commit()
-        send_registration_mail(existing_user.email, existing_user.nickname, password, True)
-        return 1, existing_user.nickname
+        send_registration_mail(existing_user.email, existing_user.name, password, True)
+        return 1, existing_user.name
     except Exception:
         ub.session.rollback()
         return 0, None
@@ -512,11 +498,37 @@ def generate_random_password():
 
 def uniq(inpt):
     output = []
+    inpt = [ " ".join(inp.split()) for inp in inpt]
     for x in inpt:
         if x not in output:
             output.append(x)
     return output
 
+def check_email(email):
+    email = valid_email(email)
+    if ub.session.query(ub.User).filter(func.lower(ub.User.email) == email.lower()).first():
+        log.error(u"Found an existing account for this e-mail address")
+        raise Exception(_(u"Found an existing account for this e-mail address"))
+    return email
+
+
+def check_username(username):
+    username = username.strip()
+    if ub.session.query(ub.User).filter(func.lower(ub.User.name) == username.lower()).scalar():
+        log.error(u"This username is already taken")
+        raise Exception (_(u"This username is already taken"))
+    return username
+
+
+def valid_email(email):
+    email = email.strip()
+    # Regex according to https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/email#validation
+    if not re.search(r"^[\w.!#$%&'*+\\/=?^_`{|}~-]+@[\w](?:[\w-]{0,61}[\w])?(?:\.[\w](?:[\w-]{0,61}[\w])?)*$",
+                     email):
+        log.error(u"Invalid e-mail address format")
+        raise Exception(_(u"Invalid e-mail address format"))
+    return email
+
 # ################################# External interface #################################
 
 
@@ -564,9 +576,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
                 else:
                     log.error('%s/cover.jpg not found on Google Drive', book.path)
                     return get_cover_on_failure(use_generic_cover_on_failure)
-            except Exception as e:
-                log.exception(e)
-                # traceback.print_exc()
+            except Exception as ex:
+                log.debug_or_exception(ex)
                 return get_cover_on_failure(use_generic_cover_on_failure)
         else:
             cover_file_path = os.path.join(config.config_calibre_dir, book.path)
@@ -589,17 +600,12 @@ def save_cover_from_url(url, book_path):
             requests.exceptions.Timeout) as ex:
         log.info(u'Cover Download Error %s', ex)
         return False, _("Error Downloading Cover")
-    except UnidentifiedImageError as ex:
+    except MissingDelegateError as ex:
         log.info(u'File Format Error %s', ex)
         return False, _("Cover Format Error")
 
 
 def save_cover_from_filestorage(filepath, saved_filename, img):
-    if hasattr(img, '_content'):
-        f = open(os.path.join(filepath, saved_filename), "wb")
-        f.write(img._content)
-        f.close()
-    else:
     # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
     if not os.path.exists(filepath):
         try:
@@ -608,6 +614,17 @@ def save_cover_from_filestorage(filepath, saved_filename, img):
             log.error(u"Failed to create path for cover")
             return False, _(u"Failed to create path for cover")
     try:
+        # upload of jgp file without wand
+        if isinstance(img, requests.Response):
+            with open(os.path.join(filepath, saved_filename), 'wb') as f:
+                f.write(img.content)
+        else:
+            if hasattr(img, "metadata"):
+                # upload of jpg/png... via url
+                img.save(filename=os.path.join(filepath, saved_filename))
+                img.close()
+            else:
+                # upload of jpg/png... from hdd
                 img.save(os.path.join(filepath, saved_filename))
     except (IOError, OSError):
         log.error(u"Cover-file is not a valid image file, or could not be stored")
@@ -619,31 +636,33 @@ def save_cover_from_filestorage(filepath, saved_filename, img):
 def save_cover(img, book_path):
     content_type = img.headers.get('content-type')
 
-    if use_PIL:
-        if content_type not in ('image/jpeg', 'image/png', 'image/webp'):
-            log.error("Only jpg/jpeg/png/webp files are supported as coverfile")
-            return False, _("Only jpg/jpeg/png/webp files are supported as coverfile")
+    if use_IM:
+        if content_type not in ('image/jpeg', 'image/png', 'image/webp', 'image/bmp'):
+            log.error("Only jpg/jpeg/png/webp/bmp files are supported as coverfile")
+            return False, _("Only jpg/jpeg/png/webp/bmp files are supported as coverfile")
         # convert to jpg because calibre only supports jpg
-        if content_type in ('image/png', 'image/webp'):
+        if content_type != 'image/jpg':
             if hasattr(img, 'stream'):
-                imgc = PILImage.open(img.stream)
+                imgc = Image(blob=img.stream)
             else:
-                imgc = PILImage.open(io.BytesIO(img.content))
-            im = imgc.convert('RGB')
-            tmp_bytesio = io.BytesIO()
-            im.save(tmp_bytesio, format='JPEG')
-            img._content = tmp_bytesio.getvalue()
+                imgc = Image(blob=io.BytesIO(img.content))
+            imgc.format = 'jpeg'
+            imgc.transform_colorspace("rgb")
+            img = imgc
     else:
         if content_type not in 'image/jpeg':
             log.error("Only jpg/jpeg files are supported as coverfile")
             return False, _("Only jpg/jpeg files are supported as coverfile")
 
     if config.config_use_google_drive:
-        tmpDir = gettempdir()
-        ret, message = save_cover_from_filestorage(tmpDir, "uploaded_cover.jpg", img)
+        tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+
+        if not os.path.isdir(tmp_dir):
+            os.mkdir(tmp_dir)
+        ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img)
         if ret is True:
-            gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'),
-                                        os.path.join(tmpDir, "uploaded_cover.jpg"))
+            gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\","/"),
+                                        os.path.join(tmp_dir, "uploaded_cover.jpg"))
             log.info("Cover is saved on Google Drive")
             return True, None
         else:
@@ -674,6 +693,7 @@ def do_download_file(book, book_format, client, data, headers):
         # ToDo Check headers parameter
         for element in headers:
             response.headers[element[0]] = element[1]
+        log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format)))
         return response
 
 ##################################
@@ -697,7 +717,7 @@ def check_unrar(unrarLocation):
                 log.debug("unrar version %s", version)
                 break
     except (OSError, UnicodeDecodeError) as err:
-        log.exception(err)
+        log.debug_or_exception(err)
         return _('Error excecuting UnRar')
 
 
@@ -713,7 +733,6 @@ def json_serial(obj):
             'seconds': obj.seconds,
             'microseconds': obj.microseconds,
         }
-        # return obj.isoformat()
     raise TypeError("Type %s not serializable" % type(obj))
 
 
@@ -737,8 +756,8 @@ def format_runtime(runtime):
 # helper function to apply localize status information in tasklist entries
 def render_task_status(tasklist):
     renderedtasklist = list()
-    for num, user, added, task in tasklist:
-        if user == current_user.nickname or current_user.role_admin():
+    for __, user, __, task in tasklist:
+        if user == current_user.name or current_user.role_admin():
             ret = {}
             if task.start_time:
                 ret['starttime'] = format_datetime(task.start_time, format='short', locale=get_locale())
@@ -776,8 +795,8 @@ def tags_filters():
 # checks if domain is in database (including wildcards)
 # example SELECT * FROM @TABLE WHERE  'abcdefg' LIKE Name;
 # from https://code.luasoftware.com/tutorials/flask/execute-raw-sql-in-flask-sqlalchemy/
+# in all calls the email address is checked for validity
 def check_valid_domain(domain_text):
-    # domain_text = domain_text.split('@', 1)[-1].lower()
     sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 1);"
     result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all()
     if not len(result):
@@ -811,6 +830,7 @@ def get_download_link(book_id, book_format, client):
     if book:
         data1 = calibre_db.get_book_format(book.id, book_format.upper())
     else:
+        log.error("Book id {} not found for downloading".format(book_id))
         abort(404)
     if data1:
         # collect downloaded books only for registered user and not for anonymous user
@@ -827,4 +847,3 @@ def get_download_link(book_id, book_format, client):
         return do_download_file(book, book_format, client, data1, headers)
     else:
         abort(404)
-

cps/isoLanguages.py

@@ -57,27 +57,30 @@ def get_language_name(locale, lang_code):
 
 def get_language_codes(locale, language_names, remainder=None):
     language_names = set(x.strip().lower() for x in language_names if x)
-    languages = list()
+    lang = list()
     for k, v in get_language_names(locale).items():
         v = v.lower()
         if v in language_names:
-            languages.append(k)
+            lang.append(k)
             language_names.remove(v)
-    if remainder is not None:
+    if remainder is not None and language_names:
         remainder.extend(language_names)
-    return languages
+    return lang
+
+
 
 def get_valid_language_codes(locale, language_names, remainder=None):
-    languages = list()
+    lang = list()
     if "" in language_names:
         language_names.remove("")
-    for k, v in get_language_names(locale).items():
+    for k, __ in get_language_names(locale).items():
         if k in language_names:
-            languages.append(k)
+            lang.append(k)
             language_names.remove(k)
     if remainder is not None and len(language_names):
         remainder.extend(language_names)
-    return languages
+    return lang
+
 
 def get_lang3(lang):
     try:

cps/jinjia.py

@@ -82,7 +82,7 @@ def formatdate_filter(val):
     except AttributeError as e:
         log.error('Babel error: %s, Current user locale: %s, Current User: %s', e,
                   current_user.locale,
-                  current_user.nickname
+                  current_user.name
                   )
         return val
 

cps/kobo.py

@@ -42,7 +42,7 @@ from flask import (
 from flask_login import current_user
 from werkzeug.datastructures import Headers
 from sqlalchemy import func
-from sqlalchemy.sql.expression import and_, or_
+from sqlalchemy.sql.expression import and_
 from sqlalchemy.exc import StatementError
 import requests
 
@@ -56,6 +56,8 @@ KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]}
 KOBO_STOREAPI_URL = "https://storeapi.kobo.com"
 KOBO_IMAGEHOST_URL = "https://kbimages1-a.akamaihd.net"
 
+SYNC_ITEM_LIMIT = 100
+
 kobo = Blueprint("kobo", __name__, url_prefix="/kobo/<auth_token>")
 kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo)
 kobo_auth.register_url_value_preprocessor(kobo)
@@ -142,68 +144,84 @@ def HandleSyncRequest():
     new_books_last_modified = sync_token.books_last_modified
     new_books_last_created = sync_token.books_last_created
     new_reading_state_last_modified = sync_token.reading_state_last_modified
+    new_archived_last_modified = datetime.datetime.min
     sync_results = []
 
     # We reload the book database so that the user get's a fresh view of the library
     # in case of external changes (e.g: adding a book through Calibre).
     calibre_db.reconnect_db(config, ub.app_DB_path)
 
-    archived_books = (
-        ub.session.query(ub.ArchivedBook)
-        .filter(ub.ArchivedBook.user_id == int(current_user.id))
-        .all()
-    )
-
-    # We join-in books that have had their Archived bit recently modified in order to either:
-    #   * Restore them to the user's device.
-    #   * Delete them from the user's device.
-    # (Ideally we would use a join for this logic, however cross-database joins don't look trivial in SqlAlchemy.)
-    recently_restored_or_archived_books = []
-    archived_book_ids = {}
-    new_archived_last_modified = datetime.datetime.min
-    for archived_book in archived_books:
-        if archived_book.last_modified > sync_token.archive_last_modified:
-            recently_restored_or_archived_books.append(archived_book.book_id)
-        if archived_book.is_archived:
-            archived_book_ids[archived_book.book_id] = True
-        new_archived_last_modified = max(
-            new_archived_last_modified, archived_book.last_modified)
-
-    # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast.
-    # It looks like it's treating the db.Books.last_modified field as a string and may fail
-    # the comparison because of the +00:00 suffix.
+    if sync_token.books_last_id > -1:
         changed_entries = (
-        calibre_db.session.query(db.Books)
-        .join(db.Data)
-        .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified,
-                    db.Books.id.in_(recently_restored_or_archived_books)))
+            calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
+            .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
+            .filter(db.Books.last_modified >= sync_token.books_last_modified)
+            .filter(db.Books.id>sync_token.books_last_id)
             .filter(db.Data.format.in_(KOBO_FORMATS))
-        .all()
+            .order_by(db.Books.last_modified)
+            .order_by(db.Books.id)
+            .limit(SYNC_ITEM_LIMIT)
+        )
+    else:
+        changed_entries = (
+            calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
+            .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
+            .filter(db.Books.last_modified > sync_token.books_last_modified)
+            .filter(db.Data.format.in_(KOBO_FORMATS))
+            .order_by(db.Books.last_modified)
+            .order_by(db.Books.id)
+            .limit(SYNC_ITEM_LIMIT)
         )
 
     reading_states_in_new_entitlements = []
     for book in changed_entries:
-        kobo_reading_state = get_or_create_reading_state(book.id)
+        formats = [data.format for data in book.Books.data]
+        if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
+            helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
+
+        kobo_reading_state = get_or_create_reading_state(book.Books.id)
         entitlement = {
-            "BookEntitlement": create_book_entitlement(book, archived=(book.id in archived_book_ids)),
-            "BookMetadata": get_metadata(book),
+            "BookEntitlement": create_book_entitlement(book.Books, archived=(book.is_archived == True)),
+            "BookMetadata": get_metadata(book.Books),
         }
 
         if kobo_reading_state.last_modified > sync_token.reading_state_last_modified:
-            entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state)
+            entitlement["ReadingState"] = get_kobo_reading_state_response(book.Books, kobo_reading_state)
             new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
-            reading_states_in_new_entitlements.append(book.id)
+            reading_states_in_new_entitlements.append(book.Books.id)
 
-        if book.timestamp > sync_token.books_last_created:
+        if book.Books.timestamp > sync_token.books_last_created:
             sync_results.append({"NewEntitlement": entitlement})
         else:
             sync_results.append({"ChangedEntitlement": entitlement})
 
         new_books_last_modified = max(
-            book.last_modified, new_books_last_modified
+            book.Books.last_modified, new_books_last_modified
         )
-        new_books_last_created = max(book.timestamp, new_books_last_created)
+        new_books_last_created = max(book.Books.timestamp, new_books_last_created)
 
+    max_change = (changed_entries
+        .from_self()
+        .filter(ub.ArchivedBook.is_archived)
+        .order_by(func.datetime(ub.ArchivedBook.last_modified).desc())
+        .first()
+    )
+    if max_change:
+        max_change = max_change.last_modified
+    else:
+        max_change = new_archived_last_modified
+    new_archived_last_modified = max(new_archived_last_modified, max_change)
+
+    # no. of books returned
+    book_count = changed_entries.count()
+
+    # last entry:
+    if book_count:
+        books_last_id = changed_entries.all()[-1].Books.id or -1
+    else:
+        books_last_id = -1
+
+    # generate reading state data
     changed_reading_states = (
         ub.session.query(ub.KoboReadingState)
         .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
@@ -225,11 +243,12 @@ def HandleSyncRequest():
     sync_token.books_last_modified = new_books_last_modified
     sync_token.archive_last_modified = new_archived_last_modified
     sync_token.reading_state_last_modified = new_reading_state_last_modified
+    sync_token.books_last_id = books_last_id
 
-    return generate_sync_response(sync_token, sync_results)
+    return generate_sync_response(sync_token, sync_results, book_count)
 
 
-def generate_sync_response(sync_token, sync_results):
+def generate_sync_response(sync_token, sync_results, set_cont=False):
     extra_headers = {}
     if config.config_kobo_proxy:
         # Merge in sync results from the official Kobo store.
@@ -243,8 +262,10 @@ def generate_sync_response(sync_token, sync_results):
             extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode")
             extra_headers["x-kobo-recent-reads"] = store_response.headers.get("x-kobo-recent-reads")
 
-        except Exception as e:
-            log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e))
+        except Exception as ex:
+            log.error("Failed to receive or parse response from Kobo's sync endpoint: {}".format(ex))
+    if set_cont:
+        extra_headers["x-kobo-sync"] = "continue"
     sync_token.to_headers(extra_headers)
 
     response = make_response(jsonify(sync_results), extra_headers)
@@ -284,7 +305,8 @@ def get_download_url_for_book(book, book_format):
             book_format=book_format.lower()
         )
     return url_for(
-        "web.download_link",
+        "kobo.download_book",
+        auth_token=kobo_auth.get_auth_token(),
         book_id=book.id,
         book_format=book_format.lower(),
         _external=True,
@@ -443,8 +465,7 @@ def HandleTagCreate():
     items_unknown_to_calibre = add_items_to_shelf(items, shelf)
     if items_unknown_to_calibre:
         log.debug("Received request to add unknown books to a collection. Silently ignoring items.")
-    ub.session.commit()
-
+    ub.session_commit()
     return make_response(jsonify(str(shelf.uuid)), 201)
 
 
@@ -476,7 +497,7 @@ def HandleTagUpdate(tag_id):
 
         shelf.name = name
         ub.session.merge(shelf)
-        ub.session.commit()
+        ub.session_commit()
     return make_response(' ', 200)
 
 
@@ -528,8 +549,7 @@ def HandleTagAddItem(tag_id):
         log.debug("Received request to add an unknown book to a collection. Silently ignoring item.")
 
     ub.session.merge(shelf)
-    ub.session.commit()
-
+    ub.session_commit()
     return make_response('', 201)
 
 
@@ -569,7 +589,7 @@ def HandleTagRemoveItem(tag_id):
             shelf.books.filter(ub.BookShelf.book_id == book.id).delete()
         except KeyError:
             items_unknown_to_calibre.append(item)
-    ub.session.commit()
+    ub.session_commit()
 
     if items_unknown_to_calibre:
         log.debug("Received request to remove an unknown book to a collecition. Silently ignoring item.")
@@ -615,7 +635,7 @@ def sync_shelves(sync_token, sync_results):
                 "ChangedTag": tag
             })
     sync_token.tags_last_modified = new_tags_last_modified
-    ub.session.commit()
+    ub.session_commit()
 
 
 # Creates a Kobo "Tag" object from a ub.Shelf object
@@ -696,7 +716,7 @@ def HandleStateRequest(book_uuid):
             abort(400, description="Malformed request data is missing 'ReadingStates' key")
 
         ub.session.merge(kobo_reading_state)
-        ub.session.commit()
+        ub.session_commit()
         return jsonify({
             "RequestResult": "Success",
             "UpdateResults": [update_results_response],
@@ -734,7 +754,7 @@ def get_or_create_reading_state(book_id):
         kobo_reading_state.statistics = ub.KoboStatistics()
         book_read.kobo_reading_state = kobo_reading_state
     ub.session.add(book_read)
-    ub.session.commit()
+    ub.session_commit()
     return book_read.kobo_reading_state
 
 
@@ -837,8 +857,7 @@ def HandleBookDeletionRequest(book_uuid):
     archived_book.last_modified = datetime.datetime.utcnow()
 
     ub.session.merge(archived_book)
-    ub.session.commit()
-
+    ub.session_commit()
     return ("", 204)
 
 
@@ -874,17 +893,6 @@ def HandleProductsRequest(dummy=None):
     return redirect_or_proxy_request()
 
 
-'''@kobo.errorhandler(404)
-def handle_404(err):
-    # This handler acts as a catch-all for endpoints that we don't have an interest in
-    # implementing (e.g: v1/analytics/gettests, v1/user/recommendations, etc)
-    if err:
-        print('404')
-        return jsonify(error=str(err)), 404
-    log.debug("Unknown Request received: %s, method: %s, data: %s", request.base_url, request.method, request.data)
-    return redirect_or_proxy_request()'''
-
-
 def make_calibre_web_auth_response():
     # As described in kobo_auth.py, CalibreWeb doesn't make use practical use of this auth/device API call for
     # authentation (nor for authorization). We return a dummy response just to keep the device happy.
@@ -911,7 +919,7 @@ def HandleAuthRequest():
     if config.config_kobo_proxy:
         try:
             return redirect_or_proxy_request()
-        except:
+        except Exception:
             log.error("Failed to receive or parse response from Kobo's auth endpoint. Falling back to un-proxied mode.")
     return make_calibre_web_auth_response()
 
@@ -928,7 +936,7 @@ def HandleInitRequest():
             store_response_json = store_response.json()
             if "Resources" in store_response_json:
                 kobo_resources = store_response_json["Resources"]
-        except:
+        except Exception:
             log.error("Failed to receive or parse response from Kobo's init endpoint. Falling back to un-proxied mode.")
     if not kobo_resources:
         kobo_resources = NATIVE_KOBO_RESOURCES()
@@ -989,7 +997,6 @@ def HandleInitRequest():
 @requires_kobo_auth
 @download_required
 def download_book(book_id, book_format):
-
     return get_download_link(book_id, book_format, "kobo")
 
 

cps/kobo_auth.py

@@ -64,11 +64,11 @@ from datetime import datetime
 from os import urandom
 
 from flask import g, Blueprint, url_for, abort, request
-from flask_login import login_user, login_required
+from flask_login import login_user, current_user, login_required
 from flask_babel import gettext as _
 
-from . import logger, ub, lm
-from .web import render_title_template
+from . import logger, config, calibre_db, db, helper, ub, lm
+from .render_template import render_title_template
 
 try:
     from functools import wraps
@@ -81,6 +81,7 @@ log = logger.create()
 
 def register_url_value_preprocessor(kobo):
     @kobo.url_value_preprocessor
+    # pylint: disable=unused-variable
     def pop_auth_token(__, values):
         g.auth_token = values.pop("auth_token")
 
@@ -147,7 +148,15 @@ def generate_auth_token(user_id):
             auth_token.token_type = 1
 
             ub.session.add(auth_token)
-            ub.session.commit()
+            ub.session_commit()
+
+        books = calibre_db.session.query(db.Books).join(db.Data).all()
+
+        for book in books:
+            formats = [data.format for data in book.data]
+            if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
+                helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
+
         return render_title_template(
             "generate_kobo_auth_url.html",
             title=_(u"Kobo Setup"),
@@ -164,5 +173,5 @@ def delete_auth_token(user_id):
     # Invalidate any prevously generated Kobo Auth token for this user.
     ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == user_id)\
         .filter(ub.RemoteAuthToken.token_type==1).delete()
-    ub.session.commit()
-    return ""
+
+    return ub.session_commit()

cps/logger.py

@@ -41,10 +41,37 @@ logging.addLevelName(logging.WARNING, "WARN")
 logging.addLevelName(logging.CRITICAL, "CRIT")
 
 
+class _Logger(logging.Logger):
+
+    def debug_or_exception(self, message, *args, **kwargs):
+        if sys.version_info > (3, 7):
+            if is_debug_enabled():
+                self.exception(message, stacklevel=2, *args, **kwargs)
+            else:
+                self.error(message, stacklevel=2, *args, **kwargs)
+        elif sys.version_info > (3, 0):
+            if is_debug_enabled():
+                self.exception(message, stack_info=True, *args, **kwargs)
+            else:
+                self.error(message, *args, **kwargs)
+        else:
+            if is_debug_enabled():
+                self.exception(message, *args, **kwargs)
+            else:
+                self.error(message, *args, **kwargs)
+
+
+    def debug_no_auth(self, message, *args, **kwargs):
+        message = message.strip("\r\n")
+        if message.startswith("send: AUTH"):
+            self.debug(message[:16], *args, **kwargs)
+        else:
+            self.debug(message, *args, **kwargs)
+
+
 def get(name=None):
     return logging.getLogger(name)
 
-
 def create():
     parent_frame = inspect.stack(0)[1]
     if hasattr(parent_frame, 'frame'):
@@ -54,7 +81,6 @@ def create():
     parent_module = inspect.getmodule(parent_frame)
     return get(parent_module.__name__)
 
-
 def is_debug_enabled():
     return logging.root.level <= logging.DEBUG
 
@@ -99,6 +125,7 @@ def setup(log_file, log_level=None):
     May be called multiple times.
     '''
     log_level = log_level or DEFAULT_LOG_LEVEL
+    logging.setLoggerClass(_Logger)
     logging.getLogger(__package__).setLevel(log_level)
 
     r = logging.root
@@ -126,11 +153,11 @@ def setup(log_file, log_level=None):
             file_handler.baseFilename = log_file
     else:
         try:
-            file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2, encoding='utf-8')
+            file_handler = RotatingFileHandler(log_file, maxBytes=100000, backupCount=2, encoding='utf-8')
         except IOError:
             if log_file == DEFAULT_LOG_FILE:
                 raise
-            file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=50000, backupCount=2, encoding='utf-8')
+            file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=100000, backupCount=2, encoding='utf-8')
             log_file = ""
     file_handler.setFormatter(FORMATTER)
 

cps/oauth.py

@@ -19,7 +19,6 @@
 from __future__ import division, print_function, unicode_literals
 from flask import session
 
-
 try:
     from flask_dance.consumer.backend.sqla import SQLAlchemyBackend, first, _get_real_user
     from sqlalchemy.orm.exc import NoResultFound
@@ -34,8 +33,8 @@ except ImportError:
     except ImportError:
         pass
 
-try:
-    class OAuthBackend(SQLAlchemyBackend):
+
+class OAuthBackend(SQLAlchemyBackend):
     """
     Stores and retrieves OAuth tokens using a relational database through
     the `SQLAlchemy`_ ORM.
@@ -162,6 +161,3 @@ try:
         self.cache.delete(self.make_cache_key(
             blueprint=blueprint, user=user, user_id=user_id,
         ))
-
-except Exception:
-    pass

cps/oauth_bb.py

@@ -30,15 +30,20 @@ from flask_babel import gettext as _
 from flask_dance.consumer import oauth_authorized, oauth_error
 from flask_dance.contrib.github import make_github_blueprint, github
 from flask_dance.contrib.google import make_google_blueprint, google
-from flask_login import login_user, current_user
+from oauthlib.oauth2 import TokenExpiredError, InvalidGrantError
+from flask_login import login_user, current_user, login_required
 from sqlalchemy.orm.exc import NoResultFound
 
 from . import constants, logger, config, app, ub
-from .web import login_required
-from .oauth import OAuthBackend, backend_resultcode
+
+try:
+    from .oauth import OAuthBackend, backend_resultcode
+except NameError:
+    pass
 
 
 oauth_check = {}
+oauthblueprints = []
 oauth = Blueprint('oauth', __name__)
 log = logger.create()
 
@@ -84,11 +89,7 @@ def register_user_with_oauth(user=None):
             except NoResultFound:
                 # no found, return error
                 return
-            try:
-                ub.session.commit()
-            except Exception as e:
-                log.exception(e)
-                ub.session.rollback()
+            ub.session_commit("User {} with OAuth for provider {} registered".format(user.name, oauth_key))
 
 
 def logout_oauth_user():
@@ -97,19 +98,122 @@ def logout_oauth_user():
             session.pop(str(oauth_key) + '_oauth_user_id')
 
 
-if ub.oauth_support:
-    oauthblueprints = []
+def oauth_update_token(provider_id, token, provider_user_id):
+    session[provider_id + "_oauth_user_id"] = provider_user_id
+    session[provider_id + "_oauth_token"] = token
+
+    # Find this OAuth token in the database, or create it
+    query = ub.session.query(ub.OAuth).filter_by(
+        provider=provider_id,
+        provider_user_id=provider_user_id,
+    )
+    try:
+        oauth_entry = query.one()
+        # update token
+        oauth_entry.token = token
+    except NoResultFound:
+        oauth_entry = ub.OAuth(
+            provider=provider_id,
+            provider_user_id=provider_user_id,
+            token=token,
+        )
+    ub.session.add(oauth_entry)
+    ub.session_commit()
+
+    # Disable Flask-Dance's default behavior for saving the OAuth token
+    # Value differrs depending on flask-dance version
+    return backend_resultcode
+
+
+def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider_name):
+    query = ub.session.query(ub.OAuth).filter_by(
+        provider=provider_id,
+        provider_user_id=provider_user_id,
+    )
+    try:
+        oauth_entry = query.first()
+        # already bind with user, just login
+        if oauth_entry.user:
+            login_user(oauth_entry.user)
+            log.debug(u"You are now logged in as: '%s'", oauth_entry.user.name)
+            flash(_(u"you are now logged in as: '%(nickname)s'", nickname= oauth_entry.user.name),
+                  category="success")
+            return redirect(url_for('web.index'))
+        else:
+            # bind to current user
+            if current_user and current_user.is_authenticated:
+                oauth_entry.user = current_user
+                try:
+                    ub.session.add(oauth_entry)
+                    ub.session.commit()
+                    flash(_(u"Link to %(oauth)s Succeeded", oauth=provider_name), category="success")
+                    log.info("Link to {} Succeeded".format(provider_name))
+                    return redirect(url_for('web.profile'))
+                except Exception as ex:
+                    log.debug_or_exception(ex)
+                    ub.session.rollback()
+            else:
+                flash(_(u"Login failed, No User Linked With OAuth Account"), category="error")
+            log.info('Login failed, No User Linked With OAuth Account')
+            return redirect(url_for('web.login'))
+            # return redirect(url_for('web.login'))
+            # if config.config_public_reg:
+            #   return redirect(url_for('web.register'))
+            # else:
+            #    flash(_(u"Public registration is not enabled"), category="error")
+            #    return redirect(url_for(redirect_url))
+    except (NoResultFound, AttributeError):
+        return redirect(url_for(redirect_url))
+
+
+def get_oauth_status():
+    status = []
+    query = ub.session.query(ub.OAuth).filter_by(
+        user_id=current_user.id,
+    )
+    try:
+        oauths = query.all()
+        for oauth_entry in oauths:
+            status.append(int(oauth_entry.provider))
+        return status
+    except NoResultFound:
+        return None
+
+
+def unlink_oauth(provider):
+    if request.host_url + 'me' != request.referrer:
+        pass
+    query = ub.session.query(ub.OAuth).filter_by(
+        provider=provider,
+        user_id=current_user.id,
+    )
+    try:
+        oauth_entry = query.one()
+        if current_user and current_user.is_authenticated:
+            oauth_entry.user = current_user
+            try:
+                ub.session.delete(oauth_entry)
+                ub.session.commit()
+                logout_oauth_user()
+                flash(_(u"Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success")
+                log.info("Unlink to {} Succeeded".format(oauth_check[provider]))
+            except Exception as ex:
+                log.debug_or_exception(ex)
+                ub.session.rollback()
+                flash(_(u"Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error")
+    except NoResultFound:
+        log.warning("oauth %s for user %d not found", provider, current_user.id)
+        flash(_(u"Not Linked to %(oauth)s", oauth=provider), category="error")
+    return redirect(url_for('web.profile'))
+
+def generate_oauth_blueprints():
     if not ub.session.query(ub.OAuthProvider).count():
+        for provider in ("github", "google"):
             oauthProvider = ub.OAuthProvider()
-        oauthProvider.provider_name = "github"
+            oauthProvider.provider_name = provider
             oauthProvider.active = False
             ub.session.add(oauthProvider)
-        ub.session.commit()
-        oauthProvider = ub.OAuthProvider()
-        oauthProvider.provider_name = "google"
-        oauthProvider.active = False
-        ub.session.add(oauthProvider)
-        ub.session.commit()
+            ub.session_commit("{} Blueprint Created".format(provider))
 
     oauth_ids = ub.session.query(ub.OAuthProvider).all()
     ele1 = dict(provider_name='github',
@@ -146,17 +250,23 @@ if ub.oauth_support:
         app.register_blueprint(blueprint, url_prefix="/login")
         if element['active']:
             register_oauth_blueprint(element['id'], element['provider_name'])
+    return oauthblueprints
 
 
+if ub.oauth_support:
+    oauthblueprints = generate_oauth_blueprints()
+
     @oauth_authorized.connect_via(oauthblueprints[0]['blueprint'])
     def github_logged_in(blueprint, token):
         if not token:
             flash(_(u"Failed to log in with GitHub."), category="error")
+            log.error("Failed to log in with GitHub")
             return False
 
         resp = blueprint.session.get("/user")
         if not resp.ok:
             flash(_(u"Failed to fetch user info from GitHub."), category="error")
+            log.error("Failed to fetch user info from GitHub")
             return False
 
         github_info = resp.json()
@@ -168,11 +278,13 @@ if ub.oauth_support:
     def google_logged_in(blueprint, token):
         if not token:
             flash(_(u"Failed to log in with Google."), category="error")
+            log.error("Failed to log in with Google")
             return False
 
         resp = blueprint.session.get("/oauth2/v2/userinfo")
         if not resp.ok:
             flash(_(u"Failed to fetch user info from Google."), category="error")
+            log.error("Failed to fetch user info from Google")
             return False
 
         google_info = resp.json()
@@ -180,117 +292,6 @@ if ub.oauth_support:
         return oauth_update_token(str(oauthblueprints[1]['id']), token, google_user_id)
 
 
-    def oauth_update_token(provider_id, token, provider_user_id):
-        session[provider_id + "_oauth_user_id"] = provider_user_id
-        session[provider_id + "_oauth_token"] = token
-
-        # Find this OAuth token in the database, or create it
-        query = ub.session.query(ub.OAuth).filter_by(
-            provider=provider_id,
-            provider_user_id=provider_user_id,
-        )
-        try:
-            oauth_entry = query.one()
-            # update token
-            oauth_entry.token = token
-        except NoResultFound:
-            oauth_entry = ub.OAuth(
-                provider=provider_id,
-                provider_user_id=provider_user_id,
-                token=token,
-            )
-        try:
-            ub.session.add(oauth_entry)
-            ub.session.commit()
-        except Exception as e:
-            log.exception(e)
-            ub.session.rollback()
-
-        # Disable Flask-Dance's default behavior for saving the OAuth token
-        # Value differrs depending on flask-dance version
-        return backend_resultcode
-
-
-    def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider_name):
-        query = ub.session.query(ub.OAuth).filter_by(
-            provider=provider_id,
-            provider_user_id=provider_user_id,
-        )
-        try:
-            oauth_entry = query.first()
-            # already bind with user, just login
-            if oauth_entry.user:
-                login_user(oauth_entry.user)
-                log.debug(u"You are now logged in as: '%s'", oauth_entry.user.nickname)
-                flash(_(u"you are now logged in as: '%(nickname)s'", nickname= oauth_entry.user.nickname),
-                      category="success")
-                return redirect(url_for('web.index'))
-            else:
-                # bind to current user
-                if current_user and current_user.is_authenticated:
-                    oauth_entry.user = current_user
-                    try:
-                        ub.session.add(oauth_entry)
-                        ub.session.commit()
-                        flash(_(u"Link to %(oauth)s Succeeded", oauth=provider_name), category="success")
-                        return redirect(url_for('web.profile'))
-                    except Exception as e:
-                        log.exception(e)
-                        ub.session.rollback()
-                else:
-                    flash(_(u"Login failed, No User Linked With OAuth Account"), category="error")
-                log.info('Login failed, No User Linked With OAuth Account')
-                return redirect(url_for('web.login'))
-                # return redirect(url_for('web.login'))
-                # if config.config_public_reg:
-                #   return redirect(url_for('web.register'))
-                # else:
-                #    flash(_(u"Public registration is not enabled"), category="error")
-                #    return redirect(url_for(redirect_url))
-        except (NoResultFound, AttributeError):
-            return redirect(url_for(redirect_url))
-
-
-    def get_oauth_status():
-        status = []
-        query = ub.session.query(ub.OAuth).filter_by(
-            user_id=current_user.id,
-        )
-        try:
-            oauths = query.all()
-            for oauth_entry in oauths:
-                status.append(int(oauth_entry.provider))
-            return status
-        except NoResultFound:
-            return None
-
-
-    def unlink_oauth(provider):
-        if request.host_url + 'me' != request.referrer:
-            pass
-        query = ub.session.query(ub.OAuth).filter_by(
-            provider=provider,
-            user_id=current_user.id,
-        )
-        try:
-            oauth_entry = query.one()
-            if current_user and current_user.is_authenticated:
-                oauth_entry.user = current_user
-                try:
-                    ub.session.delete(oauth_entry)
-                    ub.session.commit()
-                    logout_oauth_user()
-                    flash(_(u"Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success")
-                except Exception as e:
-                    log.exception(e)
-                    ub.session.rollback()
-                    flash(_(u"Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error")
-        except NoResultFound:
-            log.warning("oauth %s for user %d not found", provider, current_user.id)
-            flash(_(u"Not Linked to %(oauth)s", oauth=provider), category="error")
-        return redirect(url_for('web.profile'))
-
-
     # notify on OAuth provider error
     @oauth_error.connect_via(oauthblueprints[0]['blueprint'])
     def github_error(blueprint, error, error_description=None, error_uri=None):
@@ -305,39 +306,6 @@ if ub.oauth_support:
         )  # ToDo: Translate
         flash(msg, category="error")
 
-
-    @oauth.route('/link/github')
-    @oauth_required
-    def github_login():
-        if not github.authorized:
-            return redirect(url_for('github.login'))
-        account_info = github.get('/user')
-        if account_info.ok:
-            account_info_json = account_info.json()
-            return bind_oauth_or_register(oauthblueprints[0]['id'], account_info_json['id'], 'github.login', 'github')
-        flash(_(u"GitHub Oauth error, please retry later."), category="error")
-        return redirect(url_for('web.login'))
-
-
-    @oauth.route('/unlink/github', methods=["GET"])
-    @login_required
-    def github_login_unlink():
-        return unlink_oauth(oauthblueprints[0]['id'])
-
-
-    @oauth.route('/link/google')
-    @oauth_required
-    def google_login():
-        if not google.authorized:
-            return redirect(url_for("google.login"))
-        resp = google.get("/oauth2/v2/userinfo")
-        if resp.ok:
-            account_info_json = resp.json()
-            return bind_oauth_or_register(oauthblueprints[1]['id'], account_info_json['id'], 'google.login', 'google')
-        flash(_(u"Google Oauth error, please retry later."), category="error")
-        return redirect(url_for('web.login'))
-
-
     @oauth_error.connect_via(oauthblueprints[1]['blueprint'])
     def google_error(blueprint, error, error_description=None, error_uri=None):
         msg = (
@@ -352,7 +320,49 @@ if ub.oauth_support:
         flash(msg, category="error")
 
 
-    @oauth.route('/unlink/google', methods=["GET"])
-    @login_required
-    def google_login_unlink():
+@oauth.route('/link/github')
+@oauth_required
+def github_login():
+    if not github.authorized:
+        return redirect(url_for('github.login'))
+    try:
+        account_info = github.get('/user')
+        if account_info.ok:
+            account_info_json = account_info.json()
+            return bind_oauth_or_register(oauthblueprints[0]['id'], account_info_json['id'], 'github.login', 'github')
+        flash(_(u"GitHub Oauth error, please retry later."), category="error")
+        log.error("GitHub Oauth error, please retry later")
+    except (InvalidGrantError, TokenExpiredError) as e:
+        flash(_(u"GitHub Oauth error: {}").format(e), category="error")
+        log.error(e)
+    return redirect(url_for('web.login'))
+
+
+@oauth.route('/unlink/github', methods=["GET"])
+@login_required
+def github_login_unlink():
+    return unlink_oauth(oauthblueprints[0]['id'])
+
+
+@oauth.route('/link/google')
+@oauth_required
+def google_login():
+    if not google.authorized:
+        return redirect(url_for("google.login"))
+    try:
+        resp = google.get("/oauth2/v2/userinfo")
+        if resp.ok:
+            account_info_json = resp.json()
+            return bind_oauth_or_register(oauthblueprints[1]['id'], account_info_json['id'], 'google.login', 'google')
+        flash(_(u"Google Oauth error, please retry later."), category="error")
+        log.error("Google Oauth error, please retry later")
+    except (InvalidGrantError, TokenExpiredError) as e:
+        flash(_(u"Google Oauth error: {}").format(e), category="error")
+        log.error(e)
+    return redirect(url_for('web.login'))
+
+
+@oauth.route('/unlink/google', methods=["GET"])
+@login_required
+def google_login_unlink():
     return unlink_oauth(oauthblueprints[1]['id'])

cps/opds.py

@@ -27,13 +27,14 @@ from functools import wraps
 
 from flask import Blueprint, request, render_template, Response, g, make_response, abort
 from flask_login import current_user
-from sqlalchemy.sql.expression import func, text, or_, and_
+from sqlalchemy.sql.expression import func, text, or_, and_, true
 from werkzeug.security import check_password_hash
 
 from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages
 from .helper import get_download_link, get_book_cover
 from .pagination import Pagination
-from .web import render_read_books, download_required, load_user_from_request
+from .web import render_read_books
+from .usermanagement import load_user_from_request
 from flask_babel import gettext as _
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
@@ -93,7 +94,45 @@ def feed_cc_search(query):
 @opds.route("/opds/search", methods=["GET"])
 @requires_basic_auth_if_no_ano
 def feed_normal_search():
-    return feed_search(request.args.get("query").strip())
+    return feed_search(request.args.get("query", "").strip())
+
+
+@opds.route("/opds/books")
+@requires_basic_auth_if_no_ano
+def feed_booksindex():
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Books.sort, 1, 1)).label('id'))\
+        .filter(calibre_db.common_filters()).group_by(func.upper(func.substr(db.Books.sort, 1, 1))).all()
+
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_books',
+                               pagination=pagination)
+
+
+@opds.route("/opds/books/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_books(book_id):
+    off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Books.sort).startswith(book_id)
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
+                                                        db.Books,
+                                                        letter,
+                                                        [db.Books.sort])
+
+    return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
 @opds.route("/opds/new")
@@ -149,14 +188,41 @@ def feed_hot():
 @opds.route("/opds/author")
 @requires_basic_auth_if_no_ano
 def feed_authorindex():
-    off = request.args.get("offset") or 0
-    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
-        .filter(calibre_db.common_filters())\
-        .group_by(text('books_authors_link.author'))\
-        .order_by(db.Authors.sort).limit(config.config_books_per_page)\
-        .offset(off)
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('id'))\
+        .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all()
+
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(calibre_db.session.query(db.Authors).all()))
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_author',
+                               pagination=pagination)
+
+
+@opds.route("/opds/author/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_author(book_id):
+    off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Authors.sort).startswith(book_id)
+    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
+        .filter(calibre_db.common_filters()).filter(letter)\
+        .group_by(text('books_authors_link.author'))\
+        .order_by(db.Authors.sort)
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            entries.count())
+    entries = entries.limit(config.config_books_per_page).offset(off).all()
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_author', pagination=pagination)
 
 
@@ -200,17 +266,41 @@ def feed_publisher(book_id):
 @opds.route("/opds/category")
 @requires_basic_auth_if_no_ano
 def feed_categoryindex():
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('id'))\
+        .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all()
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_category',
+                               pagination=pagination)
+
+@opds.route("/opds/category/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_category(book_id):
     off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Tags.name).startswith(book_id)
     entries = calibre_db.session.query(db.Tags)\
         .join(db.books_tags_link)\
         .join(db.Books)\
-        .filter(calibre_db.common_filters())\
+        .filter(calibre_db.common_filters()).filter(letter)\
         .group_by(text('books_tags_link.tag'))\
-        .order_by(db.Tags.name)\
-        .offset(off)\
-        .limit(config.config_books_per_page)
+        .order_by(db.Tags.name)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(calibre_db.session.query(db.Tags).all()))
+                            entries.count())
+    entries = entries.offset(off).limit(config.config_books_per_page).all()
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_category', pagination=pagination)
 
 
@@ -228,16 +318,40 @@ def feed_category(book_id):
 @opds.route("/opds/series")
 @requires_basic_auth_if_no_ano
 def feed_seriesindex():
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('id'))\
+        .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_series',
+                               pagination=pagination)
+
+@opds.route("/opds/series/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_series(book_id):
     off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Series.sort).startswith(book_id)
     entries = calibre_db.session.query(db.Series)\
         .join(db.books_series_link)\
         .join(db.Books)\
-        .filter(calibre_db.common_filters())\
+        .filter(calibre_db.common_filters()).filter(letter)\
         .group_by(text('books_series_link.series'))\
-        .order_by(db.Series.sort)\
-        .offset(off).all()
+        .order_by(db.Series.sort)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(calibre_db.session.query(db.Series).all()))
+                            entries.count())
+    entries = entries.offset(off).limit(config.config_books_per_page).all()
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_series', pagination=pagination)
 
 
@@ -268,7 +382,7 @@ def feed_ratingindex():
                             len(entries))
     element = list()
     for entry in entries:
-        element.append(FeedObject(entry[0].id, "{} Stars".format(entry.name)))
+        element.append(FeedObject(entry[0].id, _("{} Stars").format(entry.name)))
     return render_xml_template('feed.xml', listelements=element, folder='opds.feed_ratings', pagination=pagination)
 
 
@@ -427,9 +541,14 @@ def check_auth(username, password):
             username = username.encode('windows-1252')
         except UnicodeEncodeError:
             username = username.encode('utf-8')
-    user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) ==
+    user = ub.session.query(ub.User).filter(func.lower(ub.User.name) ==
                                             username.decode('utf-8').lower()).first()
-    return bool(user and check_password_hash(str(user.password), password))
+    if bool(user and check_password_hash(str(user.password), password)):
+        return True
+    else:
+        ip_Address = request.headers.get('X-Forwarded-For', request.remote_addr)
+        log.warning('OPDS Login failed for user "%s" IP-address: %s', username.decode('utf-8'), ip_Address)
+        return False
 
 
 def authenticate():

cps/remotelogin.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2019 OzzieIsaacs, cervinko, jkrehm, bodybybuddha, ok11,
+#                            andy29485, idalin, Kyosfonica, wuqi, Kennyl, lemmsh,
+#                            falgh1, grunjol, csitko, ytils, xybydy, trasba, vrabe,
+#                            ruben-herold, marblepebble, JackED42, SiphonSquirrel,
+#                            apetresc, nanu-c, mutschler
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import json
+from datetime import datetime
+
+from flask import Blueprint, request, make_response, abort, url_for, flash, redirect
+from flask_login import login_required, current_user, login_user
+from flask_babel import gettext as _
+from sqlalchemy.sql.expression import true
+
+from . import config, logger, ub
+from .render_template import render_title_template
+
+try:
+    from functools import wraps
+except ImportError:
+    pass  # We're not using Python 3
+
+remotelogin = Blueprint('remotelogin', __name__)
+log = logger.create()
+
+
+def remote_login_required(f):
+    @wraps(f)
+    def inner(*args, **kwargs):
+        if config.config_remote_login:
+            return f(*args, **kwargs)
+        if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
+            data = {'status': 'error', 'message': 'Forbidden'}
+            response = make_response(json.dumps(data, ensure_ascii=False))
+            response.headers["Content-Type"] = "application/json; charset=utf-8"
+            return response, 403
+        abort(403)
+
+    return inner
+
+@remotelogin.route('/remote/login')
+@remote_login_required
+def remote_login():
+    auth_token = ub.RemoteAuthToken()
+    ub.session.add(auth_token)
+    ub.session_commit()
+    verify_url = url_for('remotelogin.verify_token', token=auth_token.auth_token, _external=true)
+    log.debug(u"Remot Login request with token: %s", auth_token.auth_token)
+    return render_title_template('remote_login.html', title=_(u"login"), token=auth_token.auth_token,
+                                 verify_url=verify_url, page="remotelogin")
+
+
+@remotelogin.route('/verify/<token>')
+@remote_login_required
+@login_required
+def verify_token(token):
+    auth_token = ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first()
+
+    # Token not found
+    if auth_token is None:
+        flash(_(u"Token not found"), category="error")
+        log.error(u"Remote Login token not found")
+        return redirect(url_for('web.index'))
+
+    # Token expired
+    elif datetime.now() > auth_token.expiration:
+        ub.session.delete(auth_token)
+        ub.session_commit()
+
+        flash(_(u"Token has expired"), category="error")
+        log.error(u"Remote Login token expired")
+        return redirect(url_for('web.index'))
+
+    # Update token with user information
+    auth_token.user_id = current_user.id
+    auth_token.verified = True
+    ub.session_commit()
+
+    flash(_(u"Success! Please return to your device"), category="success")
+    log.debug(u"Remote Login token for userid %s verified", auth_token.user_id)
+    return redirect(url_for('web.index'))
+
+
+@remotelogin.route('/ajax/verify_token', methods=['POST'])
+@remote_login_required
+def token_verified():
+    token = request.form['token']
+    auth_token = ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first()
+
+    data = {}
+
+    # Token not found
+    if auth_token is None:
+        data['status'] = 'error'
+        data['message'] = _(u"Token not found")
+
+    # Token expired
+    elif datetime.now() > auth_token.expiration:
+        ub.session.delete(auth_token)
+        ub.session_commit()
+
+        data['status'] = 'error'
+        data['message'] = _(u"Token has expired")
+
+    elif not auth_token.verified:
+        data['status'] = 'not_verified'
+
+    else:
+        user = ub.session.query(ub.User).filter(ub.User.id == auth_token.user_id).first()
+        login_user(user)
+
+        ub.session.delete(auth_token)
+        ub.session_commit("User {} logged in via remotelogin, token deleted".format(user.name))
+
+        data['status'] = 'success'
+        log.debug(u"Remote Login for userid %s succeded", user.id)
+        flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.name), category="success")
+
+    response = make_response(json.dumps(data, ensure_ascii=False))
+    response.headers["Content-Type"] = "application/json; charset=utf-8"
+
+    return response

cps/render_template.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2020 OzzieIsaacs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from flask import render_template
+from flask_babel import gettext as _
+from flask import g
+from werkzeug.local import LocalProxy
+from flask_login import current_user
+
+from . import config, constants, ub, logger, db, calibre_db
+from .ub import User
+
+
+log = logger.create()
+
+def get_sidebar_config(kwargs=None):
+    kwargs = kwargs or []
+    if 'content' in kwargs:
+        content = kwargs['content']
+        content = isinstance(content, (User, LocalProxy)) and not content.role_anonymous()
+    else:
+        content = 'conf' in kwargs
+    sidebar = list()
+    sidebar.append({"glyph": "glyphicon-book", "text": _('Books'), "link": 'web.index', "id": "new",
+                    "visibility": constants.SIDEBAR_RECENT, 'public': True, "page": "root",
+                    "show_text": _('Show recent books'), "config_show":False})
+    sidebar.append({"glyph": "glyphicon-fire", "text": _('Hot Books'), "link": 'web.books_list', "id": "hot",
+                    "visibility": constants.SIDEBAR_HOT, 'public': True, "page": "hot",
+                    "show_text": _('Show Hot Books'), "config_show": True})
+    if current_user.role_admin():
+        sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.download_list',
+                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+                        "page": "download", "show_text": _('Show Downloaded Books'),
+                        "config_show": content})
+    else:
+        sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.books_list',
+                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+                        "page": "download", "show_text": _('Show Downloaded Books'),
+                        "config_show": content})
+    sidebar.append(
+        {"glyph": "glyphicon-star", "text": _('Top Rated Books'), "link": 'web.books_list', "id": "rated",
+         "visibility": constants.SIDEBAR_BEST_RATED, 'public': True, "page": "rated",
+         "show_text": _('Show Top Rated Books'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-eye-open", "text": _('Read Books'), "link": 'web.books_list', "id": "read",
+                    "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous),
+                    "page": "read", "show_text": _('Show read and unread'), "config_show": content})
+    sidebar.append(
+        {"glyph": "glyphicon-eye-close", "text": _('Unread Books'), "link": 'web.books_list', "id": "unread",
+         "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous), "page": "unread",
+         "show_text": _('Show unread'), "config_show": False})
+    sidebar.append({"glyph": "glyphicon-random", "text": _('Discover'), "link": 'web.books_list', "id": "rand",
+                    "visibility": constants.SIDEBAR_RANDOM, 'public': True, "page": "discover",
+                    "show_text": _('Show random books'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-inbox", "text": _('Categories'), "link": 'web.category_list', "id": "cat",
+                    "visibility": constants.SIDEBAR_CATEGORY, 'public': True, "page": "category",
+                    "show_text": _('Show category selection'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-bookmark", "text": _('Series'), "link": 'web.series_list', "id": "serie",
+                    "visibility": constants.SIDEBAR_SERIES, 'public': True, "page": "series",
+                    "show_text": _('Show series selection'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-user", "text": _('Authors'), "link": 'web.author_list', "id": "author",
+                    "visibility": constants.SIDEBAR_AUTHOR, 'public': True, "page": "author",
+                    "show_text": _('Show author selection'), "config_show": True})
+    sidebar.append(
+        {"glyph": "glyphicon-text-size", "text": _('Publishers'), "link": 'web.publisher_list', "id": "publisher",
+         "visibility": constants.SIDEBAR_PUBLISHER, 'public': True, "page": "publisher",
+         "show_text": _('Show publisher selection'), "config_show":True})
+    sidebar.append({"glyph": "glyphicon-flag", "text": _('Languages'), "link": 'web.language_overview', "id": "lang",
+                    "visibility": constants.SIDEBAR_LANGUAGE, 'public': (g.user.filter_language() == 'all'),
+                    "page": "language",
+                    "show_text": _('Show language selection'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-star-empty", "text": _('Ratings'), "link": 'web.ratings_list', "id": "rate",
+                    "visibility": constants.SIDEBAR_RATING, 'public': True,
+                    "page": "rating", "show_text": _('Show ratings selection'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-file", "text": _('File formats'), "link": 'web.formats_list', "id": "format",
+                    "visibility": constants.SIDEBAR_FORMAT, 'public': True,
+                    "page": "format", "show_text": _('Show file formats selection'), "config_show": True})
+    sidebar.append(
+        {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived",
+         "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived",
+         "show_text": _('Show archived books'), "config_show": content})
+    sidebar.append(
+        {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_table', "id": "list",
+         "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list",
+         "show_text": _('Show Books List'), "config_show": content})
+
+    return sidebar
+
+def get_readbooks_ids():
+    if not config.config_read_column:
+        readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\
+            .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all()
+        return frozenset([x.book_id for x in readBooks])
+    else:
+        try:
+            readBooks = calibre_db.session.query(db.cc_classes[config.config_read_column])\
+                .filter(db.cc_classes[config.config_read_column].value == True).all()
+            return frozenset([x.book for x in readBooks])
+        except (KeyError, AttributeError):
+            log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
+            return []
+
+# Returns the template for rendering and includes the instance name
+def render_title_template(*args, **kwargs):
+    sidebar = get_sidebar_config(kwargs)
+    return render_template(instance=config.config_calibre_web_title, sidebar=sidebar,
+                           accept=constants.EXTENSIONS_UPLOAD, read_book_ids=get_readbooks_ids(),
+                           *args, **kwargs)

cps/server.py

@@ -22,6 +22,7 @@ import os
 import errno
 import signal
 import socket
+import subprocess  # nosec
 
 try:
     from gevent.pywsgi import WSGIServer
@@ -136,6 +137,64 @@ class WebServer(object):
 
         return sock, _readable_listen_address(*address)
 
+    @staticmethod
+    def _get_args_for_reloading():
+        """Determine how the script was executed, and return the args needed
+        to execute it again in a new process.
+        Code from https://github.com/pyload/pyload. Author GammaC0de, voulter
+        """
+        rv = [sys.executable]
+        py_script = sys.argv[0]
+        args = sys.argv[1:]
+        # Need to look at main module to determine how it was executed.
+        __main__ = sys.modules["__main__"]
+
+        # The value of __package__ indicates how Python was called. It may
+        # not exist if a setuptools script is installed as an egg. It may be
+        # set incorrectly for entry points created with pip on Windows.
+        if getattr(__main__, "__package__", None) is None or (
+            os.name == "nt"
+            and __main__.__package__ == ""
+            and not os.path.exists(py_script)
+            and os.path.exists("{}.exe".format(py_script))
+        ):
+            # Executed a file, like "python app.py".
+            py_script = os.path.abspath(py_script)
+
+            if os.name == "nt":
+                # Windows entry points have ".exe" extension and should be
+                # called directly.
+                if not os.path.exists(py_script) and os.path.exists("{}.exe".format(py_script)):
+                    py_script += ".exe"
+
+                if (
+                        os.path.splitext(sys.executable)[1] == ".exe"
+                        and os.path.splitext(py_script)[1] == ".exe"
+                ):
+                    rv.pop(0)
+
+            rv.append(py_script)
+        else:
+            # Executed a module, like "python -m module".
+            if sys.argv[0] == "-m":
+                args = sys.argv
+            else:
+                if os.path.isfile(py_script):
+                    # Rewritten by Python from "-m script" to "/path/to/script.py".
+                    py_module = __main__.__package__
+                    name = os.path.splitext(os.path.basename(py_script))[0]
+
+                    if name != "__main__":
+                        py_module += ".{}".format(name)
+                else:
+                    # Incorrectly rewritten by pydevd debugger from "-m script" to "script".
+                    py_module = py_script
+
+                rv.extend(("-m", py_module.lstrip(".")))
+
+        rv.extend(args)
+        return rv
+
     def _start_gevent(self):
         ssl_args = self.ssl_args or {}
 
@@ -192,18 +251,16 @@ class WebServer(object):
         finally:
             self.wsgiserver = None
 
+        # prevent irritating log of pending tasks message from asyncio
+        logger.get('asyncio').setLevel(logger.logging.CRITICAL)
+
         if not self.restart:
             log.info("Performing shutdown of Calibre-Web")
-            # prevent irritiating log of pending tasks message from asyncio
-            logger.get('asyncio').setLevel(logger.logging.CRITICAL)
             return True
 
         log.info("Performing restart of Calibre-Web")
-        arguments = list(sys.argv)
-        arguments.insert(0, sys.executable)
-        if os.name == 'nt':
-            arguments = ["\"%s\"" % a for a in arguments]
-        os.execv(sys.executable, arguments)
+        args = self._get_args_for_reloading()
+        subprocess.call(args, close_fds=True)  # nosec
         return True
 
     def _killServer(self, __, ___):

cps/services/SyncToken.py

@@ -22,6 +22,7 @@ from base64 import b64decode, b64encode
 from jsonschema import validate, exceptions, __version__
 from datetime import datetime
 try:
+    # pylint: disable=unused-import
     from urllib import unquote
 except ImportError:
     from urllib.parse import unquote
@@ -64,7 +65,7 @@ class SyncToken:
         books_last_modified: Datetime representing the last modified book that the device knows about.
     """
 
-    SYNC_TOKEN_HEADER = "x-kobo-synctoken"
+    SYNC_TOKEN_HEADER = "x-kobo-synctoken"  # nosec
     VERSION = "1-1-0"
     LAST_MODIFIED_ADDED_VERSION = "1-1-0"
     MIN_VERSION = "1-0-0"
@@ -85,6 +86,7 @@ class SyncToken:
             "archive_last_modified": {"type": "string"},
             "reading_state_last_modified": {"type": "string"},
             "tags_last_modified": {"type": "string"},
+            "books_last_id": {"type": "integer", "optional": True}
         },
     }
 
@@ -96,18 +98,20 @@ class SyncToken:
         archive_last_modified=datetime.min,
         reading_state_last_modified=datetime.min,
         tags_last_modified=datetime.min,
-    ):
+        books_last_id=-1
+    ):  # nosec
         self.raw_kobo_store_token = raw_kobo_store_token
         self.books_last_created = books_last_created
         self.books_last_modified = books_last_modified
         self.archive_last_modified = archive_last_modified
         self.reading_state_last_modified = reading_state_last_modified
         self.tags_last_modified = tags_last_modified
+        self.books_last_id = books_last_id
 
     @staticmethod
     def from_headers(headers):
         sync_token_header = headers.get(SyncToken.SYNC_TOKEN_HEADER, "")
-        if sync_token_header == "":
+        if sync_token_header == "":  # nosec
             return SyncToken()
 
         # On the first sync from a Kobo device, we may receive the SyncToken
@@ -137,9 +141,12 @@ class SyncToken:
             archive_last_modified = get_datetime_from_json(data_json, "archive_last_modified")
             reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified")
             tags_last_modified = get_datetime_from_json(data_json, "tags_last_modified")
+            books_last_id = data_json["books_last_id"]
         except TypeError:
             log.error("SyncToken timestamps don't parse to a datetime.")
             return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
+        except KeyError:
+            books_last_id = -1
 
         return SyncToken(
             raw_kobo_store_token=raw_kobo_store_token,
@@ -147,7 +154,8 @@ class SyncToken:
             books_last_modified=books_last_modified,
             archive_last_modified=archive_last_modified,
             reading_state_last_modified=reading_state_last_modified,
-            tags_last_modified=tags_last_modified
+            tags_last_modified=tags_last_modified,
+            books_last_id=books_last_id
         )
 
     def set_kobo_store_header(self, store_headers):
@@ -170,7 +178,8 @@ class SyncToken:
                 "books_last_created": to_epoch_timestamp(self.books_last_created),
                 "archive_last_modified": to_epoch_timestamp(self.archive_last_modified),
                 "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified),
-                "tags_last_modified": to_epoch_timestamp(self.tags_last_modified)
+                "tags_last_modified": to_epoch_timestamp(self.tags_last_modified),
+                "books_last_id":self.books_last_id
             },
         }
         return b64encode_json(token)

cps/services/__init__.py

@@ -45,3 +45,9 @@ except ImportError as err:
     log.debug("Cannot import SyncToken, syncing books with Kobo Devices will not work: %s", err)
     kobo = None
     SyncToken = None
+
+try:
+    from . import gmail
+except ImportError as err:
+    log.debug("Cannot import gmail, sending books via Gmail Oauth2 Verification will not work: %s", err)
+    gmail = None

cps/services/gmail.py

@@ -0,0 +1,83 @@
+from __future__ import print_function
+import os.path
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
+from google.oauth2.credentials import Credentials
+
+from datetime import datetime
+import base64
+from flask_babel import gettext as _
+from ..constants import BASE_DIR
+from .. import logger
+
+
+log = logger.create()
+
+SCOPES = ['openid', 'https://www.googleapis.com/auth/gmail.send', 'https://www.googleapis.com/auth/userinfo.email']
+
+def setup_gmail(token):
+    # If there are no (valid) credentials available, let the user log in.
+    creds = None
+    if "token" in token:
+        creds = Credentials(
+            token=token['token'],
+            refresh_token=token['refresh_token'],
+            token_uri=token['token_uri'],
+            client_id=token['client_id'],
+            client_secret=token['client_secret'],
+            scopes=token['scopes'],
+        )
+        creds.expiry = datetime.fromisoformat(token['expiry'])
+
+    if not creds or not creds.valid:
+        # don't forget to dump one more time after the refresh
+        # also, some file-locking routines wouldn't be needless
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            cred_file = os.path.join(BASE_DIR, 'gmail.json')
+            if not os.path.exists(cred_file):
+                raise Exception(_("Found no valid gmail.json file with OAuth information"))
+            flow = InstalledAppFlow.from_client_secrets_file(
+                os.path.join(BASE_DIR, 'gmail.json'), SCOPES)
+            creds = flow.run_local_server(port=0)
+            user_info = get_user_info(creds)
+        return {
+            'token': creds.token,
+            'refresh_token': creds.refresh_token,
+            'token_uri': creds.token_uri,
+            'client_id': creds.client_id,
+            'client_secret': creds.client_secret,
+            'scopes': creds.scopes,
+            'expiry': creds.expiry.isoformat(),
+            'email': user_info
+        }
+    return {}
+
+def get_user_info(credentials):
+    user_info_service = build(serviceName='oauth2', version='v2',credentials=credentials)
+    user_info = user_info_service.userinfo().get().execute()
+    return user_info.get('email', "")
+
+def send_messsage(token, msg):
+    log.debug("Start sending email via Gmail")
+    creds = Credentials(
+        token=token['token'],
+        refresh_token=token['refresh_token'],
+        token_uri=token['token_uri'],
+        client_id=token['client_id'],
+        client_secret=token['client_secret'],
+        scopes=token['scopes'],
+    )
+    creds.expiry = datetime.fromisoformat(token['expiry'])
+    if creds and creds.expired and creds.refresh_token:
+        creds.refresh(Request())
+    service = build('gmail', 'v1', credentials=creds)
+    message_as_bytes = msg.as_bytes()  # the message should converted from string to bytes.
+    message_as_base64 = base64.urlsafe_b64encode(message_as_bytes)  # encode in base64 (printable letters coding)
+    raw = message_as_base64.decode()  # convert to something  JSON serializable
+    body = {'raw': raw}
+
+    (service.users().messages().send(userId='me', body=body).execute())
+    log.debug("Email send successfully via Gmail")

cps/services/worker.py

@@ -45,7 +45,7 @@ class ImprovedQueue(queue.Queue):
         with self.mutex:
             return list(self.queue)
 
-#Class for all worker tasks in the background
+# Class for all worker tasks in the background
 class WorkerThread(threading.Thread):
     _instance = None
 
@@ -69,6 +69,7 @@ class WorkerThread(threading.Thread):
     def add(cls, user, task):
         ins = cls.getInstance()
         ins.num += 1
+        log.debug("Add Task for user: {}: {}".format(user, task))
         ins.queue.put(QueuedTask(
             num=ins.num,
             user=user,
@@ -110,7 +111,7 @@ class WorkerThread(threading.Thread):
                 # We don't use a daemon here because we don't want the tasks to just be abruptly halted, leading to
                 # possible file / database corruption
                 item = self.queue.get(timeout=1)
-            except queue.Empty as ex:
+            except queue.Empty:
                 time.sleep(1)
                 continue
 
@@ -159,9 +160,9 @@ class CalibreTask:
         # catch any unhandled exceptions in a task and automatically fail it
         try:
             self.run(*args)
-        except Exception as e:
-            self._handleError(str(e))
-            log.exception(e)
+        except Exception as ex:
+            self._handleError(str(ex))
+            log.debug_or_exception(ex)
 
         self.end_time = datetime.now()
 
@@ -210,7 +211,6 @@ class CalibreTask:
         self._progress = x
 
     def _handleError(self, error_message):
-        log.exception(error_message)
         self.stat = STAT_FAIL
         self.progress = 1
         self.error = error_message

cps/shelf.py

@@ -22,15 +22,17 @@
 
 from __future__ import division, print_function, unicode_literals
 from datetime import datetime
+import sys
 
 from flask import Blueprint, request, flash, redirect, url_for
 from flask_babel import gettext as _
 from flask_login import login_required, current_user
-from sqlalchemy.sql.expression import func
+from sqlalchemy.sql.expression import func, true
 from sqlalchemy.exc import OperationalError, InvalidRequestError
 
-from . import logger, ub, calibre_db
-from .web import login_required_if_no_ano, render_title_template
+from . import logger, ub, calibre_db, db
+from .render_template import render_title_template
+from .usermanagement import login_required_if_no_ano
 
 
 shelf = Blueprint('shelf', __name__)
@@ -97,12 +99,14 @@ def add_to_shelf(shelf_id, book_id):
         ub.session.commit()
     except (OperationalError, InvalidRequestError):
         ub.session.rollback()
+        log.error("Settings DB is not Writeable")
         flash(_(u"Settings DB is not Writeable"), category="error")
         if "HTTP_REFERER" in request.environ:
             return redirect(request.environ["HTTP_REFERER"])
         else:
             return redirect(url_for('web.index'))
     if not xhr:
+        log.debug("Book has been added to shelf: {}".format(shelf.name))
         flash(_(u"Book has been added to shelf: %(sname)s", sname=shelf.name), category="success")
         if "HTTP_REFERER" in request.environ:
             return redirect(request.environ["HTTP_REFERER"])
@@ -121,6 +125,7 @@ def search_to_shelf(shelf_id):
         return redirect(url_for('web.index'))
 
     if not check_shelf_edit_permissions(shelf):
+        log.warning("You are not allowed to add a book to the the shelf: {}".format(shelf.name))
         flash(_(u"You are not allowed to add a book to the the shelf: %(name)s", name=shelf.name), category="error")
         return redirect(url_for('web.index'))
 
@@ -138,18 +143,14 @@ def search_to_shelf(shelf_id):
             books_for_shelf = ub.searched_ids[current_user.id]
 
         if not books_for_shelf:
-            log.error("Books are already part of %s", shelf)
+            log.error("Books are already part of {}".format(shelf.name))
             flash(_(u"Books are already part of the shelf: %(name)s", name=shelf.name), category="error")
             return redirect(url_for('web.index'))
 
-        maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first()
-        if maxOrder[0] is None:
-            maxOrder = 0
-        else:
-            maxOrder = maxOrder[0]
+        maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first()[0] or 0
 
         for book in books_for_shelf:
-            maxOrder = maxOrder + 1
+            maxOrder += 1
             shelf.books.append(ub.BookShelf(shelf=shelf.id, book_id=book, order=maxOrder))
         shelf.last_modified = datetime.utcnow()
         try:
@@ -158,8 +159,10 @@ def search_to_shelf(shelf_id):
             flash(_(u"Books have been added to shelf: %(sname)s", sname=shelf.name), category="success")
         except (OperationalError, InvalidRequestError):
             ub.session.rollback()
-            flash(_(u"Settings DB is not Writeable"), category="error")
+            log.error("Settings DB is not Writeable")
+            flash(_("Settings DB is not Writeable"), category="error")
     else:
+        log.error("Could not add books to shelf: {}".format(shelf.name))
         flash(_(u"Could not add books to shelf: %(sname)s", sname=shelf.name), category="error")
     return redirect(url_for('web.index'))
 
@@ -170,7 +173,7 @@ def remove_from_shelf(shelf_id, book_id):
     xhr = request.headers.get('X-Requested-With') == 'XMLHttpRequest'
     shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
     if shelf is None:
-        log.error("Invalid shelf specified: %s", shelf_id)
+        log.error("Invalid shelf specified: {}".format(shelf_id))
         if not xhr:
             return redirect(url_for('web.index'))
         return "Invalid shelf specified", 400
@@ -199,7 +202,8 @@ def remove_from_shelf(shelf_id, book_id):
             ub.session.commit()
         except (OperationalError, InvalidRequestError):
             ub.session.rollback()
-            flash(_(u"Settings DB is not Writeable"), category="error")
+            log.error("Settings DB is not Writeable")
+            flash(_("Settings DB is not Writeable"), category="error")
             if "HTTP_REFERER" in request.environ:
                 return redirect(request.environ["HTTP_REFERER"])
             else:
@@ -213,6 +217,7 @@ def remove_from_shelf(shelf_id, book_id):
         return "", 204
     else:
         if not xhr:
+            log.warning("You are not allowed to remove a book from shelf: {}".format(shelf.name))
             flash(_(u"Sorry you are not allowed to remove a book from this shelf: %(sname)s", sname=shelf.name),
                   category="error")
             return redirect(url_for('web.index'))
@@ -223,96 +228,79 @@ def remove_from_shelf(shelf_id, book_id):
 @login_required
 def create_shelf():
     shelf = ub.Shelf()
-    if request.method == "POST":
-        to_save = request.form.to_dict()
-        if "is_public" in to_save:
-            shelf.is_public = 1
-        shelf.name = to_save["title"]
-        shelf.user_id = int(current_user.id)
-
-        is_shelf_name_unique = False
-        if shelf.is_public == 1:
-            is_shelf_name_unique = ub.session.query(ub.Shelf) \
-                .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 1)) \
-                .first() is None
-
-            if not is_shelf_name_unique:
-                flash(_(u"A public shelf with the name '%(title)s' already exists.", title=to_save["title"]),
-                      category="error")
-        else:
-            is_shelf_name_unique = ub.session.query(ub.Shelf) \
-                .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 0) &
-                        (ub.Shelf.user_id == int(current_user.id)))\
-                                       .first() is None
-
-            if not is_shelf_name_unique:
-                flash(_(u"A private shelf with the name '%(title)s' already exists.", title=to_save["title"]),
-                      category="error")
-
-        if is_shelf_name_unique:
-            try:
-                ub.session.add(shelf)
-                ub.session.commit()
-                flash(_(u"Shelf %(title)s created", title=to_save["title"]), category="success")
-                return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id))
-            except (OperationalError, InvalidRequestError):
-                ub.session.rollback()
-                flash(_(u"Settings DB is not Writeable"), category="error")
-            except Exception:
-                ub.session.rollback()
-                flash(_(u"There was an error"), category="error")
-        return render_title_template('shelf_edit.html', shelf=shelf, title=_(u"Create a Shelf"), page="shelfcreate")
-    else:
-        return render_title_template('shelf_edit.html', shelf=shelf, title=_(u"Create a Shelf"), page="shelfcreate")
+    return create_edit_shelf(shelf, title=_(u"Create a Shelf"), page="shelfcreate")
 
 
 @shelf.route("/shelf/edit/<int:shelf_id>", methods=["GET", "POST"])
 @login_required
 def edit_shelf(shelf_id):
     shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
+    return create_edit_shelf(shelf, title=_(u"Edit a shelf"), page="shelfedit", shelf_id=shelf_id)
+
+
+# if shelf ID is set, we are editing a shelf
+def create_edit_shelf(shelf, title, page, shelf_id=False):
     if request.method == "POST":
         to_save = request.form.to_dict()
+        if "is_public" in to_save:
+            shelf.is_public = 1
+        else:
+            shelf.is_public = 0
+        if check_shelf_is_unique(shelf, to_save, shelf_id):
+            shelf.name = to_save["title"]
+            # shelf.last_modified = datetime.utcnow()
+            if not shelf_id:
+                shelf.user_id = int(current_user.id)
+                ub.session.add(shelf)
+                shelf_action = "created"
+                flash_text = _(u"Shelf %(title)s created", title=to_save["title"])
+            else:
+                shelf_action = "changed"
+                flash_text = _(u"Shelf %(title)s changed", title=to_save["title"])
+            try:
+                ub.session.commit()
+                log.info(u"Shelf {} {}".format(to_save["title"], shelf_action))
+                flash(flash_text, category="success")
+                return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id))
+            except (OperationalError, InvalidRequestError) as ex:
+                ub.session.rollback()
+                log.debug_or_exception(ex)
+                log.error("Settings DB is not Writeable")
+                flash(_("Settings DB is not Writeable"), category="error")
+            except Exception as ex:
+                ub.session.rollback()
+                log.debug_or_exception(ex)
+                flash(_(u"There was an error"), category="error")
+    return render_title_template('shelf_edit.html', shelf=shelf, title=title, page=page)
 
-        is_shelf_name_unique = False
+
+def check_shelf_is_unique(shelf, to_save, shelf_id=False):
+    if shelf_id:
+        ident = ub.Shelf.id != shelf_id
+    else:
+        ident = true()
     if shelf.is_public == 1:
         is_shelf_name_unique = ub.session.query(ub.Shelf) \
                                    .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 1)) \
-                .filter(ub.Shelf.id != shelf_id) \
+                                   .filter(ident) \
                                    .first() is None
 
         if not is_shelf_name_unique:
+            log.error("A public shelf with the name '{}' already exists.".format(to_save["title"]))
             flash(_(u"A public shelf with the name '%(title)s' already exists.", title=to_save["title"]),
                   category="error")
     else:
         is_shelf_name_unique = ub.session.query(ub.Shelf) \
                                    .filter((ub.Shelf.name == to_save["title"]) & (ub.Shelf.is_public == 0) &
-                        (ub.Shelf.user_id == int(current_user.id)))\
-                                       .filter(ub.Shelf.id != shelf_id)\
+                                           (ub.Shelf.user_id == int(current_user.id))) \
+                                   .filter(ident) \
                                    .first() is None
 
         if not is_shelf_name_unique:
+            log.error("A private shelf with the name '{}' already exists.".format(to_save["title"]))
             flash(_(u"A private shelf with the name '%(title)s' already exists.", title=to_save["title"]),
                   category="error")
-
-        if is_shelf_name_unique:
-            shelf.name = to_save["title"]
-            shelf.last_modified = datetime.utcnow()
-            if "is_public" in to_save:
-                shelf.is_public = 1
-            else:
-                shelf.is_public = 0
-            try:
-                ub.session.commit()
-                flash(_(u"Shelf %(title)s changed", title=to_save["title"]), category="success")
-            except (OperationalError, InvalidRequestError):
-                ub.session.rollback()
-                flash(_(u"Settings DB is not Writeable"), category="error")
-            except Exception:
-                ub.session.rollback()
-                flash(_(u"There was an error"), category="error")
-        return render_title_template('shelf_edit.html', shelf=shelf, title=_(u"Edit a shelf"), page="shelfedit")
-    else:
-        return render_title_template('shelf_edit.html', shelf=shelf, title=_(u"Edit a shelf"), page="shelfedit")
+    return is_shelf_name_unique
 
 
 def delete_shelf_helper(cur_shelf):
@@ -322,9 +310,7 @@ def delete_shelf_helper(cur_shelf):
     ub.session.delete(cur_shelf)
     ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete()
     ub.session.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id))
-    ub.session.commit()
-    log.info("successfully deleted %s", cur_shelf)
-
+    ub.session_commit("successfully deleted Shelf {}".format(cur_shelf.name))
 
 
 @shelf.route("/shelf/delete/<int:shelf_id>")
@@ -333,44 +319,25 @@ def delete_shelf(shelf_id):
     cur_shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
     try:
         delete_shelf_helper(cur_shelf)
-    except (OperationalError, InvalidRequestError):
+    except InvalidRequestError:
         ub.session.rollback()
-        flash(_(u"Settings DB is not Writeable"), category="error")
+        log.error("Settings DB is not Writeable")
+        flash(_("Settings DB is not Writeable"), category="error")
     return redirect(url_for('web.index'))
 
 
-@shelf.route("/shelf/<int:shelf_id>", defaults={'shelf_type': 1})
-@shelf.route("/shelf/<int:shelf_id>/<int:shelf_type>")
+@shelf.route("/simpleshelf/<int:shelf_id>")
 @login_required_if_no_ano
-def show_shelf(shelf_type, shelf_id):
-    shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
+def show_simpleshelf(shelf_id):
+    return render_show_shelf(2, shelf_id, 1, None)
 
-    result = list()
-    # user is allowed to access shelf
-    if shelf and check_shelf_view_permissions(shelf):
-        page = "shelf.html" if shelf_type == 1 else 'shelfdown.html'
 
-        books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\
-            .order_by(ub.BookShelf.order.asc()).all()
-        for book in books_in_shelf:
-            cur_book = calibre_db.get_filtered_book(book.book_id)
-            if cur_book:
-                result.append(cur_book)
-            else:
-                cur_book = calibre_db.get_book(book.book_id)
-                if not cur_book:
-                    log.info('Not existing book %s in %s deleted', book.book_id, shelf)
-                    try:
-                        ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete()
-                        ub.session.commit()
-                    except (OperationalError, InvalidRequestError):
-                        ub.session.rollback()
-                        flash(_(u"Settings DB is not Writeable"), category="error")
-        return render_title_template(page, entries=result, title=_(u"Shelf: '%(name)s'", name=shelf.name),
-                                     shelf=shelf, page="shelf")
-    else:
-        flash(_(u"Error opening shelf. Shelf does not exist or is not accessible"), category="error")
-        return redirect(url_for("web.index"))
+@shelf.route("/shelf/<int:shelf_id>", defaults={"sort_param": "order", 'page': 1})
+@shelf.route("/shelf/<int:shelf_id>/<sort_param>", defaults={'page': 1})
+@shelf.route("/shelf/<int:shelf_id>/<sort_param>/<int:page>")
+@login_required_if_no_ano
+def show_shelf(shelf_id, sort_param, page):
+    return render_show_shelf(1, shelf_id, page, sort_param)
 
 
 @shelf.route("/shelf/order/<int:shelf_id>", methods=["GET", "POST"])
@@ -389,27 +356,86 @@ def order_shelf(shelf_id):
             ub.session.commit()
         except (OperationalError, InvalidRequestError):
             ub.session.rollback()
-            flash(_(u"Settings DB is not Writeable"), category="error")
+            log.error("Settings DB is not Writeable")
+            flash(_("Settings DB is not Writeable"), category="error")
 
     shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
     result = list()
     if shelf and check_shelf_view_permissions(shelf):
-        books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
-            .order_by(ub.BookShelf.order.asc()).all()
-        for book in books_in_shelf2:
-            cur_book = calibre_db.get_filtered_book(book.book_id)
-            if cur_book:
-                result.append({'title': cur_book.title,
-                               'id': cur_book.id,
-                               'author': cur_book.authors,
-                               'series': cur_book.series,
-                               'series_index': cur_book.series_index})
-            else:
-                cur_book = calibre_db.get_book(book.book_id)
-                result.append({'title': _('Hidden Book'),
-                               'id': cur_book.id,
-                               'author': [],
-                               'series': []})
+        result = calibre_db.session.query(db.Books)\
+            .join(ub.BookShelf,ub.BookShelf.book_id == db.Books.id , isouter=True) \
+            .add_columns(calibre_db.common_filters().label("visible")) \
+            .filter(ub.BookShelf.shelf == shelf_id).order_by(ub.BookShelf.order.asc()).all()
     return render_title_template('shelf_order.html', entries=result,
                                  title=_(u"Change order of Shelf: '%(name)s'", name=shelf.name),
                                  shelf=shelf, page="shelforder")
+
+
+def change_shelf_order(shelf_id, order):
+    result = calibre_db.session.query(db.Books).join(ub.BookShelf,ub.BookShelf.book_id == db.Books.id)\
+        .filter(ub.BookShelf.shelf == shelf_id).order_by(*order).all()
+    for index, entry in enumerate(result):
+        book = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
+            .filter(ub.BookShelf.book_id == entry.id).first()
+        book.order = index
+    ub.session_commit("Shelf-id:{} - Order changed".format(shelf_id))
+
+
+def render_show_shelf(shelf_type, shelf_id, page_no, sort_param):
+    shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
+
+    # check user is allowed to access shelf
+    if shelf and check_shelf_view_permissions(shelf):
+
+        if shelf_type == 1:
+            # order = [ub.BookShelf.order.asc()]
+            if sort_param == 'pubnew':
+                change_shelf_order(shelf_id, [db.Books.pubdate.desc()])
+            if sort_param == 'pubold':
+                change_shelf_order(shelf_id, [db.Books.pubdate])
+            if sort_param == 'abc':
+                change_shelf_order(shelf_id, [db.Books.sort])
+            if sort_param == 'zyx':
+                change_shelf_order(shelf_id, [db.Books.sort.desc()])
+            if sort_param == 'new':
+                change_shelf_order(shelf_id, [db.Books.timestamp.desc()])
+            if sort_param == 'old':
+                change_shelf_order(shelf_id, [db.Books.timestamp])
+            if sort_param == 'authaz':
+                change_shelf_order(shelf_id, [db.Books.author_sort.asc()])
+            if sort_param == 'authza':
+                change_shelf_order(shelf_id, [db.Books.author_sort.desc()])
+            page = "shelf.html"
+            pagesize = 0
+        else:
+            pagesize = sys.maxsize
+            page = 'shelfdown.html'
+
+        result, __, pagination = calibre_db.fill_indexpage(page_no, pagesize,
+                                                            db.Books,
+                                                            ub.BookShelf.shelf == shelf_id,
+                                                            [ub.BookShelf.order.asc()],
+                                                            ub.BookShelf,ub.BookShelf.book_id == db.Books.id)
+        # delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web
+        wrong_entries = calibre_db.session.query(ub.BookShelf)\
+            .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True)\
+            .filter(db.Books.id == None).all()
+        for entry in wrong_entries:
+            log.info('Not existing book {} in {} deleted'.format(entry.book_id, shelf))
+            try:
+                ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == entry.book_id).delete()
+                ub.session.commit()
+            except (OperationalError, InvalidRequestError):
+                ub.session.rollback()
+                log.error("Settings DB is not Writeable")
+                flash(_("Settings DB is not Writeable"), category="error")
+
+        return render_title_template(page,
+                                     entries=result,
+                                     pagination=pagination,
+                                     title=_(u"Shelf: '%(name)s'", name=shelf.name),
+                                     shelf=shelf,
+                                     page="shelf")
+    else:
+        flash(_(u"Error opening shelf. Shelf does not exist or is not accessible"), category="error")
+        return redirect(url_for("web.index"))

cps/static/css/caliBlur.css

@@ -240,7 +240,7 @@ body.blur .row-fluid .col-sm-10 {
 
 .col-sm-10 .book-meta > div.btn-toolbar:after {
     content: '';
-    direction: block;
+    direction: ltr;
     position: fixed;
     top: 120px;
     right: 0;
@@ -398,20 +398,17 @@ body.blur .row-fluid .col-sm-10 {
 
 .shelforder #sortTrue > div:hover {
     background-color: hsla(0, 0%, 100%, .06) !important;
-    cursor: move;
     cursor: grab;
-    cursor: -webkit-grab;
     color: #eee
 }
 
 .shelforder #sortTrue > div:active {
     cursor: grabbing;
-    cursor: -webkit-grabbing
 }
 
 .shelforder #sortTrue > div:before {
     content: "\EA53";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     margin-right: 30px;
     margin-left: 15px;
     vertical-align: bottom;
@@ -446,7 +443,7 @@ body.blur .row-fluid .col-sm-10 {
 
 body.shelforder > div.container-fluid > div.row-fluid > div.col-sm-10:before {
     content: "\e155";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -494,7 +491,7 @@ body.shelforder > div.container-fluid > div.row-fluid > div.col-sm-10:before {
 }
 
 #have_read_cb + label:before, #have_read_cb:checked + label:before {
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-size: 16px;
     height: 40px;
     width: 60px;
@@ -550,13 +547,12 @@ body.shelforder > div.container-fluid > div.row-fluid > div.col-sm-10:before {
     height: 60px;
     width: 50px;
     cursor: pointer;
-    margin: 0;
     display: inline-block;
-    margin-top: -4px;
+    margin: -4px 0 0;
 }
 
 #archived_cb + label:before, #archived_cb:checked + label:before {
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-size: 16px;
     height: 40px;
     width: 60px;
@@ -581,10 +577,6 @@ body.shelforder > div.container-fluid > div.row-fluid > div.col-sm-10:before {
     color: hsla(0, 0%, 100%, .7)
 }
 
-div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > .downloadBtn {
-    border-left: 2px solid rgba(0, 0, 0, .15)
-}
-
 div[aria-label="Edit/Delete book"] > .btn {
     width: 50px;
     height: 60px;
@@ -618,7 +610,7 @@ div[aria-label="Edit/Delete book"] > .btn > span {
 
 div[aria-label="Edit/Delete book"] > .btn > span:before {
     content: "\EA5d";
-    font-family: plex-icons;
+    font-family: plex-icons, serif;
     font-size: 20px;
     padding: 16px 15px;
     display: inline-block;
@@ -760,7 +752,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > div.navbar-header > a
 
 .home-btn {
     color: hsla(0, 0%, 100%, .7);
-    line-height: 34.29px;
+    line-height: 34px;
     margin: 0;
     padding: 0;
     position: absolute;
@@ -770,7 +762,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > div.navbar-header > a
 
 .home-btn > a {
     color: rgba(255, 255, 255, .7);
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     line-height: 60px;
     position: relative;
     text-align: center;
@@ -800,7 +792,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > div.home-btn > a:hove
 
 .glyphicon-search:before {
     content: "\EA4F";
-    font-family: plex-icons
+    font-family: plex-icons, serif
 }
 
 #nav_about:after, .profileDrop > span:after, .profileDrop > span:before {
@@ -966,7 +958,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > form.search-focus > d
 
 #form-upload .form-group .btn:before {
     content: "\e043";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -991,7 +983,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > form.search-focus > d
 #form-upload .form-group .btn:after {
     content: "\EA13";
     position: absolute;
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-size: 8px;
     background: #3c444a;
     color: hsla(0, 0%, 100%, .7);
@@ -1019,7 +1011,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > form.search-focus > d
     text-transform: none;
     font-weight: 400;
     font-style: normal;
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     -webkit-font-smoothing: antialiased;
     -moz-osx-font-smoothing: grayscale;
     line-height: 1;
@@ -1075,7 +1067,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > form.search-focus > d
 
 body > div.navbar.navbar-default.navbar-static-top > div > form > div > span > button:before {
     content: "\EA32";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     color: #eee;
     background: #555;
     font-size: 10px;
@@ -1097,7 +1089,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > form > div > span > b
 body > div.navbar.navbar-default.navbar-static-top > div > form:before {
     content: "\EA4F";
     display: block;
-    font-family: plex-icons;
+    font-family: plex-icons, serif;
     position: absolute;
     color: #eee;
     font-weight: 400;
@@ -1120,7 +1112,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > form:before {
 body > div.navbar.navbar-default.navbar-static-top > div > form.search-focus > div > span.input-group-btn:before {
     content: "\EA4F";
     display: block;
-    font-family: plex-icons;
+    font-family: plex-icons, serif;
     position: absolute;
     left: -298px;
     top: 8px;
@@ -1193,7 +1185,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > div.navbar-collapse.c
 
 body > div.navbar.navbar-default.navbar-static-top > div > div.navbar-collapse.collapse > ul > li > #top_admin > .glyphicon-dashboard::before {
     content: "\EA31";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-size: 20px
 }
 
@@ -1272,7 +1264,7 @@ body > div.container-fluid > div > div.col-sm-10 > div > form > a:hover {
     user-select: none
 }
 
-.navigation li, .navigation li:not(ul>li) {
+.navigation li, .navigation li:not(ul > li) {
     border-radius: 0 4px 4px 0
 }
 
@@ -1352,32 +1344,32 @@ body > div.container-fluid > div > div.col-sm-10 > div > form > a:hover {
 
 #nav_hot .glyphicon-fire::before {
     content: "\1F525";
-    font-family: glyphicons regular
+    font-family: glyphicons regular, serif
 }
 
 .glyphicon-star:before {
     content: "\EA10";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #nav_rand .glyphicon-random::before {
     content: "\EA44";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 .glyphicon-list::before {
     content: "\EA4D";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #nav_about .glyphicon-info-sign::before {
     content: "\EA26";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #nav_cat .glyphicon-inbox::before, .glyphicon-tags::before {
     content: "\E067";
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     margin-left: 2px
 }
 
@@ -1423,7 +1415,7 @@ body > div.container-fluid > div > div.col-sm-10 > div > form > a:hover {
 
 .navigation .create-shelf a:before {
     content: "\EA13";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-size: 100%;
     padding-right: 10px;
     vertical-align: middle
@@ -1473,7 +1465,7 @@ body > div.container-fluid > div > div.col-sm-10 > div > form > a:hover {
 
 #books > .cover > a:before, #books_rand > .cover > a:before, .book.isotope-item > .cover > a:before, body > div.container-fluid > div.row-fluid > div.col-sm-10 > div.discover > form > div.col-sm-12 > div.col-sm-12 > div.col-sm-2 > a:before {
     content: "\e352";
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     background: var(--color-secondary);
     border-radius: 50%;
     font-weight: 400;
@@ -1521,8 +1513,8 @@ body > div.container-fluid > div.row-fluid > div.col-sm-10 > div.discover > form
     top: 0;
     left: 0;
     opacity: 0;
-    background: -webkit-radial-gradient(50% 50%, farthest-corner, rgba(50, 50, 50, .5) 50%, #323232 100%);
-    background: -o-radial-gradient(50% 50%, farthest-corner, rgba(50, 50, 50, .5) 50%, #323232 100%);
+    background: -webkit-radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
+    background: -o-radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
     background: radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
     z-index: -9
 }
@@ -1562,8 +1554,8 @@ body > div.container-fluid > div.row-fluid > div.col-sm-10 > div.discover > form
     top: 0;
     left: 0;
     opacity: 0;
-    background: -webkit-radial-gradient(50% 50%, farthest-corner, rgba(50, 50, 50, .5) 50%, #323232 100%);
-    background: -o-radial-gradient(50% 50%, farthest-corner, rgba(50, 50, 50, .5) 50%, #323232 100%);
+    background: -webkit-radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
+    background: -o-radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
     background: radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%)
 }
 
@@ -1739,7 +1731,7 @@ body > div.container-fluid > div.row-fluid > div.col-sm-10 {
 
 body.me > div.container-fluid > div.row-fluid > div.col-sm-10:before {
     content: '';
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     font-size: 6vw;
@@ -1787,6 +1779,12 @@ body > div.container-fluid > div > div.col-sm-10 > div.discover {
     margin-top: 0
 }
 
+.container-fluid .book .meta .series {
+    /* font-weight: 400; */
+    /* font-size: 12px; */
+    color: hsla(0, 0%, 100%, .45);
+}
+
 .container-fluid .book .meta > p {
     -o-text-overflow: ellipsis;
     text-overflow: ellipsis;
@@ -1947,7 +1945,7 @@ body > div.container-fluid > div > div.col-sm-10 > div.pagination .page-next > a
 body > div.container-fluid > div > div.col-sm-10 > div.pagination .page-previous > a
 {
     top: 0;
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-weight: 100;
     -webkit-font-smoothing: antialiased;
     line-height: 60px;
@@ -2026,7 +2024,7 @@ body.authorlist > div.container-fluid > div > div.col-sm-10 > div.container > di
 
 body.serieslist > div.container-fluid > div > div.col-sm-10:before {
     content: "\e044";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -2123,15 +2121,14 @@ body > div.container-fluid > div > div.col-sm-10 > div.discover > div.container
     transition: all 0s
 }
 
-
-.book-meta > .bookinfo > .tags .btn-info, .well > form > .btn {
+.well > form > .btn {
     vertical-align: middle;
     -o-transition: background-color .2s, color .2s
 }
 
 body.catlist > div.container-fluid > div.row-fluid > div.col-sm-10:before {
     content: "\E067";
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -2151,7 +2148,7 @@ body.catlist > div.container-fluid > div.row-fluid > div.col-sm-10:before {
 
 
 body.authorlist > div.container-fluid > div.row-fluid > div.col-sm-10:before, body.langlist > div.container-fluid > div > div.col-sm-10:before {
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -2492,7 +2489,6 @@ body > div.container-fluid > div > div.col-sm-10 > div.col-sm-8 > form > .btn.bt
 }
 
 textarea {
-    resize: none;
     resize: vertical
 }
 
@@ -2838,7 +2834,7 @@ body > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-8 > form
 
 body.advanced_search > div.container-fluid > div > div.col-sm-10 > div.col-sm-8:before {
     content: "\EA4F";
-    font-family: plex-icons;
+    font-family: plex-icons, serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -2936,8 +2932,9 @@ body > div.container-fluid > div > div.col-sm-10 > div > div > div.col-sm-3.col-
 }
 
 #bookDetailsModal > .modal-dialog.modal-lg > .modal-content > .modal-body > div > div > div > div.col-sm-3.col-lg-3.col-xs-5 > div.cover, body > div.container-fluid > div > div.col-sm-10 > div > div > div.col-sm-3.col-lg-3.col-xs-5 > div.cover {
-    margin: 0;
+    margin: auto;
     width: 100%;
+    max-width: 200px;
 }
 
 #bookDetailsModal > .modal-dialog.modal-lg > .modal-content > .modal-body > div > div > div > div.col-sm-3.col-lg-3.col-xs-5 > div.cover > img, body > div.container-fluid > div > div.col-sm-10 > div > div > div.col-sm-3.col-lg-3.col-xs-5 > div.cover > img {
@@ -2962,46 +2959,35 @@ body > div.container-fluid > div > div.col-sm-10 > div > div > div.col-sm-3.col-
     margin-top: 24px
 }
 
-.book-meta > .bookinfo > .publishers > span:first-of-type, .book-meta > .bookinfo > .publishing-date > span:first-of-type {
+.book-meta > .bookinfo > .languages > span:first-of-type,
+.book-meta > .bookinfo > .publishers > span:first-of-type,
+.book-meta > .bookinfo > .publishing-date > span:first-of-type,
+.real_custom_columns > span:first-of-type {
     color: hsla(0, 0%, 100%, .45);
     text-transform: uppercase;
-    font-family: Open Sans Bold, Helvetica Neue, Helvetica, Arial, sans-serif
+    font-family: Open Sans Bold, Helvetica Neue, Helvetica, Arial, sans-serif;
+    width: 200px;
+    display: inline-block
 }
 
-.book-meta > .bookinfo > .publishers > span:last-of-type, .book-meta > .bookinfo > .publishing-date > span:last-of-type {
+.book-meta > .bookinfo > .languages > span:last-of-type,
+.book-meta > .bookinfo > .publishers > span:last-of-type,
+.book-meta > .bookinfo > .publishing-date > span:last-of-type,
+.real_custom_columns > span:last-of-type {
     font-family: Open Sans Semibold, Helvetica Neue, Helvetica, Arial, sans-serif;
     color: #fff;
     font-size: 15px;
     -webkit-font-smoothing: antialiased
 }
 
-.book-meta > .bookinfo > .publishers > span:last-of-type {
-    padding-left: 90px
+.book-meta > .bookinfo > .languages > span > a,
+.book-meta > .bookinfo > .publishers > span > a,
+.book-meta > .bookinfo > .publishing-date > span > a,
+.real_custom_columns > span > a {
+    color: #fff
 }
 
-.real_custom_columns > span:last-of-type {
-    padding-left: 90px
-}
-
-.book-meta > .bookinfo > .publishing-date > span:last-of-type {
-    padding-left: 90px
-}
-
-.book-meta > .bookinfo > .languages > span:first-of-type {
-    color: hsla(0, 0%, 100%, .45);
-    text-transform: uppercase;
-    font-family: Open Sans Bold, Helvetica Neue, Helvetica, Arial, sans-serif
-}
-
-.book-meta > .bookinfo > .languages > span:last-of-type {
-    font-size: 15px;
-    font-family: Open Sans Semibold, Helvetica Neue, Helvetica, Arial, sans-serif;
-    -webkit-font-smoothing: antialiased;
-    color: #fff;
-    padding-left: 85px
-}
-
-.book-meta > .bookinfo > .tags .btn-info, .book-meta > h2, body.book .author {
+.book-meta > h2, body.book .author {
     font-family: Open Sans Bold, Helvetica Neue, Helvetica, Arial, sans-serif
 }
 
@@ -3082,34 +3068,10 @@ body.book .author {
     background-color: rgba(0, 0, 0, .3)
 }
 
-.book-meta > .bookinfo > .identifiers > p > .btn-success, .book-meta > .bookinfo > .tags .btn-info {
-    overflow: hidden;
-    text-align: center;
-    white-space: nowrap;
-    margin: 2px 3px 0 0;
-    padding: 0 10px
-}
-
-.book-meta > .bookinfo > .tags .btn-info {
-    background-color: rgba(0, 0, 0, .15);
-    color: hsla(0, 0%, 100%, .7);
-    font-size: 13px;
-    display: inline-block;
-    border-radius: 4px;
-    -webkit-transition: background-color .2s, color .2s;
-    transition: background-color .2s, color .2s;
-    text-transform: none
-}
-
 .dropdown-menu, .tooltip.in {
     -webkit-transition: opacity .15s ease-out, -webkit-transform .15s cubic-bezier(.6, .4, .2, 1.4)
 }
 
-.book-meta > .bookinfo > .tags .btn-info:hover {
-    color: #fff;
-    text-decoration: underline
-}
-
 .book-meta > .bookinfo > .identifiers, .book-meta > .bookinfo > .tags {
     padding-left: 40px;
     margin: 10px 0
@@ -3180,6 +3142,10 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > div.
     overflow: hidden;
     padding: 0
 }
+#readbtn {
+  height: 100%;
+  padding: 16px;
+}
 
 #add-to-shelf > span.caret, #btnGroupDrop1 > span.caret, #read-in-browser > span.caret, .btn-toolbar > .btn-group > #btnGroupDrop2 > span.caret, .btn-toolbar > .btn-group > .btn-group > #btnGroupDrop2 > span.caret {
     padding-bottom: 5px
@@ -3195,7 +3161,7 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > div.
 
 #add-to-shelf > span.glyphicon.glyphicon-list:before {
     content: "\EA59";
-    font-family: plex-icons;
+    font-family: plex-icons, serif;
     font-size: 18px
 }
 
@@ -3207,7 +3173,7 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > div.
 
 #read-in-browser > span.glyphicon-eye-open:before, .btn-toolbar > .btn-group > .btn-group > #btnGroupDrop2 > span.glyphicon-eye-open:before {
     content: "\e352";
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     font-size: 18px;
     padding-right: 5px
 }
@@ -3219,7 +3185,7 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > div.
 #btnGroupDrop1 > span.glyphicon-download:before {
     font-size: 20px;
     content: "\ea66";
-    font-family: plex-icons
+    font-family: plex-icons, serif
 }
 
 .col-sm-10 .book-meta > div.btn-toolbar {
@@ -3323,7 +3289,6 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] .dropd
     -webkit-box-shadow: 0 4px 10px rgba(0, 0, 0, .35);
     box-shadow: 0 4px 10px rgba(0, 0, 0, .35);
     -o-transition: opacity .15s ease-out, transform .15s cubic-bezier(.6, .4, .2, 1.4);
-    transition: opacity .15s ease-out, transform .15s cubic-bezier(.6, .4, .2, 1.4);
     transition: opacity .15s ease-out, transform .15s cubic-bezier(.6, .4, .2, 1.4), -webkit-transform .15s cubic-bezier(.6, .4, .2, 1.4);
     -webkit-transform-origin: center top;
     -ms-transform-origin: center top;
@@ -3351,7 +3316,8 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] .dropd
     box-shadow: none
 }
 
-.book-meta > .bookinfo > .identifiers > p > .btn-success {
+.book-meta > .bookinfo .btn-info,
+.book-meta > .bookinfo .btn-success {
     background-color: rgba(0, 0, 0, .15);
     color: hsla(0, 0%, 100%, .7);
     font-size: 13px;
@@ -3365,11 +3331,21 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] .dropd
     text-transform: none
 }
 
-.book-meta > .bookinfo > .identifiers > p > .btn-success:hover {
+.book-meta > .bookinfo .btn-info:hover,
+.book-meta > .bookinfo .btn-success:hover {
     color: #fff;
     text-decoration: underline
 }
 
+.book-meta > .bookinfo .btn-info,
+.book-meta > .bookinfo .btn-success {
+    overflow: hidden;
+    text-align: center;
+    white-space: nowrap;
+    margin: 2px 3px 0 0;
+    padding: 0 10px
+}
+
 #bookDetailsModal .book-meta {
     color: hsla(0, 0%, 100%, .7);
     height: calc(100% - 120px);
@@ -3441,7 +3417,7 @@ body > div.container-fluid > div > div.col-sm-10 > div.discover > .btn-primary:l
 
 .book-meta > div.more-stuff > .btn-toolbar > .btn-group[aria-label="Remove from shelves"] > a > .glyphicon-remove:before {
     content: "\ea64";
-    font-family: plex-icons
+    font-family: plex-icons, serif
 }
 
 body > div.container-fluid > div.row-fluid > div.col-sm-10 > div.discover > form > .col-sm-6 {
@@ -3555,7 +3531,7 @@ body.shelf > div.container-fluid > div > div.col-sm-10 > div.discover > .shelf-b
 
 body.shelf > div.container-fluid > div > div.col-sm-10 > div.discover > .shelf-btn-group > [data-target="#DeleteShelfDialog"]:before {
     content: "\EA6D";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     position: absolute;
     color: hsla(0, 0%, 100%, .7);
     font-size: 20px;
@@ -3585,7 +3561,7 @@ body.shelf > div.container-fluid > div > div.col-sm-10 > div.discover > .shelf-b
 
 body.shelf > div.container-fluid > div > div.col-sm-10 > div.discover > .shelf-btn-group > [href*=edit]:before {
     content: "\EA5d";
-    font-family: plex-icons;
+    font-family: plex-icons, serif;
     position: absolute;
     color: hsla(0, 0%, 100%, .7);
     font-size: 20px;
@@ -3615,7 +3591,7 @@ body.shelf > div.container-fluid > div > div.col-sm-10 > div.discover > .shelf-b
 
 body.shelf > div.container-fluid > div > div.col-sm-10 > div.discover > .shelf-btn-group > [href*=order]:before {
     content: "\E409";
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     position: absolute;
     color: hsla(0, 0%, 100%, .7);
     font-size: 20px;
@@ -3752,7 +3728,7 @@ body > div.navbar.navbar-default.navbar-static-top > div > div.navbar-header > a
 
 .plexBack > a {
     color: rgba(255, 255, 255, .7);
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     -webkit-font-variant-ligatures: normal;
     font-variant-ligatures: normal;
     line-height: 60px;
@@ -3864,11 +3840,9 @@ body.login > div.container-fluid > div.row-fluid > div.col-sm-10::before, body.l
     -webkit-transform: translateY(-50%);
     -ms-transform: translateY(-50%);
     transform: translateY(-50%);
-    border-style: solid;
     vertical-align: middle;
     -webkit-transition: border .2s, -webkit-transform .4s;
     -o-transition: border .2s, transform .4s;
-    transition: border .2s, transform .4s;
     transition: border .2s, transform .4s, -webkit-transform .4s;
     margin: 9px 6px
 }
@@ -3887,11 +3861,9 @@ body.login > div.container-fluid > div.row-fluid > div.col-sm-10::before, body.l
     -webkit-transform: translateY(-50%);
     -ms-transform: translateY(-50%);
     transform: translateY(-50%);
-    border-style: solid;
     vertical-align: middle;
     -webkit-transition: border .2s, -webkit-transform .4s;
     -o-transition: border .2s, transform .4s;
-    transition: border .2s, transform .4s;
     transition: border .2s, transform .4s, -webkit-transform .4s;
     margin: 12px 6px
 }
@@ -3971,7 +3943,7 @@ body.author img.bg-blur[src=undefined] {
 
 body.author:not(.authorlist) .undefined-img:before {
     content: "\e008";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -4120,7 +4092,7 @@ body.shelf.modal-open > .container-fluid {
     font-size: 18px;
     color: #999;
     display: inline-block;
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     font-style: normal;
     font-weight: 400
 }
@@ -4221,7 +4193,7 @@ body.shelf.modal-open > .container-fluid {
 
 #remove-from-shelves > .btn > span:before {
     content: "\EA52";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     color: transparent;
     padding-left: 5px
 }
@@ -4233,7 +4205,7 @@ body.shelf.modal-open > .container-fluid {
 
 #remove-from-shelves > a:first-of-type:before {
     content: "\EA4D";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     position: absolute;
     color: hsla(0, 0%, 100%, .45);
     font-style: normal;
@@ -4273,7 +4245,7 @@ body.shelf.modal-open > .container-fluid {
     content: "\E208";
     padding-right: 10px;
     display: block;
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     font-style: normal;
     font-weight: 400;
     position: absolute;
@@ -4284,7 +4256,6 @@ body.shelf.modal-open > .container-fluid {
     opacity: .5;
     -webkit-transition: -webkit-transform .3s ease-out;
     -o-transition: transform .3s ease-out;
-    transition: transform .3s ease-out;
     transition: transform .3s ease-out, -webkit-transform .3s ease-out;
     -webkit-transform: translate(0, -60px);
     -ms-transform: translate(0, -60px);
@@ -4344,7 +4315,7 @@ body.advanced_search > div.container-fluid > div > div.col-sm-10 > div.col-sm-8
 
 .glyphicon-remove:before {
     content: "\EA52";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-weight: 400
 }
 
@@ -4430,7 +4401,7 @@ body.advanced_search > div.container-fluid > div.row-fluid > div.col-sm-10 > div
 
     body:not(.blur) #nav_new:before {
         content: "\EA4F";
-        font-family: plex-icons;
+        font-family: plex-icons, serif;
         font-style: normal;
         font-weight: 400;
         line-height: 1;
@@ -4456,7 +4427,7 @@ body.advanced_search > div.container-fluid > div.row-fluid > div.col-sm-10 > div
         color: hsla(0, 0%, 100%, .7);
         cursor: pointer;
         display: block;
-        font-family: plex-icons-new;
+        font-family: plex-icons-new, serif;
         font-size: 20px;
         font-stretch: 100%;
         font-style: normal;
@@ -4552,12 +4523,12 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d
 }
 
 body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > div.row > div.col .table > tbody > tr > td, body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > div.row > div.col .table > tbody > tr > th, body.admin > div.container-fluid > div > div.col-sm-10 > div.discover > .table > tbody > tr > td, body.admin > div.container-fluid > div > div.col-sm-10 > div.discover > .table > tbody > tr > th {
-    border: collapse
+    border: collapse;
 }
 
 body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10::before, body.newuser.admin > div.container-fluid > div.row-fluid > div.col-sm-10::before {
     content: '';
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     font-size: 6vw;
@@ -4661,7 +4632,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     content: "\e352";
     display: inline-block;
     position: absolute;
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     background: var(--color-secondary);
     color: #fff;
     border-radius: 50%;
@@ -4699,8 +4670,8 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     top: 0;
     left: 0;
     opacity: 0;
-    background: -webkit-radial-gradient(50% 50%, farthest-corner, rgba(50, 50, 50, .5) 50%, #323232 100%);
-    background: -o-radial-gradient(50% 50%, farthest-corner, rgba(50, 50, 50, .5) 50%, #323232 100%);
+    background: -webkit-radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
+    background: -o-radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%);
     background: radial-gradient(farthest-corner at 50% 50%, rgba(50, 50, 50, .5) 50%, #323232 100%)
 }
 
@@ -4752,7 +4723,7 @@ body.admin td > a:hover {
 
 .glyphicon-ok::before {
     content: "\EA55";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-weight: 400
 }
 
@@ -4821,7 +4792,7 @@ body:not(.blur):not(.login):not(.me):not(.author):not(.editbook):not(.upload):no
     background-position: center center, center center, center center !important;
     background-size: auto, auto, cover !important;
     -webkit-background-size: auto, auto, cover !important;
-    -moz-background-size: autom, auto, cover !important;
+    -moz-background-size: auto, auto, cover !important;
     -o-background-size: auto, auto, cover !important;
     width: 100%;
     height: 60px;
@@ -4887,7 +4858,6 @@ body.read:not(.blur) a[href*=readbooks] {
 .tooltip.in {
     opacity: 1;
     -o-transition: opacity .15s ease-out, transform .15s cubic-bezier(.6, .4, .2, 1.4);
-    transition: opacity .15s ease-out, transform .15s cubic-bezier(.6, .4, .2, 1.4);
     transition: opacity .15s ease-out, transform .15s cubic-bezier(.6, .4, .2, 1.4), -webkit-transform .15s cubic-bezier(.6, .4, .2, 1.4);
     -webkit-transform: translate(0) scale(1);
     -ms-transform: translate(0) scale(1);
@@ -4987,7 +4957,7 @@ body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div
 
 body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-3 > div.text-center > #delete:before, body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-3 > div.text-center > #delete:before {
     content: "\EA6D";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-size: 18px;
     color: hsla(0, 0%, 100%, .7)
 }
@@ -5072,7 +5042,7 @@ body.tasks > div.container-fluid > div > div.col-sm-10 > div.discover > div.boot
 
 body.tasks > div.container-fluid > div > div.col-sm-10 > div.discover > div.bootstrap-table > div.fixed-table-container > div.fixed-table-body > #table > thead > tr > th > .th-inner.asc:after {
     content: "\EA58";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-weight: 400;
     right: 20px;
     position: absolute
@@ -5080,7 +5050,7 @@ body.tasks > div.container-fluid > div > div.col-sm-10 > div.discover > div.boot
 
 body.tasks > div.container-fluid > div > div.col-sm-10 > div.discover > div.bootstrap-table > div.fixed-table-container > div.fixed-table-body > #table > thead > tr > th > .th-inner.desc:after {
     content: "\EA57";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-weight: 400;
     right: 20px;
     position: absolute
@@ -5143,7 +5113,7 @@ body.tasks > div.container-fluid > div > div.col-sm-10 > div.discover > div.boot
 
 .epub-back:before {
     content: "\EA1C";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-weight: 400;
     color: #4f4f4f;
     position: absolute;
@@ -5306,7 +5276,7 @@ body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm
 
 body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-3 > div.text-center > #delete:before, body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-3 > div.text-center > #delete:before {
     content: "\EA6D";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-size: 18px;
     color: hsla(0, 0%, 100%, .7);
     vertical-align: super
@@ -5466,7 +5436,7 @@ body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm
 
 #main-nav + #scnd-nav .create-shelf a:before {
     content: "\EA13";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     font-size: 100%;
     padding-right: 10px;
     vertical-align: middle
@@ -5511,7 +5481,7 @@ body.admin.modal-open .navbar {
     content: "\E208";
     padding-right: 10px;
     display: block;
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     font-style: normal;
     font-weight: 400;
     position: absolute;
@@ -5522,7 +5492,6 @@ body.admin.modal-open .navbar {
     opacity: .5;
     -webkit-transition: -webkit-transform .3s ease-out;
     -o-transition: transform .3s ease-out;
-    transition: transform .3s ease-out;
     transition: transform .3s ease-out, -webkit-transform .3s ease-out;
     -webkit-transform: translate(0, -60px);
     -ms-transform: translate(0, -60px);
@@ -5576,22 +5545,22 @@ body.admin.modal-open .navbar {
 
 #RestartDialog > .modal-dialog > .modal-content > .modal-header:before {
     content: "\EA4F";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before {
     content: "\E064";
-    font-family: glyphicons regular
+    font-family: glyphicons regular, serif
 }
 
 #StatusDialog > .modal-dialog > .modal-content > .modal-header:before {
     content: "\EA15";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #deleteModal > .modal-dialog > .modal-content > .modal-header:before {
     content: "\EA6D";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #RestartDialog > .modal-dialog > .modal-content > .modal-header:after {
@@ -5982,7 +5951,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
 
     .home-btn {
         height: 48px;
-        line-height: 28.29px;
+        line-height: 28px;
         right: 10px;
         left: auto
     }
@@ -5994,7 +5963,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
 
     .plexBack {
         height: 48px;
-        line-height: 28.29px;
+        line-height: 28px;
         left: 48px;
         display: none
     }
@@ -6073,7 +6042,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     body > div.navbar.navbar-default.navbar-static-top > div > form.search-focus > div > span.input-group-btn:before {
         content: "\EA33";
         display: block;
-        font-family: plex-icons-new;
+        font-family: plex-icons-new, serif;
         position: fixed;
         left: 0;
         top: 0;
@@ -6225,7 +6194,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
 
     #form-upload .form-group .btn:before {
         content: "\e043";
-        font-family: 'Glyphicons Halflings';
+        font-family: 'Glyphicons Halflings', serif;
         line-height: 1;
         -webkit-font-smoothing: antialiased;
         color: #fff;
@@ -6243,7 +6212,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     #form-upload .form-group .btn:after {
         content: "\EA13";
         position: absolute;
-        font-family: plex-icons-new;
+        font-family: plex-icons-new, serif;
         font-size: 8px;
         background: #3c444a;
         color: #fff;
@@ -6296,7 +6265,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     }
 
     #top_admin, #top_tasks {
-        padding: 11.5px 15px;
+        padding: 12px 15px;
         font-size: 13px;
         line-height: 1.71428571;
         overflow: hidden
@@ -6305,7 +6274,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     #top_admin > .glyphicon, #top_tasks > .glyphicon-tasks {
         position: relative;
         top: 0;
-        font-family: 'Glyphicons Halflings';
+        font-family: 'Glyphicons Halflings', serif;
         line-height: 1;
         border-radius: 0;
         background: 0 0;
@@ -6324,7 +6293,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
 
     #top_tasks > .glyphicon-tasks::before, body > div.navbar.navbar-default.navbar-static-top > div > div.navbar-collapse.collapse > ul > li > #top_admin > .glyphicon-dashboard::before {
         text-transform: none;
-        font-family: plex-icons-new;
+        font-family: plex-icons-new, serif;
         -webkit-font-smoothing: antialiased;
         -moz-osx-font-smoothing: grayscale;
         text-rendering: optimizeLegibility;
@@ -6649,7 +6618,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
 
     .author > .container-fluid > .row-fluid > .col-sm-10 > h2:after {
         content: "\e008";
-        font-family: 'Glyphicons Halflings';
+        font-family: 'Glyphicons Halflings', serif;
         font-style: normal;
         font-weight: 400;
         line-height: 1;
@@ -6854,7 +6823,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         color: hsla(0, 0%, 100%, .7);
         cursor: pointer;
         display: block;
-        font-family: plex-icons-new;
+        font-family: plex-icons-new, serif;
         font-size: 20px;
         font-stretch: 100%;
         font-style: normal;
@@ -6983,16 +6952,12 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         margin: 45px
     }
 
-    .book-meta > .bookinfo > .publishing-date > span:last-of-type {
-        padding-left: 25px
-    }
-
-    .book-meta > .bookinfo > .publishers > span:last-of-type {
-        padding-left: 70px
-    }
-
-    .book-meta > .bookinfo > .languages > span:last-of-type {
-        padding-left: 65px
+    .book-meta > .bookinfo > .languages > span:first-of-type,
+    .book-meta > .bookinfo > .publishers > span:first-of-type,
+    .book-meta > .bookinfo > .publishing-date > span:first-of-type,
+    .real_custom_columns > span:first-of-type {
+        width: 50%;
+        max-width: 200px;
     }
 
     .book-meta > .bookinfo .publishers {
@@ -7025,11 +6990,9 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         -webkit-transform: translateY(-50%);
         -ms-transform: translateY(-50%);
         transform: translateY(-50%);
-        border-style: solid;
         vertical-align: middle;
         -webkit-transition: border .2s, -webkit-transform .4s;
         -o-transition: border .2s, transform .4s;
-        transition: border .2s, transform .4s;
         transition: border .2s, transform .4s, -webkit-transform .4s;
         margin: 12px 6px
     }
@@ -7048,18 +7011,16 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         -webkit-transform: translateY(-50%);
         -ms-transform: translateY(-50%);
         transform: translateY(-50%);
-        border-style: solid;
         vertical-align: middle;
         -webkit-transition: border .2s, -webkit-transform .4s;
         -o-transition: border .2s, transform .4s;
-        transition: border .2s, transform .4s;
         transition: border .2s, transform .4s, -webkit-transform .4s;
         margin: 9px 6px
     }
 
     body.author:not(.authorlist) .blur-wrapper:before, body.author > .container-fluid > .row-fluid > .col-sm-10 > h2:after {
         content: "\e008";
-        font-family: 'Glyphicons Halflings';
+        font-family: 'Glyphicons Halflings', serif;
         font-weight: 400;
         z-index: 9;
         line-height: 1;
@@ -7390,7 +7351,6 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         transform: translate3d(0, 0, 0);
         -webkit-transition: -webkit-transform .5s;
         -o-transition: transform .5s;
-        transition: transform .5s;
         transition: transform .5s, -webkit-transform .5s;
         z-index: 99
     }
@@ -7405,7 +7365,6 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         transform: translate3d(-240px, 0, 0);
         -webkit-transition: -webkit-transform .5s;
         -o-transition: transform .5s;
-        transition: transform .5s;
         transition: transform .5s, -webkit-transform .5s;
         top: 0;
         margin: 0;
@@ -7444,7 +7403,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         text-align: center;
         min-width: 40px;
         pointer-events: none;
-        color: #
+        // color: #
     }
 
     .col-xs-12 > .row > .col-xs-10 {
@@ -7555,7 +7514,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
 
 body.publisherlist > div.container-fluid > div > div.col-sm-10:before {
     content: "\e241";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -7575,7 +7534,7 @@ body.publisherlist > div.container-fluid > div > div.col-sm-10:before {
 
 body.ratingslist > div.container-fluid > div > div.col-sm-10:before {
     content: "\e007";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -7601,7 +7560,7 @@ body.ratingslist > div.container-fluid > div > div.col-sm-10:before {
 
 body.formatslist > div.container-fluid > div > div.col-sm-10:before {
     content: "\e022";
-    font-family: 'Glyphicons Halflings';
+    font-family: 'Glyphicons Halflings', serif;
     font-style: normal;
     font-weight: 400;
     line-height: 1;
@@ -7776,7 +7735,7 @@ body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover .editabl
 
 body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover .glyphicon-trash:before {
     content: "\EA6D";
-    font-family: plex-icons-new
+    font-family: plex-icons-new, serif
 }
 
 #DeleteDomain {
@@ -7799,7 +7758,7 @@ body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover .glyphic
     content: "\E208";
     padding-right: 10px;
     display: block;
-    font-family: Glyphicons Regular;
+    font-family: Glyphicons Regular, serif;
     font-style: normal;
     font-weight: 400;
     position: absolute;
@@ -7810,7 +7769,6 @@ body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover .glyphic
     opacity: .5;
     -webkit-transition: -webkit-transform .3s ease-out;
     -o-transition: transform .3s ease-out;
-    transition: transform .3s ease-out;
     transition: transform .3s ease-out, -webkit-transform .3s ease-out;
     -webkit-transform: translate(0, -60px);
     -ms-transform: translate(0, -60px);
@@ -7849,7 +7807,7 @@ body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover .glyphic
 
 #DeleteDomain > .modal-dialog > .modal-content > .modal-header:before {
     content: "\EA6D";
-    font-family: plex-icons-new;
+    font-family: plex-icons-new, serif;
     padding-right: 10px;
     font-size: 18px;
     color: #999;

cps/static/css/caliBlur_override.css

@@ -1,17 +1,24 @@
-body.serieslist.grid-view div.container-fluid>div>div.col-sm-10:before{
+body.serieslist.grid-view div.container-fluid > div > div.col-sm-10::before {
   display: none;
 }
 
-.cover .badge{
+.cover .badge {
   position: absolute;
   top: 0;
   left: 0;
+  color: #fff;
   background-color: #cc7b19;
   border-radius: 0;
   padding: 0 8px;
-    box-shadow: 0 0 4px rgba(0,0,0,.6);
+  box-shadow: 0 0 4px rgba(0, 0, 0, 0.6);
   line-height: 24px;
 }
-.cover{
-    box-shadow: 0 0 4px rgba(0,0,0,.6);
+
+.cover {
+  box-shadow: 0 0 4px rgba(0, 0, 0, 0.6);
+}
+
+.cover .read {
+  padding: 0 0;
+  line-height: 15px;
 }

cps/static/css/kthoom.css

@@ -33,7 +33,6 @@ body {
   position: relative;
   cursor: pointer;
   padding: 4px;
-
   transition: all 0.2s ease;
 }
 
@@ -45,7 +44,7 @@ body {
 
 #sidebar a.active,
 #sidebar a.active img + span {
-  background-color: #45B29D;
+  background-color: #45b29d;
 }
 
 #sidebar li img {
@@ -85,15 +84,24 @@ body {
 #progress .bar-load,
 #progress .bar-read {
   display: flex;
-  align-items: flex-end;
-  justify-content: flex-end;
   position: absolute;
   top: 0;
-  left: 0;
   bottom: 0;
   transition: width 150ms ease-in-out;
 }
 
+#progress .from-left {
+  left: 0;
+  align-items: flex-end;
+  justify-content: flex-end;
+}
+
+#progress .from-right {
+  right: 0;
+  align-items: flex-start;
+  justify-content: flex-start;
+}
+
 #progress .bar-load {
   color: #000;
   background-color: #ccc;

cps/static/css/libs/images/findbarButton-next-dark.svg

@@ -0,0 +1,6 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
+fill="rgba(255,255,255,1)"><path d="M8 12a1 1 0 0 1-.707-.293l-5-5a1 1 0 0 1 1.414-1.414L8
+9.586l4.293-4.293a1 1 0 0 1 1.414 1.414l-5 5A1 1 0 0 1 8 12z"></path></svg>

cps/static/css/libs/images/findbarButton-next.svg

@@ -0,0 +1,4 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"><path d="M8 12a1 1 0 0 1-.707-.293l-5-5a1 1 0 0 1 1.414-1.414L8 9.586l4.293-4.293a1 1 0 0 1 1.414 1.414l-5 5A1 1 0 0 1 8 12z"></path></svg>

cps/static/css/libs/images/findbarButton-previous-dark.svg

@@ -0,0 +1,5 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
+fill="rgba(255,255,255,1)"><path d="M13 11a1 1 0 0 1-.707-.293L8 6.414l-4.293 4.293a1 1 0 0 1-1.414-1.414l5-5a1 1 0 0 1 1.414 0l5 5A1 1 0 0 1 13 11z"></path></svg>

cps/static/css/libs/images/findbarButton-previous.svg

@@ -0,0 +1,4 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"><path d="M13 11a1 1 0 0 1-.707-.293L8 6.414l-4.293 4.293a1 1 0 0 1-1.414-1.414l5-5a1 1 0 0 1 1.414 0l5 5A1 1 0 0 1 13 11z"></path></svg>

cps/static/css/libs/images/loading-dark.svg

@@ -0,0 +1,24 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
+fill="rgba(255,255,255,1)" style="animation:spinLoadingIcon 1s steps(12,end)
+infinite"><style>@keyframes
+spinLoadingIcon{to{transform:rotate(360deg)}}</style><path
+d="M7 3V1s0-1 1-1 1 1 1 1v2s0 1-1 1-1-1-1-1z"/><path d="M4.63
+4.1l-1-1.73S3.13 1.5 4 1c.87-.5 1.37.37 1.37.37l1 1.73s.5.87-.37
+1.37c-.87.57-1.37-.37-1.37-.37z" fill-opacity=".93"/><path
+d="M3.1 6.37l-1.73-1S.5 4.87 1 4c.5-.87 1.37-.37 1.37-.37l1.73 1s.87.5.37
+1.37c-.5.87-1.37.37-1.37.37z" fill-opacity=".86"/><path d="M3
+9H1S0 9 0 8s1-1 1-1h2s1 0 1 1-1 1-1 1z" fill-opacity=".79"/><path d="M4.1 11.37l-1.73 1S1.5 12.87 1
+12c-.5-.87.37-1.37.37-1.37l1.73-1s.87-.5 1.37.37c.5.87-.37 1.37-.37 1.37z"
+fill-opacity=".72"/><path d="M3.63 13.56l1-1.73s.5-.87
+1.37-.37c.87.5.37 1.37.37 1.37l-1 1.73s-.5.87-1.37.37c-.87-.5-.37-1.37-.37-1.37z"
+fill-opacity=".65"/><path d="M7 15v-2s0-1 1-1 1 1 1 1v2s0 1-1
+1-1-1-1-1z" fill-opacity=".58"/><path d="M10.63
+14.56l-1-1.73s-.5-.87.37-1.37c.87-.5 1.37.37 1.37.37l1 1.73s.5.87-.37
+1.37c-.87.5-1.37-.37-1.37-.37z" fill-opacity=".51"/><path
+d="M13.56 12.37l-1.73-1s-.87-.5-.37-1.37c.5-.87 1.37-.37 1.37-.37l1.73 1s.87.5.37
+1.37c-.5.87-1.37.37-1.37.37z" fill-opacity=".44"/><path d="M15
+9h-2s-1 0-1-1 1-1 1-1h2s1 0 1 1-1 1-1 1z" fill-opacity=".37"/><path d="M14.56 5.37l-1.73
+1s-.87.5-1.37-.37c-.5-.87.37-1.37.37-1.37l1.73-1s.87-.5 1.37.37c.5.87-.37 1.37-.37
+1.37z" fill-opacity=".3"/><path d="M9.64 3.1l.98-1.66s.5-.874
+1.37-.37c.87.5.37 1.37.37 1.37l-1 1.73s-.5.87-1.37.37c-.87-.5-.37-1.37-.37-1.37z"
+fill-opacity=".23"/></svg>

cps/static/css/libs/images/loading.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16" style="animation:spinLoadingIcon 1s steps(12,end) infinite"><style>@keyframes spinLoadingIcon{to{transform:rotate(360deg)}}</style><path d="M7 3V1s0-1 1-1 1 1 1 1v2s0 1-1 1-1-1-1-1z"/><path d="M4.63 4.1l-1-1.73S3.13 1.5 4 1c.87-.5 1.37.37 1.37.37l1 1.73s.5.87-.37 1.37c-.87.57-1.37-.37-1.37-.37z" fill-opacity=".93"/><path d="M3.1 6.37l-1.73-1S.5 4.87 1 4c.5-.87 1.37-.37 1.37-.37l1.73 1s.87.5.37 1.37c-.5.87-1.37.37-1.37.37z" fill-opacity=".86"/><path d="M3 9H1S0 9 0 8s1-1 1-1h2s1 0 1 1-1 1-1 1z" fill-opacity=".79"/><path d="M4.1 11.37l-1.73 1S1.5 12.87 1 12c-.5-.87.37-1.37.37-1.37l1.73-1s.87-.5 1.37.37c.5.87-.37 1.37-.37 1.37z" fill-opacity=".72"/><path d="M3.63 13.56l1-1.73s.5-.87 1.37-.37c.87.5.37 1.37.37 1.37l-1 1.73s-.5.87-1.37.37c-.87-.5-.37-1.37-.37-1.37z" fill-opacity=".65"/><path d="M7 15v-2s0-1 1-1 1 1 1 1v2s0 1-1 1-1-1-1-1z" fill-opacity=".58"/><path d="M10.63 14.56l-1-1.73s-.5-.87.37-1.37c.87-.5 1.37.37 1.37.37l1 1.73s.5.87-.37 1.37c-.87.5-1.37-.37-1.37-.37z" fill-opacity=".51"/><path d="M13.56 12.37l-1.73-1s-.87-.5-.37-1.37c.5-.87 1.37-.37 1.37-.37l1.73 1s.87.5.37 1.37c-.5.87-1.37.37-1.37.37z" fill-opacity=".44"/><path d="M15 9h-2s-1 0-1-1 1-1 1-1h2s1 0 1 1-1 1-1 1z" fill-opacity=".37"/><path d="M14.56 5.37l-1.73 1s-.87.5-1.37-.37c-.5-.87.37-1.37.37-1.37l1.73-1s.87-.5 1.37.37c.5.87-.37 1.37-.37 1.37z" fill-opacity=".3"/><path d="M9.64 3.1l.98-1.66s.5-.874 1.37-.37c.87.5.37 1.37.37 1.37l-1 1.73s-.5.87-1.37.37c-.87-.5-.37-1.37-.37-1.37z" fill-opacity=".23"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-documentProperties-dark.svg

@@ -0,0 +1,16 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16
+16"
+fill="rgba(255,255,255,1)">
+<path 
+   d="M8 16a8 8 0 1 1 8-8 8.009 8.009 0 0 1-8 8zM8 2a6 6 0 1 0 6 6 6.006 6.006 0 0 0-6-6z">
+</path>
+<path 
+   d="M8 7a1 1 0 0 0-1 1v3a1 1 0 0 0 2 0V8a1 1 0 0 0-1-1z">
+</path>
+<circle
+   cx="8" cy="5" r="1.188">
+</circle>
+</svg>

cps/static/css/libs/images/secondaryToolbarButton-documentProperties.svg

@@ -0,0 +1,15 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16
+16">
+<path 
+   d="M8 16a8 8 0 1 1 8-8 8.009 8.009 0 0 1-8 8zM8 2a6 6 0 1 0 6 6 6.006 6.006 0 0 0-6-6z">
+</path>
+<path 
+   d="M8 7a1 1 0 0 0-1 1v3a1 1 0 0 0 2 0V8a1 1 0 0 0-1-1z">
+</path>
+<circle
+   cx="8" cy="5" r="1.188">
+</circle>
+</svg>

cps/static/css/libs/images/secondaryToolbarButton-firstPage-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
+fill="rgba(255,255,255,1)"><path d="M13 13c-.3 0-.5-.1-.7-.3L8 8.4l-4.3 4.3c-.9.9-2.3-.5-1.4-1.4l5-5c.4-.4 1-.4 1.4 0l5 5c.6.6.2 1.7-.7 1.7zm0-11H3C1.7 2 1.7 4 3 4h10c1.3 0 1.3-2 0-2z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-firstPage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"><path d="M13 13c-.3 0-.5-.1-.7-.3L8 8.4l-4.3 4.3c-.9.9-2.3-.5-1.4-1.4l5-5c.4-.4 1-.4 1.4 0l5 5c.6.6.2 1.7-.7 1.7zm0-11H3C1.7 2 1.7 4 3 4h10c1.3 0 1.3-2 0-2z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-handTool-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
+fill="rgba(255,255,255,1)"><path d="M15 3.7V13c0 1.5-1.53 3-3 3H7.13c-.72 0-1.63-.5-2.13-1l-5-5s.84-1 .87-1c.13-.1.33-.2.53-.2.1 0 .3.1.4.2L4 10.6V2.7c0-.6.4-1 1-1s1 .4 1 1v4.6h1V1c0-.6.4-1 1-1s1 .4 1 1v6.3h1V1.7c0-.6.4-1 1-1s1 .4 1 1v5.7h1V3.7c0-.6.4-1 1-1s1 .4 1 1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-handTool.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"><path d="M15 3.7V13c0 1.5-1.53 3-3 3H7.13c-.72 0-1.63-.5-2.13-1l-5-5s.84-1 .87-1c.13-.1.33-.2.53-.2.1 0 .3.1.4.2L4 10.6V2.7c0-.6.4-1 1-1s1 .4 1 1v4.6h1V1c0-.6.4-1 1-1s1 .4 1 1v6.3h1V1.7c0-.6.4-1 1-1s1 .4 1 1v5.7h1V3.7c0-.6.4-1 1-1s1 .4 1 1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-lastPage-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
+fill="rgba(255,255,255,1)"><path d="M8 10c-.3 0-.5-.1-.7-.3l-5-5c-.9-.9.5-2.3 1.4-1.4L8 7.6l4.3-4.3c.9-.9 2.3.5 1.4 1.4l-5 5c-.2.2-.4.3-.7.3zm5 2H3c-1.3 0-1.3 2 0 2h10c1.3 0 1.3-2 0-2z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-lastPage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"><path d="M8 10c-.3 0-.5-.1-.7-.3l-5-5c-.9-.9.5-2.3 1.4-1.4L8 7.6l4.3-4.3c.9-.9 2.3.5 1.4 1.4l-5 5c-.2.2-.4.3-.7.3zm5 2H3c-1.3 0-1.3 2 0 2h10c1.3 0 1.3-2 0-2z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-rotateCcw-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
+fill="rgba(255,255,255,1)"><path d="M1 1a1 1 0 011 1v2.4A7 7 0 118 15a7 7 0 01-4.9-2 1 1 0 011.4-1.5 5 5 0 10-1-5.5H6a1 1 0 010 2H1a1 1 0 01-1-1V2a1 1 0 011-1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-rotateCcw.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"><path d="M1 1a1 1 0 011 1v2.4A7 7 0 118 15a7 7 0 01-4.9-2 1 1 0 011.4-1.5 5 5 0 10-1-5.5H6a1 1 0 010 2H1a1 1 0 01-1-1V2a1 1 0 011-1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-rotateCw-dark.svg

@@ -0,0 +1,5 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
+fill="rgba(255,255,255,1)"><path d="M15 1a1 1 0 0 0-1 1v2.418A6.995 6.995 0 1 0 8 15a6.954 6.954 0 0 0 4.95-2.05 1 1 0 0 0-1.414-1.414A5.019 5.019 0 1 1 12.549 6H10a1 1 0 0 0 0 2h5a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1z"></path></svg>

cps/static/css/libs/images/secondaryToolbarButton-rotateCw.svg

@@ -0,0 +1,4 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"><path d="M15 1a1 1 0 0 0-1 1v2.418A6.995 6.995 0 1 0 8 15a6.954 6.954 0 0 0 4.95-2.05 1 1 0 0 0-1.414-1.414A5.019 5.019 0 1 1 12.549 6H10a1 1 0 0 0 0 2h5a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1z"></path></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollHorizontal-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
+fill="rgba(255,255,255,1)"><path d="M0 4h1.5c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5H0zM9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4zM16 4h-1.5c-1 0-1.5.5-1.5 1.5v5c0 1 .5 1.5 1.5 1.5H16z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollHorizontal.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"><path d="M0 4h1.5c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5H0zM9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4zM16 4h-1.5c-1 0-1.5.5-1.5 1.5v5c0 1 .5 1.5 1.5 1.5H16z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollVertical-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
+fill="rgba(255,255,255,1)"><path d="M9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4zM11 0v.5c0 1-.5 1.5-1.5 1.5h-3C5.5 2 5 1.5 5 .5V0h6zM11 16v-.5c0-1-.5-1.5-1.5-1.5h-3c-1 0-1.5.5-1.5 1.5v.5h6z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollVertical.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"><path d="M9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4zM11 0v.5c0 1-.5 1.5-1.5 1.5h-3C5.5 2 5 1.5 5 .5V0h6zM11 16v-.5c0-1-.5-1.5-1.5-1.5h-3c-1 0-1.5.5-1.5 1.5v.5h6z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollWrapped-dark.svg

@@ -0,0 +1,2 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
+fill="rgba(255,255,255,1)"><path d="M5.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C1 4.5 1.5 4 2.5 4zM7 0v.5C7 1.5 6.5 2 5.5 2h-3C1.5 2 1 1.5 1 .5V0h6zM7 16v-.5c0-1-.5-1.5-1.5-1.5h-3c-1 0-1.5.5-1.5 1.5v.5h6zM13.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5c0-1 .5-1.5 1.5-1.5zM15 0v.5c0 1-.5 1.5-1.5 1.5h-3C9.5 2 9 1.5 9 .5V0h6zM15 16v-.507c0-1-.5-1.5-1.5-1.5h-3C9.5 14 9 14.5 9 15.5v.5h6z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollWrapped.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"><path d="M5.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C1 4.5 1.5 4 2.5 4zM7 0v.5C7 1.5 6.5 2 5.5 2h-3C1.5 2 1 1.5 1 .5V0h6zM7 16v-.5c0-1-.5-1.5-1.5-1.5h-3c-1 0-1.5.5-1.5 1.5v.5h6zM13.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5c0-1 .5-1.5 1.5-1.5zM15 0v.5c0 1-.5 1.5-1.5 1.5h-3C9.5 2 9 1.5 9 .5V0h6zM15 16v-.507c0-1-.5-1.5-1.5-1.5h-3C9.5 14 9 14.5 9 15.5v.5h6z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-selectTool-dark.svg

@@ -0,0 +1,5 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
+fill="rgba(255,255,255,1)"><path d="M12.408 8.217l-8.083-6.7A.2.2 0 0 0 4 1.672V12.3a.2.2 0 0 0 .333.146l2.56-2.372 1.857 3.9A1.125 1.125 0 1 0 10.782 13L8.913 9.075l3.4-.51a.2.2 0 0 0 .095-.348z"></path></svg>