Merge remote-tracking branch 'upstream/master'

cps/__init__.py

@@ -37,6 +37,7 @@ from . import config_sql, logger, cache_buster, cli, ub, db
 from .reverseproxy import ReverseProxied
 from .server import WebServer
 
+
 mimetypes.init()
 mimetypes.add_type('application/xhtml+xml', '.xhtml')
 mimetypes.add_type('application/epub+zip', '.epub')
@@ -59,7 +60,7 @@ app = Flask(__name__)
 app.config.update(
     SESSION_COOKIE_HTTPONLY=True,
     SESSION_COOKIE_SAMESITE='Lax',
-    REMEMBER_COOKIE_SAMESITE='Lax',
+    REMEMBER_COOKIE_SAMESITE='Lax',  # will be available in flask-login 0.5.1 earliest
 )
 
 
@@ -82,6 +83,8 @@ log = logger.create()
 
 from . import services
 
+calibre_db = db.CalibreDB()
+
 def create_app():
     app.wsgi_app = ReverseProxied(app.wsgi_app)
     # For python2 convert path to unicode
@@ -98,7 +101,8 @@ def create_app():
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
 
     web_server.init_app(app, config)
-    db.setup_db(config)
+    calibre_db.setup_db(config, cli.settingspath)
+    calibre_db.start()
 
     babel.init_app(app)
     _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

cps/about.py

@@ -30,7 +30,7 @@ import babel, pytz, requests, sqlalchemy
 import werkzeug, flask, flask_login, flask_principal, jinja2
 from flask_babel import gettext as _
 
-from . import db, converter, uploader, server, isoLanguages, constants
+from . import db, calibre_db, converter, uploader, server, isoLanguages, constants
 from .web import render_title_template
 try:
     from flask_login import __version__ as flask_loginVersion
@@ -85,10 +85,12 @@ _VERSIONS.update(uploader.get_versions())
 @about.route("/stats")
 @flask_login.login_required
 def stats():
-    counter = db.session.query(db.Books).count()
-    authors = db.session.query(db.Authors).count()
-    categorys = db.session.query(db.Tags).count()
-    series = db.session.query(db.Series).count()
-    _VERSIONS['ebook converter'] = _(converter.get_version())
+    counter = calibre_db.session.query(db.Books).count()
+    authors = calibre_db.session.query(db.Authors).count()
+    categorys = calibre_db.session.query(db.Tags).count()
+    series = calibre_db.session.query(db.Series).count()
+    _VERSIONS['ebook converter'] = _(converter.get_calibre_version())
+    _VERSIONS['unrar'] = _(converter.get_unrar_version())
+    _VERSIONS['kepubify'] = _(converter.get_kepubify_version())
     return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=_VERSIONS,
                                  categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")

cps/admin.py

@@ -22,6 +22,7 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
+import re
 import base64
 import json
 import time
@@ -37,8 +38,8 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql.expression import func
 
 from . import constants, logger, helper, services
-from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
-from .helper import speaking_language, check_valid_domain, send_test_mail, reset_password, generate_password_hash
+from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
+from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano
 
@@ -85,7 +86,7 @@ def shutdown():
     showtext = {}
     if task in (0, 1):  # valid commandos received
         # close all database connections
-        db.dispose()
+        calibre_db.dispose()
         ub.dispose()
 
         if task == 0:
@@ -98,7 +99,7 @@ def shutdown():
 
     if task == 2:
         log.warning("reconnecting to calibre database")
-        db.setup_db(config)
+        calibre_db.setup_db(config, ub.app_DB_path)
         showtext['text'] = _(u'Reconnect successful')
         return json.dumps(showtext)
 
@@ -147,10 +148,10 @@ def configuration():
 @login_required
 @admin_required
 def view_configuration():
-    readColumn = db.session.query(db.Custom_Columns)\
-            .filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
-    restrictColumns= db.session.query(db.Custom_Columns)\
-            .filter(and_(db.Custom_Columns.datatype == 'text',db.Custom_Columns.mark_for_delete == 0)).all()
+    readColumn = calibre_db.session.query(db.Custom_Columns)\
+            .filter(and_(db.Custom_Columns.datatype == 'bool', db.Custom_Columns.mark_for_delete == 0)).all()
+    restrictColumns= calibre_db.session.query(db.Custom_Columns)\
+            .filter(and_(db.Custom_Columns.datatype == 'text', db.Custom_Columns.mark_for_delete == 0)).all()
     return render_title_template("config_view_edit.html", conf=config, readColumns=readColumn,
                                  restrictColumns=restrictColumns,
                                  title=_(u"UI Configuration"), page="uiconfig")
@@ -168,7 +169,6 @@ def update_view_configuration():
 
     _config_string("config_calibre_web_title")
     _config_string("config_columns_to_ignore")
-    # _config_string("config_mature_content_tags")
     reboot_required |= _config_string("config_title_regex")
 
     _config_int("config_read_column")
@@ -426,7 +426,6 @@ def delete_restriction(res_type):
     return ""
 
 
-#@admi.route("/ajax/listrestriction/<int:type>/<int:user_id>", defaults={'user_id': '0'})
 @admi.route("/ajax/listrestriction/<int:res_type>")
 @login_required
 @admin_required
@@ -472,6 +471,7 @@ def list_restriction(res_type):
     response.headers["Content-Type"] = "application/json; charset=utf-8"
     return response
 
+
 @admi.route("/config", methods=["GET", "POST"])
 @unconfigured
 def basic_configuration():
@@ -481,19 +481,23 @@ def basic_configuration():
     return _configuration_result()
 
 
-def _configuration_update_helper():
-    reboot_required = False
-    db_change = False
-    to_save = request.form.to_dict()
+def _config_int(to_save, x, func=int):
+    return config.set_from_dictionary(to_save, x, func)
 
-    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
-    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
-    _config_checkbox = lambda x: config.set_from_dictionary(to_save, x, lambda y: y == "on", False)
-    _config_checkbox_int = lambda x: config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0)
 
-    db_change |= _config_string("config_calibre_dir")
+def _config_checkbox(to_save, x):
+    return config.set_from_dictionary(to_save, x, lambda y: y == "on", False)
 
-    # Google drive setup
+
+def _config_checkbox_int(to_save, x):
+    return config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0)
+
+
+def _config_string(to_save, x):
+    return config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
+
+
+def _configuration_gdrive_helper(to_save):
     if not os.path.isfile(gdriveutils.SETTINGS_YAML):
         config.config_use_google_drive = False
 
@@ -512,46 +516,58 @@ def _configuration_update_helper():
 
     # always show google drive settings, but in case of error deny support
     config.config_use_google_drive = (not gdriveError) and ("config_use_google_drive" in to_save)
-    if _config_string("config_google_drive_folder"):
+    if _config_string(to_save, "config_google_drive_folder"):
         gdriveutils.deleteDatabaseOnChange()
+    return gdriveError
 
-    reboot_required |= _config_int("config_port")
+def _configuration_oauth_helper(to_save):
+    active_oauths = 0
+    reboot_required = False
+    for element in oauthblueprints:
+        if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \
+            or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']:
+            reboot_required = True
+            element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"]
+            element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"]
+        if to_save["config_" + str(element['id']) + "_oauth_client_id"] \
+            and to_save["config_" + str(element['id']) + "_oauth_client_secret"]:
+            active_oauths += 1
+            element["active"] = 1
+        else:
+            element["active"] = 0
+        ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update(
+            {"oauth_client_id": to_save["config_" + str(element['id']) + "_oauth_client_id"],
+             "oauth_client_secret": to_save["config_" + str(element['id']) + "_oauth_client_secret"],
+             "active": element["active"]})
+    return reboot_required
 
-    reboot_required |= _config_string("config_keyfile")
-    if config.config_keyfile and not os.path.isfile(config.config_keyfile):
-        return _configuration_result(_('Keyfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+def _configuration_logfile_helper(to_save, gdriveError):
+    reboot_required = False
+    reboot_required |= _config_int(to_save, "config_log_level")
+    reboot_required |= _config_string(to_save, "config_logfile")
+    if not logger.is_valid_logfile(config.config_logfile):
+        return reboot_required, _configuration_result(_('Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
 
-    reboot_required |= _config_string("config_certfile")
-    if config.config_certfile and not os.path.isfile(config.config_certfile):
-        return _configuration_result(_('Certfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+    reboot_required |= _config_checkbox_int(to_save, "config_access_log")
+    reboot_required |= _config_string(to_save, "config_access_logfile")
+    if not logger.is_valid_logfile(config.config_access_logfile):
+        return reboot_required, _configuration_result(_('Access Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+    return reboot_required, None
 
-    _config_checkbox_int("config_uploading")
-    _config_checkbox_int("config_anonbrowse")
-    _config_checkbox_int("config_public_reg")
-    reboot_required |= _config_checkbox_int("config_kobo_sync")
-    _config_checkbox_int("config_kobo_proxy")
-
-
-    _config_int("config_ebookconverter")
-    _config_string("config_calibre")
-    _config_string("config_converterpath")
-
-    reboot_required |= _config_int("config_login_type")
-
-    #LDAP configurator,
-    if config.config_login_type == constants.LOGIN_LDAP:
-        reboot_required |= _config_string("config_ldap_provider_url")
-        reboot_required |= _config_int("config_ldap_port")
-        reboot_required |= _config_int("config_ldap_authentication")
-        reboot_required |= _config_string("config_ldap_dn")
-        reboot_required |= _config_string("config_ldap_serv_username")
-        reboot_required |= _config_string("config_ldap_user_object")
-        reboot_required |= _config_string("config_ldap_group_object_filter")
-        reboot_required |= _config_string("config_ldap_group_members_field")
-        reboot_required |= _config_checkbox("config_ldap_openldap")
-        reboot_required |= _config_int("config_ldap_encryption")
-        reboot_required |= _config_string("config_ldap_cert_path")
-        _config_string("config_ldap_group_name")
+def _configuration_ldap_helper(to_save, gdriveError):
+    reboot_required = False
+    reboot_required |= _config_string(to_save, "config_ldap_provider_url")
+    reboot_required |= _config_int(to_save, "config_ldap_port")
+    reboot_required |= _config_int(to_save, "config_ldap_authentication")
+    reboot_required |= _config_string(to_save, "config_ldap_dn")
+    reboot_required |= _config_string(to_save, "config_ldap_serv_username")
+    reboot_required |= _config_string(to_save, "config_ldap_user_object")
+    reboot_required |= _config_string(to_save, "config_ldap_group_object_filter")
+    reboot_required |= _config_string(to_save, "config_ldap_group_members_field")
+    reboot_required |= _config_checkbox(to_save, "config_ldap_openldap")
+    reboot_required |= _config_int(to_save, "config_ldap_encryption")
+    reboot_required |= _config_string(to_save, "config_ldap_cert_path")
+    _config_string(to_save, "config_ldap_group_name")
     if "config_ldap_serv_password" in to_save and to_save["config_ldap_serv_password"] != "":
         reboot_required |= 1
         config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8')
@@ -561,95 +577,112 @@ def _configuration_update_helper():
         or not config.config_ldap_port \
         or not config.config_ldap_dn \
         or not config.config_ldap_user_object:
-                return _configuration_result(_('Please Enter a LDAP Provider, '
+        return reboot_required, _configuration_result(_('Please Enter a LDAP Provider, '
                                        'Port, DN and User Object Identifier'), gdriveError)
 
     if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS:
         if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE:
             if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password):
-                    return _configuration_result('Please Enter a LDAP Service Account and Password', gdriveError)
+                return reboot_required, _configuration_result('Please Enter a LDAP Service Account and Password', gdriveError)
         else:
             if not config.config_ldap_serv_username:
-                    return _configuration_result('Please Enter a LDAP Service Account', gdriveError)
-
-        #_config_checkbox("config_ldap_use_ssl")
-        #_config_checkbox("config_ldap_use_tls")
-        # reboot_required |= _config_checkbox("config_ldap_openldap")
-        # _config_checkbox("config_ldap_require_cert")
+                return reboot_required, _configuration_result('Please Enter a LDAP Service Account', gdriveError)
 
     if config.config_ldap_group_object_filter:
         if config.config_ldap_group_object_filter.count("%s") != 1:
-                return _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'),
+            return reboot_required, _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'),
                                          gdriveError)
         if config.config_ldap_group_object_filter.count("(") != config.config_ldap_group_object_filter.count(")"):
-                return _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'),
+            return reboot_required, _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'),
                                          gdriveError)
 
     if config.config_ldap_user_object.count("%s") != 1:
-            return _configuration_result(_('LDAP User Object Filter needs to Have One "%s" Format Identifier'),
+        return reboot_required, _configuration_result(_('LDAP User Object Filter needs to Have One "%s" Format Identifier'),
                                      gdriveError)
     if config.config_ldap_user_object.count("(") != config.config_ldap_user_object.count(")"):
-            return _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'),
+        return reboot_required, _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'),
                                      gdriveError)
 
     if config.config_ldap_cert_path and not os.path.isdir(config.config_ldap_cert_path):
-            return _configuration_result(_('LDAP Certificate Location is not Valid, Please Enter Correct Path'),
+        return reboot_required, _configuration_result(_('LDAP Certificate Location is not Valid, Please Enter Correct Path'),
                                      gdriveError)
+    return reboot_required, None
+
+
+def _configuration_update_helper():
+    reboot_required = False
+    db_change = False
+    to_save = request.form.to_dict()
+
+    to_save['config_calibre_dir'] = re.sub('[\\/]metadata\.db$', '', to_save['config_calibre_dir'], flags=re.IGNORECASE)
+    db_change |= _config_string(to_save, "config_calibre_dir")
+
+    # Google drive setup
+    gdriveError = _configuration_gdrive_helper(to_save)
+
+    reboot_required |= _config_int(to_save, "config_port")
+
+    reboot_required |= _config_string(to_save, "config_keyfile")
+    if config.config_keyfile and not os.path.isfile(config.config_keyfile):
+        return _configuration_result(_('Keyfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+
+    reboot_required |= _config_string(to_save, "config_certfile")
+    if config.config_certfile and not os.path.isfile(config.config_certfile):
+        return _configuration_result(_('Certfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+
+    _config_checkbox_int(to_save, "config_uploading")
+    _config_checkbox_int(to_save, "config_anonbrowse")
+    _config_checkbox_int(to_save, "config_public_reg")
+    _config_checkbox_int(to_save, "config_register_email")
+    reboot_required |= _config_checkbox_int(to_save, "config_kobo_sync")
+    _config_checkbox_int(to_save, "config_kobo_proxy")
+
+    _config_string(to_save, "config_upload_formats")
+    constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in config.config_upload_formats.split(',')]
+
+    _config_string(to_save, "config_calibre")
+    _config_string(to_save, "config_converterpath")
+    _config_string(to_save, "config_kepubifypath")
+
+    reboot_required |= _config_int(to_save, "config_login_type")
+
+    #LDAP configurator,
+    if config.config_login_type == constants.LOGIN_LDAP:
+        reboot, message = _configuration_ldap_helper(to_save, gdriveError)
+        if message:
+            return message
+        reboot_required |= reboot
 
     # Remote login configuration
-    _config_checkbox("config_remote_login")
+    _config_checkbox(to_save, "config_remote_login")
     if not config.config_remote_login:
         ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.token_type==0).delete()
 
     # Goodreads configuration
-    _config_checkbox("config_use_goodreads")
-    _config_string("config_goodreads_api_key")
-    _config_string("config_goodreads_api_secret")
+    _config_checkbox(to_save, "config_use_goodreads")
+    _config_string(to_save, "config_goodreads_api_key")
+    _config_string(to_save, "config_goodreads_api_secret")
     if services.goodreads_support:
         services.goodreads_support.connect(config.config_goodreads_api_key,
                                            config.config_goodreads_api_secret,
                                            config.config_use_goodreads)
 
-    _config_int("config_updatechannel")
+    _config_int(to_save, "config_updatechannel")
 
     # Reverse proxy login configuration
-    _config_checkbox("config_allow_reverse_proxy_header_login")
-    _config_string("config_reverse_proxy_login_header_name")
+    _config_checkbox(to_save, "config_allow_reverse_proxy_header_login")
+    _config_string(to_save, "config_reverse_proxy_login_header_name")
 
-    # GitHub OAuth configuration
+    # OAuth configuration
     if config.config_login_type == constants.LOGIN_OAUTH:
-        active_oauths = 0
-
-        for element in oauthblueprints:
-            if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \
-                or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']:
-                reboot_required = True
-                element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"]
-                element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"]
-            if to_save["config_"+str(element['id'])+"_oauth_client_id"] \
-               and to_save["config_"+str(element['id'])+"_oauth_client_secret"]:
-                active_oauths += 1
-                element["active"] = 1
-            else:
-                element["active"] = 0
-            ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update(
-                {"oauth_client_id":to_save["config_"+str(element['id'])+"_oauth_client_id"],
-                "oauth_client_secret":to_save["config_"+str(element['id'])+"_oauth_client_secret"],
-                "active":element["active"]})
-
-
-    reboot_required |= _config_int("config_log_level")
-    reboot_required |= _config_string("config_logfile")
-    if not logger.is_valid_logfile(config.config_logfile):
-        return _configuration_result(_('Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
-
-    reboot_required |= _config_checkbox_int("config_access_log")
-    reboot_required |= _config_string("config_access_logfile")
-    if not logger.is_valid_logfile(config.config_access_logfile):
-        return _configuration_result(_('Access Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+        reboot_required |= _configuration_oauth_helper(to_save)
 
+    reboot, message = _configuration_logfile_helper(to_save, gdriveError)
+    if message:
+        return message
+    reboot_required |= reboot
     # Rarfile Content configuration
-    _config_string("config_rarfile_location")
+    _config_string(to_save, "config_rarfile_location")
     unrar_status = helper.check_unrar(config.config_rarfile_location)
     if unrar_status:
         return _configuration_result(unrar_status, gdriveError)
@@ -663,9 +696,10 @@ def _configuration_update_helper():
         return _configuration_result('%s' % e, gdriveError)
 
     if db_change:
-        # reload(db)
-        if not db.setup_db(config):
+        if not calibre_db.setup_db(config, ub.app_DB_path):
             return _configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdriveError)
+        if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK):
+            flash(_(u"DB is not Writeable"), category="warning")
 
     config.save()
     flash(_(u"Calibre-Web configuration updated"), category="success")
@@ -701,16 +735,7 @@ def _configuration_result(error_flash=None, gdriveError=None):
                                  title=_(u"Basic Configuration"), page="config")
 
 
-@admi.route("/admin/user/new", methods=["GET", "POST"])
-@login_required
-@admin_required
-def new_user():
-    content = ub.User()
-    languages = speaking_language()
-    translations = [LC('en')] + babel.list_translations()
-    kobo_support = feature_support['kobo'] and config.config_kobo_sync
-    if request.method == "POST":
-        to_save = request.form.to_dict()
+def _handle_new_user(to_save, content,languages, translations, kobo_support):
     content.default_language = to_save["default_language"]
     # content.mature_content = "Show_mature_content" in to_save
     content.locale = to_save.get("locale", content.locale)
@@ -727,9 +752,9 @@ def new_user():
                                      registered_oauth=oauth_check, kobo_support=kobo_support,
                                      title=_(u"Add new user"))
     content.password = generate_password_hash(to_save["password"])
-        existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower())\
+    existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower()) \
         .first()
-        existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower())\
+    existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
         .first()
     if not existing_user and not existing_email:
         content.nickname = to_save["nickname"]
@@ -757,80 +782,9 @@ def new_user():
     except IntegrityError:
         ub.session.rollback()
         flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
-    else:
-        content.role = config.config_default_role
-        content.sidebar_view = config.config_default_show
-    return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
-                                 languages=languages, title=_(u"Add new user"), page="newuser",
-                                 kobo_support=kobo_support, registered_oauth=oauth_check)
 
 
-@admi.route("/admin/mailsettings")
-@login_required
-@admin_required
-def edit_mailsettings():
-    content = config.get_mail_settings()
-    # log.debug("edit_mailsettings %r", content)
-    return render_title_template("email_edit.html", content=content, title=_(u"Edit e-mail server settings"),
-                                 page="mailset")
-
-
-@admi.route("/admin/mailsettings", methods=["POST"])
-@login_required
-@admin_required
-def update_mailsettings():
-    to_save = request.form.to_dict()
-    log.debug("update_mailsettings %r", to_save)
-
-    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
-    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
-
-    _config_string("mail_server")
-    _config_int("mail_port")
-    _config_int("mail_use_ssl")
-    _config_string("mail_login")
-    _config_string("mail_password")
-    _config_string("mail_from")
-    config.save()
-
-    if to_save.get("test"):
-        if current_user.email:
-            result = send_test_mail(current_user.email, current_user.nickname)
-            if result is None:
-                flash(_(u"Test e-mail successfully send to %(kindlemail)s", kindlemail=current_user.email),
-                      category="success")
-            else:
-                flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error")
-        else:
-            flash(_(u"Please configure your e-mail address first..."), category="error")
-    else:
-        flash(_(u"E-mail server settings updated"), category="success")
-
-    return edit_mailsettings()
-
-
-@admi.route("/admin/user/<int:user_id>", methods=["GET", "POST"])
-@login_required
-@admin_required
-def edit_user(user_id):
-    content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()  # type: ub.User
-    if not content:
-        flash(_(u"User not found"), category="error")
-        return redirect(url_for('admin.admin'))
-    downloads = list()
-    languages = speaking_language()
-    translations = babel.list_translations() + [LC('en')]
-    kobo_support = feature_support['kobo'] and config.config_kobo_sync
-    for book in content.downloads:
-        downloadbook = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
-        if downloadbook:
-            downloads.append(downloadbook)
-        else:
-            ub.delete_download(book.book_id)
-            # ub.session.query(ub.Downloads).filter(book.book_id == ub.Downloads.book_id).delete()
-            # ub.session.commit()
-    if request.method == "POST":
-        to_save = request.form.to_dict()
+def _handle_edit_user(to_save, content,languages, translations, kobo_support, downloads):
     if "delete" in to_save:
         if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
                                             ub.User.id != content.id).count():
@@ -885,7 +839,7 @@ def edit_user(user_id):
                 return render_title_template("user_edit.html",
                                              translations=translations,
                                              languages=languages,
-                                                 mail_configured = config.get_mail_server_configured(),
+                                             mail_configured=config.get_mail_server_configured(),
                                              kobo_support=kobo_support,
                                              new_user=0,
                                              content=content,
@@ -917,6 +871,89 @@ def edit_user(user_id):
     except IntegrityError:
         ub.session.rollback()
         flash(_(u"An unknown error occured."), category="error")
+
+
+@admi.route("/admin/user/new", methods=["GET", "POST"])
+@login_required
+@admin_required
+def new_user():
+    content = ub.User()
+    languages = calibre_db.speaking_language()
+    translations = [LC('en')] + babel.list_translations()
+    kobo_support = feature_support['kobo'] and config.config_kobo_sync
+    if request.method == "POST":
+        to_save = request.form.to_dict()
+        _handle_new_user(to_save, content, languages, translations, kobo_support)
+    else:
+        content.role = config.config_default_role
+        content.sidebar_view = config.config_default_show
+    return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
+                                 languages=languages, title=_(u"Add new user"), page="newuser",
+                                 kobo_support=kobo_support, registered_oauth=oauth_check)
+
+
+@admi.route("/admin/mailsettings")
+@login_required
+@admin_required
+def edit_mailsettings():
+    content = config.get_mail_settings()
+    return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"),
+                                 page="mailset")
+
+
+@admi.route("/admin/mailsettings", methods=["POST"])
+@login_required
+@admin_required
+def update_mailsettings():
+    to_save = request.form.to_dict()
+    # log.debug("update_mailsettings %r", to_save)
+
+    _config_string(to_save, "mail_server")
+    _config_int(to_save, "mail_port")
+    _config_int(to_save, "mail_use_ssl")
+    _config_string(to_save, "mail_login")
+    _config_string(to_save, "mail_password")
+    _config_string(to_save, "mail_from")
+    _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024)
+    config.save()
+
+    if to_save.get("test"):
+        if current_user.email:
+            result = send_test_mail(current_user.email, current_user.nickname)
+            if result is None:
+                flash(_(u"Test e-mail successfully send to %(kindlemail)s", kindlemail=current_user.email),
+                      category="success")
+            else:
+                flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error")
+        else:
+            flash(_(u"Please configure your e-mail address first..."), category="error")
+    else:
+        flash(_(u"E-mail server settings updated"), category="success")
+
+    return edit_mailsettings()
+
+
+@admi.route("/admin/user/<int:user_id>", methods=["GET", "POST"])
+@login_required
+@admin_required
+def edit_user(user_id):
+    content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()  # type: ub.User
+    if not content:
+        flash(_(u"User not found"), category="error")
+        return redirect(url_for('admin.admin'))
+    downloads = list()
+    languages = calibre_db.speaking_language()
+    translations = babel.list_translations() + [LC('en')]
+    kobo_support = feature_support['kobo'] and config.config_kobo_sync
+    for book in content.downloads:
+        downloadbook = calibre_db.get_book(book.book_id)
+        if downloadbook:
+            downloads.append(downloadbook)
+        else:
+            ub.delete_download(book.book_id)
+    if request.method == "POST":
+        to_save = request.form.to_dict()
+        _handle_edit_user(to_save, content, languages, translations, kobo_support, downloads)
     return render_title_template("user_edit.html",
                                  translations=translations,
                                  languages=languages,

cps/comic.py

@@ -18,10 +18,17 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
+import io
 
 from . import logger, isoLanguages
 from .constants import BookMeta
 
+try:
+    from PIL import Image as PILImage
+    use_PIL = True
+except ImportError as e:
+    use_PIL = False
+
 
 log = logger.create()
 
@@ -29,18 +36,43 @@ log = logger.create()
 try:
     from comicapi.comicarchive import ComicArchive, MetaDataStyle
     use_comic_meta = True
+    try:
+        from comicapi import __version__ as comic_version
+    except (ImportError):
+        comic_version = ''
 except ImportError as e:
-    log.debug('cannot import comicapi, extracting comic metadata will not work: %s', e)
+    log.debug('Cannot import comicapi, extracting comic metadata will not work: %s', e)
     import zipfile
     import tarfile
     try:
         import rarfile
         use_rarfile = True
     except ImportError as e:
-        log.debug('cannot import rarfile, extracting cover files from rar files will not work: %s', e)
+        log.debug('Cannot import rarfile, extracting cover files from rar files will not work: %s', e)
         use_rarfile = False
     use_comic_meta = False
 
+def _cover_processing(tmp_file_name, img, extension):
+    if use_PIL:
+        # convert to jpg because calibre only supports jpg
+        if extension in ('.png',  '.webp'):
+            imgc = PILImage.open(io.BytesIO(img))
+            im = imgc.convert('RGB')
+            tmp_bytesio = io.BytesIO()
+            im.save(tmp_bytesio, format='JPEG')
+            img = tmp_bytesio.getvalue()
+
+    prefix = os.path.dirname(tmp_file_name)
+    if img:
+        tmp_cover_name = prefix + '/cover.jpg'
+        image = open(tmp_cover_name, 'wb')
+        image.write(img)
+        image.close()
+    else:
+        tmp_cover_name = None
+    return tmp_cover_name
+
+
 
 def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
     cover_data = extension = None
@@ -50,7 +82,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
             ext = os.path.splitext(name)
             if len(ext) > 1:
                 extension = ext[1].lower()
-                if extension == '.jpg' or extension == '.jpeg':
+                if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                     cover_data = archive.getPage(index)
                     break
     else:
@@ -60,7 +92,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
                 ext = os.path.splitext(name)
                 if len(ext) > 1:
                     extension = ext[1].lower()
-                    if extension == '.jpg' or extension == '.jpeg':
+                    if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                         cover_data = cf.read(name)
                         break
         elif original_file_extension.upper() == '.CBT':
@@ -69,7 +101,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
                 ext = os.path.splitext(name)
                 if len(ext) > 1:
                     extension = ext[1].lower()
-                    if extension == '.jpg' or extension == '.jpeg':
+                    if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                         cover_data = cf.extractfile(name).read()
                         break
         elif original_file_extension.upper() == '.CBR' and use_rarfile:
@@ -80,21 +112,12 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
                     ext = os.path.splitext(name)
                     if len(ext) > 1:
                         extension = ext[1].lower()
-                        if extension == '.jpg' or extension == '.jpeg':
+                        if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                             cover_data = cf.read(name)
                             break
             except Exception as e:
                 log.debug('Rarfile failed with error: %s', e)
-
-    prefix = os.path.dirname(tmp_file_name)
-    if cover_data:
-        tmp_cover_name = prefix + '/cover' + extension
-        image = open(tmp_cover_name, 'wb')
-        image.write(cover_data)
-        image.close()
-    else:
-        tmp_cover_name = None
-    return tmp_cover_name
+    return _cover_processing(tmp_file_name, cover_data, extension)
 
 
 def get_comic_info(tmp_file_path, original_file_name, original_file_extension, rarExceutable):

cps/config_sql.py

@@ -53,6 +53,7 @@ class _Settings(_Base):
     mail_login = Column(String, default='mail@example.com')
     mail_password = Column(String, default='mypassword')
     mail_from = Column(String, default='automailer <mail@example.com>')
+    mail_size = Column(Integer, default=25*1024*1024)
 
     config_calibre_dir = Column(String)
     config_port = Column(Integer, default=constants.DEFAULT_PORT)
@@ -96,7 +97,7 @@ class _Settings(_Base):
     config_use_goodreads = Column(Boolean, default=False)
     config_goodreads_api_key = Column(String)
     config_goodreads_api_secret = Column(String)
-
+    config_register_email = Column(Boolean, default=False)
     config_login_type = Column(Integer, default=0)
 
     config_kobo_proxy = Column(Boolean, default=False)
@@ -116,10 +117,11 @@ class _Settings(_Base):
     config_ldap_group_members_field = Column(String, default='memberUid')
     config_ldap_group_name = Column(String, default='calibreweb')
 
-    config_ebookconverter = Column(Integer, default=0)
-    config_converterpath = Column(String)
+    config_kepubifypath = Column(String, default=None)
+    config_converterpath = Column(String, default=None)
     config_calibre = Column(String)
-    config_rarfile_location = Column(String)
+    config_rarfile_location = Column(String, default=None)
+    config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
 
     config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
 
@@ -140,6 +142,22 @@ class _ConfigSQL(object):
         self.config_calibre_dir = None
         self.load()
 
+        change = False
+        if self.config_converterpath == None:
+            change = True
+            self.config_converterpath = autodetect_calibre_binary()
+
+        if self.config_kepubifypath == None:
+            change = True
+            self.config_kepubifypath = autodetect_kepubify_binary()
+
+        if self.config_rarfile_location == None:
+            change = True
+            self.config_rarfile_location = autodetect_unrar_binary()
+        if change:
+            self.save()
+
+
     def _read_from_storage(self):
         if self._settings is None:
             log.debug("_ConfigSQL._read_from_storage")
@@ -264,7 +282,8 @@ class _ConfigSQL(object):
                 setattr(self, k, v)
 
         if self.config_google_drive_watch_changes_response:
-            self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response)
+            self.config_google_drive_watch_changes_response = \
+                json.loads(self.config_google_drive_watch_changes_response)
 
         have_metadata_db = bool(self.config_calibre_dir)
         if have_metadata_db:
@@ -272,8 +291,13 @@ class _ConfigSQL(object):
                 db_file = os.path.join(self.config_calibre_dir, 'metadata.db')
                 have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
-
-        logger.setup(self.config_logfile, self.config_log_level)
+        constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in self.config_upload_formats.split(',')]
+        logfile = logger.setup(self.config_logfile, self.config_log_level)
+        if logfile != self.config_logfile:
+            log.warning("Log path %s not valid, falling back to default", self.config_logfile)
+            self.config_logfile = logfile
+            self._session.merge(s)
+            self._session.commit()
 
     def save(self):
         '''Apply all configuration values to the underlying storage.'''
@@ -334,17 +358,41 @@ def _migrate_table(session, orm_class):
     if changed:
         session.commit()
 
+
 def autodetect_calibre_binary():
     if sys.platform == "win32":
-        calibre_path = ["C:\\program files\calibre\calibre-convert.exe",
-                        "C:\\program files(x86)\calibre\calibre-convert.exe"]
+        calibre_path = ["C:\\program files\calibre\ebook-convert.exe",
+                        "C:\\program files(x86)\calibre\ebook-convert.exe",
+                        "C:\\program files(x86)\calibre2\ebook-convert.exe",
+                        "C:\\program files\calibre2\ebook-convert.exe"]
     else:
         calibre_path = ["/opt/calibre/ebook-convert"]
     for element in calibre_path:
         if os.path.isfile(element) and os.access(element, os.X_OK):
             return element
-    return None
+    return ""
 
+def autodetect_unrar_binary():
+    if sys.platform == "win32":
+        calibre_path = ["C:\\program files\\WinRar\\unRAR.exe",
+                        "C:\\program files(x86)\\WinRar\\unRAR.exe"]
+    else:
+        calibre_path = ["/usr/bin/unrar"]
+    for element in calibre_path:
+        if os.path.isfile(element) and os.access(element, os.X_OK):
+            return element
+    return ""
+
+def autodetect_kepubify_binary():
+    if sys.platform == "win32":
+        calibre_path = ["C:\\program files\\kepubify\\kepubify-windows-64Bit.exe",
+                        "C:\\program files(x86)\\kepubify\\kepubify-windows-64Bit.exe"]
+    else:
+        calibre_path = ["/opt/kepubify/kepubify-linux-64bit", "/opt/kepubify/kepubify-linux-32bit"]
+    for element in calibre_path:
+        if os.path.isfile(element) and os.access(element, os.X_OK):
+            return element
+    return ""
 
 def _migrate_database(session):
     # make sure the table is created, if it does not exist

cps/constants.py

@@ -81,6 +81,7 @@ SIDEBAR_PUBLISHER       = 1 << 12
 SIDEBAR_RATING          = 1 << 13
 SIDEBAR_FORMAT          = 1 << 14
 SIDEBAR_ARCHIVED        = 1 << 15
+# SIDEBAR_LIST            = 1 << 16
 
 ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_ANONYMOUS
 ADMIN_USER_SIDEBAR      = (SIDEBAR_ARCHIVED << 1) - 1
@@ -111,11 +112,9 @@ del env_CALIBRE_PORT
 
 
 EXTENSIONS_AUDIO    = {'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac'}
-EXTENSIONS_CONVERT  = {'pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt'}
-EXTENSIONS_UPLOAD   = {'txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx',
+EXTENSIONS_CONVERT  = ['pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt']
+EXTENSIONS_UPLOAD   = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx',
                        'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac'}
-# EXTENSIONS_READER = set(['txt', 'pdf', 'epub', 'zip', 'cbz', 'tar', 'cbt'] +
-#                         (['rar','cbr'] if feature_support['rar'] else []))
 
 
 def has_flag(value, bit_flag):

cps/converter.py

@@ -19,6 +19,7 @@
 from __future__ import division, print_function, unicode_literals
 import os
 import re
+import sys
 from flask_babel import gettext as _
 
 from . import config, logger
@@ -29,8 +30,8 @@ log = logger.create()
 
 # _() necessary to make babel aware of string for translation
 _NOT_CONFIGURED = _('not configured')
-_NOT_INSTALLED = 'not installed'
-_EXECUTION_ERROR = 'Execution permissions missing'
+_NOT_INSTALLED = _('not installed')
+_EXECUTION_ERROR = _('Execution permissions missing')
 
 
 def _get_command_version(path, pattern, argument=None):
@@ -48,10 +49,15 @@ def _get_command_version(path, pattern, argument=None):
     return _NOT_INSTALLED
 
 
-def get_version():
-    version = None
-    if config.config_ebookconverter == 1:
-        version = _get_command_version(config.config_converterpath, r'Amazon kindlegen\(')
-    elif config.config_ebookconverter == 2:
-        version = _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version')
-    return version or _NOT_CONFIGURED
+def get_calibre_version():
+    return _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') \
+           or _NOT_CONFIGURED
+
+
+def get_unrar_version():
+    return _get_command_version(config.config_rarfile_location, r'UNRAR.*\d') or _NOT_CONFIGURED
+
+def get_kepubify_version():
+    return _get_command_version(config.config_kepubifypath, r'kepubify\s','--version') or _NOT_CONFIGURED
+
+

cps/db.py

@@ -22,18 +22,34 @@ import sys
 import os
 import re
 import ast
+import json
+from datetime import datetime
+import threading
 
 from sqlalchemy import create_engine
-from sqlalchemy import Table, Column, ForeignKey
-from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float, DateTime
+from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
+from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.exc import OperationalError
+from flask_login import current_user
+from sqlalchemy.sql.expression import and_, true, false, text, func, or_
+from babel import Locale as LC
+from babel.core import UnknownLocaleError
+from flask_babel import gettext as _
+
+from . import logger, ub, isoLanguages
+from .pagination import Pagination
+
+try:
+    import unidecode
+    use_unidecode = True
+except ImportError:
+    use_unidecode = False
 
 
-session = None
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
-engine = None
 
 Base = declarative_base()
 
@@ -72,9 +88,9 @@ class Identifiers(Base):
     __tablename__ = 'identifiers'
 
     id = Column(Integer, primary_key=True)
-    type = Column(String)
-    val = Column(String)
-    book = Column(Integer, ForeignKey('books.id'))
+    type = Column(String(collation='NOCASE'), nullable=False, default="isbn")
+    val = Column(String(collation='NOCASE'), nullable=False)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False)
 
     def __init__(self, val, id_type, book):
         self.val = val
@@ -126,8 +142,8 @@ class Comments(Base):
     __tablename__ = 'comments'
 
     id = Column(Integer, primary_key=True)
-    text = Column(String)
-    book = Column(Integer, ForeignKey('books.id'))
+    text = Column(String(collation='NOCASE'), nullable=False)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False)
 
     def __init__(self, text, book):
         self.text = text
@@ -141,7 +157,7 @@ class Tags(Base):
     __tablename__ = 'tags'
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    name = Column(String)
+    name = Column(String(collation='NOCASE'), unique=True, nullable=False)
 
     def __init__(self, name):
         self.name = name
@@ -154,9 +170,9 @@ class Authors(Base):
     __tablename__ = 'authors'
 
     id = Column(Integer, primary_key=True)
-    name = Column(String)
-    sort = Column(String)
-    link = Column(String)
+    name = Column(String(collation='NOCASE'), unique=True, nullable=False)
+    sort = Column(String(collation='NOCASE'))
+    link = Column(String, nullable=False, default="")
 
     def __init__(self, name, sort, link):
         self.name = name
@@ -171,8 +187,8 @@ class Series(Base):
     __tablename__ = 'series'
 
     id = Column(Integer, primary_key=True)
-    name = Column(String)
-    sort = Column(String)
+    name = Column(String(collation='NOCASE'), unique=True, nullable=False)
+    sort = Column(String(collation='NOCASE'))
 
     def __init__(self, name, sort):
         self.name = name
@@ -186,7 +202,7 @@ class Ratings(Base):
     __tablename__ = 'ratings'
 
     id = Column(Integer, primary_key=True)
-    rating = Column(Integer)
+    rating = Column(Integer, CheckConstraint('rating>-1 AND rating<11'), unique=True)
 
     def __init__(self, rating):
         self.rating = rating
@@ -199,7 +215,7 @@ class Languages(Base):
     __tablename__ = 'languages'
 
     id = Column(Integer, primary_key=True)
-    lang_code = Column(String)
+    lang_code = Column(String(collation='NOCASE'), nullable=False, unique=True)
 
     def __init__(self, lang_code):
         self.lang_code = lang_code
@@ -212,8 +228,8 @@ class Publishers(Base):
     __tablename__ = 'publishers'
 
     id = Column(Integer, primary_key=True)
-    name = Column(String)
-    sort = Column(String)
+    name = Column(String(collation='NOCASE'), nullable=False, unique=True)
+    sort = Column(String(collation='NOCASE'))
 
     def __init__(self, name, sort):
         self.name = name
@@ -225,12 +241,13 @@ class Publishers(Base):
 
 class Data(Base):
     __tablename__ = 'data'
+    __table_args__ = {'schema':'calibre'}
 
     id = Column(Integer, primary_key=True)
-    book = Column(Integer, ForeignKey('books.id'))
-    format = Column(String)
-    uncompressed_size = Column(Integer)
-    name = Column(String)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False)
+    format = Column(String(collation='NOCASE'), nullable=False)
+    uncompressed_size = Column(Integer, nullable=False)
+    name = Column(String, nullable=False)
 
     def __init__(self, book, book_format, uncompressed_size, name):
         self.book = book
@@ -247,17 +264,20 @@ class Books(Base):
 
     DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00"
 
-    id = Column(Integer, primary_key=True)
-    title = Column(String)
-    sort = Column(String)
-    author_sort = Column(String)
-    timestamp = Column(TIMESTAMP)
-    pubdate = Column(String)
-    series_index = Column(String)
-    last_modified = Column(TIMESTAMP)
-    path = Column(String)
-    has_cover = Column(Integer)
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    title = Column(String(collation='NOCASE'), nullable=False, default='Unknown')
+    sort = Column(String(collation='NOCASE'))
+    author_sort = Column(String(collation='NOCASE'))
+    timestamp = Column(TIMESTAMP, default=datetime.utcnow)
+    pubdate = Column(String) # , default=datetime.utcnow)
+    series_index = Column(String, nullable=False, default="1.0")
+    last_modified = Column(TIMESTAMP, default=datetime.utcnow)
+    path = Column(String, default="", nullable=False)
+    has_cover = Column(Integer, default=0)
     uuid = Column(String)
+    isbn = Column(String(collation='NOCASE'), default="")
+    # Iccn = Column(String(collation='NOCASE'), default="")
+    flags = Column(Integer, nullable=False, default=1)
 
     authors = relationship('Authors', secondary=books_authors_link, backref='books')
     tags = relationship('Tags', secondary=books_tags_link, backref='books',order_by="Tags.name")
@@ -310,24 +330,47 @@ class Custom_Columns(Base):
         return display_dict
 
 
-def update_title_sort(config, conn=None):
-    # user defined sort function for calibre databases (Series, etc.)
-    def _title_sort(title):
-        # calibre sort stuff
-        title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
-        match = title_pat.search(title)
-        if match:
-            prep = match.group(1)
-            title = title[len(prep):] + ', ' + prep
-        return title.strip()
+class CalibreDB(threading.Thread):
 
-    conn = conn or session.connection().connection.connection
-    conn.create_function("title_sort", 1, _title_sort)
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.engine = None
+        self.session = None
+        self.queue = None
+        self.log = None
+        self.config = None
+
+    def add_queue(self,queue):
+        self.queue = queue
+        self.log = logger.create()
+
+    def run(self):
+        while True:
+            i = self.queue.get()
+            if i == 'dummy':
+                self.queue.task_done()
+                break
+            if i['task'] == 'add_format':
+                cur_book = self.session.query(Books).filter(Books.id == i['id']).first()
+                cur_book.data.append(i['format'])
+                try:
+                    # db.session.merge(cur_book)
+                    self.session.commit()
+                except OperationalError as e:
+                    self.session.rollback()
+                    self.log.error("Database error: %s", e)
+                    # self._handleError(_(u"Database error: %(error)s.", error=e))
+                    # return
+            self.queue.task_done()
 
 
-def setup_db(config):
-    dispose()
-    global engine
+    def stop(self):
+        self.queue.put('dummy')
+
+    def setup_db(self, config, app_db_path):
+        self.config = config
+        self.dispose()
+        # global engine
 
         if not config.config_calibre_dir:
             config.invalidate()
@@ -339,18 +382,22 @@ def setup_db(config):
             return False
 
         try:
-        engine = create_engine('sqlite:///{0}'.format(dbpath),
+            #engine = create_engine('sqlite:///{0}'.format(dbpath),
+            self.engine = create_engine('sqlite://',
                                    echo=False,
                                    isolation_level="SERIALIZABLE",
                                    connect_args={'check_same_thread': False})
-        conn = engine.connect()
+            self.engine.execute("attach database '{}' as calibre;".format(dbpath))
+            self.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
+
+            conn = self.engine.connect()
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as e:
             config.invalidate(e)
             return False
 
         config.db_configured = True
-    update_title_sort(config, conn.connection)
+        self.update_title_sort(config, conn.connection)
 
         if not cc_classes:
             cc = conn.execute("SELECT id, datatype FROM custom_columns")
@@ -390,29 +437,187 @@ def setup_db(config):
 
             for cc_id in cc_ids:
                 if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'):
-                setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]],
+                    setattr(Books,
+                            'custom_column_' + str(cc_id[0]),
+                            relationship(cc_classes[cc_id[0]],
                                          primaryjoin=(
                                          Books.id == cc_classes[cc_id[0]].book),
                                          backref='books'))
                 else:
-                setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]],
+                    setattr(Books,
+                            'custom_column_' + str(cc_id[0]),
+                            relationship(cc_classes[cc_id[0]],
                                          secondary=books_custom_column_links[cc_id[0]],
                                          backref='books'))
 
-
-    global session
         Session = scoped_session(sessionmaker(autocommit=False,
                                               autoflush=False,
-                                             bind=engine))
-    session = Session()
+                                              bind=self.engine))
+        self.session = Session()
         return True
 
+    def get_book(self, book_id):
+        return self.session.query(Books).filter(Books.id == book_id).first()
 
-def dispose():
-    global session
+    def get_filtered_book(self, book_id, allow_show_archived=False):
+        return self.session.query(Books).filter(Books.id == book_id).\
+            filter(self.common_filters(allow_show_archived)).first()
 
-    old_session = session
-    session = None
+    def get_book_by_uuid(self, book_uuid):
+        return self.session.query(Books).filter(Books.uuid == book_uuid).first()
+
+    def get_book_format(self, book_id, format):
+        return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == format).first()
+
+    # Language and content filters for displaying in the UI
+    def common_filters(self, allow_show_archived=False):
+        if not allow_show_archived:
+            archived_books = (
+                ub.session.query(ub.ArchivedBook)
+                    .filter(ub.ArchivedBook.user_id == int(current_user.id))
+                    .filter(ub.ArchivedBook.is_archived == True)
+                    .all()
+            )
+            archived_book_ids = [archived_book.book_id for archived_book in archived_books]
+            archived_filter = Books.id.notin_(archived_book_ids)
+        else:
+            archived_filter = true()
+
+        if current_user.filter_language() != "all":
+            lang_filter = Books.languages.any(Languages.lang_code == current_user.filter_language())
+        else:
+            lang_filter = true()
+        negtags_list = current_user.list_denied_tags()
+        postags_list = current_user.list_allowed_tags()
+        neg_content_tags_filter = false() if negtags_list == [''] else Books.tags.any(Tags.name.in_(negtags_list))
+        pos_content_tags_filter = true() if postags_list == [''] else Books.tags.any(Tags.name.in_(postags_list))
+        if self.config.config_restricted_column:
+            pos_cc_list = current_user.allowed_column_value.split(',')
+            pos_content_cc_filter = true() if pos_cc_list == [''] else \
+                getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
+                    any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
+            neg_cc_list = current_user.denied_column_value.split(',')
+            neg_content_cc_filter = false() if neg_cc_list == [''] else \
+                getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
+                    any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
+        else:
+            pos_content_cc_filter = true()
+            neg_content_cc_filter = false()
+        return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
+                    pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
+
+    # Fill indexpage with all requested data from database
+    def fill_indexpage(self, page, database, db_filter, order, *join):
+        return self.fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
+
+    def fill_indexpage_with_archived_books(self, page, database, db_filter, order, allow_show_archived, *join):
+        if current_user.show_detail_random():
+            randm = self.session.query(Books) \
+                .filter(self.common_filters(allow_show_archived)) \
+                .order_by(func.random()) \
+                .limit(self.config.config_random_books)
+        else:
+            randm = false()
+        off = int(int(self.config.config_books_per_page) * (page - 1))
+        query = self.session.query(database) \
+            .join(*join, isouter=True) \
+            .filter(db_filter) \
+            .filter(self.common_filters(allow_show_archived))
+        pagination = Pagination(page, self.config.config_books_per_page,
+                                len(query.all()))
+        entries = query.order_by(*order).offset(off).limit(self.config.config_books_per_page).all()
+        for book in entries:
+            book = self.order_authors(book)
+        return entries, randm, pagination
+
+    # Orders all Authors in the list according to authors sort
+    def order_authors(self, entry):
+        sort_authors = entry.author_sort.split('&')
+        authors_ordered = list()
+        error = False
+        for auth in sort_authors:
+            # ToDo: How to handle not found authorname
+            result = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).first()
+            if not result:
+                error = True
+                break
+            authors_ordered.append(result)
+        if not error:
+            entry.authors = authors_ordered
+        return entry
+
+    def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()):
+        query = query or ''
+        self.session.connection().connection.connection.create_function("lower", 1, lcase)
+        entries = self.session.query(database).filter(tag_filter). \
+            filter(func.lower(database.name).ilike("%" + query + "%")).all()
+        json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
+        return json_dumps
+
+    def check_exists_book(self, authr, title):
+        self.session.connection().connection.connection.create_function("lower", 1, lcase)
+        q = list()
+        authorterms = re.split(r'\s*&\s*', authr)
+        for authorterm in authorterms:
+            q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
+
+        return self.session.query(Books)\
+            .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
+
+    # read search results from calibre-database and return it (function is used for feed and simple search
+    def get_search_results(self, term):
+        term.strip().lower()
+        self.session.connection().connection.connection.create_function("lower", 1, lcase)
+        q = list()
+        authorterms = re.split("[, ]+", term)
+        for authorterm in authorterms:
+            q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
+
+        return self.session.query(Books).filter(self.common_filters()).filter(
+            or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")),
+                Books.series.any(func.lower(Series.name).ilike("%" + term + "%")),
+                Books.authors.any(and_(*q)),
+                Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")),
+                func.lower(Books.title).ilike("%" + term + "%")
+                )).order_by(Books.sort).all()
+
+    # Creates for all stored languages a translated speaking name in the array for the UI
+    def speaking_language(self, languages=None):
+        from . import get_locale
+
+        if not languages:
+            languages = self.session.query(Languages) \
+                .join(books_languages_link) \
+                .join(Books) \
+                .filter(self.common_filters()) \
+                .group_by(text('books_languages_link.lang_code')).all()
+        for lang in languages:
+            try:
+                cur_l = LC.parse(lang.lang_code)
+                lang.name = cur_l.get_language_name(get_locale())
+            except UnknownLocaleError:
+                lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
+        return languages
+
+    def update_title_sort(self, config, conn=None):
+        # user defined sort function for calibre databases (Series, etc.)
+        def _title_sort(title):
+            # calibre sort stuff
+            title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
+            match = title_pat.search(title)
+            if match:
+                prep = match.group(1)
+                title = title[len(prep):] + ', ' + prep
+            return title.strip()
+
+        conn = conn or self.session.connection().connection.connection
+        conn.create_function("title_sort", 1, _title_sort)
+
+    def dispose(self):
+        # global session
+
+        old_session = self.session
+        self.session = None
         if old_session:
             try: old_session.close()
             except: pass
@@ -434,7 +639,15 @@ def dispose():
                 if table is not None:
                     Base.metadata.remove(table)
 
-def reconnect_db(config):
-    session.close()
-    engine.dispose()
-    setup_db(config)
+    def reconnect_db(self, config, app_db_path):
+        self.session.close()
+        self.engine.dispose()
+        self.setup_db(config, app_db_path)
+
+def lcase(s):
+    try:
+        return unidecode.unidecode(s.lower())
+    except Exception as e:
+        log = logger.create()
+        log.exception(e)
+        return s.lower()

cps/editbooks.py

@@ -24,16 +24,17 @@ from __future__ import division, print_function, unicode_literals
 import os
 from datetime import datetime
 import json
-from shutil import move, copyfile
+from shutil import copyfile
 from uuid import uuid4
 
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
 from flask_babel import gettext as _
 from flask_login import current_user, login_required
+from sqlalchemy.exc import OperationalError
 
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
-from . import config, get_locale, db, ub, worker
-from .helper import order_authors, common_filters
+from . import config, get_locale, ub, worker, db
+from . import calibre_db
 from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
 
 
@@ -174,13 +175,15 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session):
 @login_required
 def delete_book(book_id, book_format):
     if current_user.role_delete_books():
-        book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+        book = calibre_db.get_book(book_id)
         if book:
             try:
                 result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
                 if not result:
                     flash(error, category="error")
                     return redirect(url_for('editbook.edit_book', book_id=book_id))
+                if error:
+                    flash(error, category="warning")
                 if not book_format:
                     # delete book from Shelfs, Downloads, Read list
                     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
@@ -190,13 +193,13 @@ def delete_book(book_id, book_format):
 
                     # check if only this book links to:
                     # author, language, series, tags, custom columns
-                    modify_database_object([u''], book.authors, db.Authors, db.session, 'author')
-                    modify_database_object([u''], book.tags, db.Tags, db.session, 'tags')
-                    modify_database_object([u''], book.series, db.Series, db.session, 'series')
-                    modify_database_object([u''], book.languages, db.Languages, db.session, 'languages')
-                    modify_database_object([u''], book.publishers, db.Publishers, db.session, 'publishers')
+                    modify_database_object([u''], book.authors, db.Authors, calibre_db.session, 'author')
+                    modify_database_object([u''], book.tags, db.Tags, calibre_db.session, 'tags')
+                    modify_database_object([u''], book.series, db.Series, calibre_db.session, 'series')
+                    modify_database_object([u''], book.languages, db.Languages, calibre_db.session, 'languages')
+                    modify_database_object([u''], book.publishers, db.Publishers, calibre_db.session, 'publishers')
 
-                    cc = db.session.query(db.Custom_Columns).\
+                    cc = calibre_db.session.query(db.Custom_Columns).\
                         filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
                     for c in cc:
                         cc_string = "custom_column_" + str(c.id)
@@ -206,32 +209,32 @@ def delete_book(book_id, book_format):
                                     del_cc = getattr(book, cc_string)[0]
                                     getattr(book, cc_string).remove(del_cc)
                                     log.debug('remove ' + str(c.id))
-                                    db.session.delete(del_cc)
-                                    db.session.commit()
+                                    calibre_db.session.delete(del_cc)
+                                    calibre_db.session.commit()
                                 elif c.datatype == 'rating':
                                     del_cc = getattr(book, cc_string)[0]
                                     getattr(book, cc_string).remove(del_cc)
                                     if len(del_cc.books) == 0:
                                         log.debug('remove ' + str(c.id))
-                                        db.session.delete(del_cc)
-                                        db.session.commit()
+                                        calibre_db.session.delete(del_cc)
+                                        calibre_db.session.commit()
                                 else:
                                     del_cc = getattr(book, cc_string)[0]
                                     getattr(book, cc_string).remove(del_cc)
                                     log.debug('remove ' + str(c.id))
-                                    db.session.delete(del_cc)
-                                    db.session.commit()
+                                    calibre_db.session.delete(del_cc)
+                                    calibre_db.session.commit()
                         else:
                             modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id],
-                                                   db.session, 'custom')
-                    db.session.query(db.Books).filter(db.Books.id == book_id).delete()
+                                                   calibre_db.session, 'custom')
+                    calibre_db.session.query(db.Books).filter(db.Books.id == book_id).delete()
                 else:
-                    db.session.query(db.Data).filter(db.Data.book == book.id).\
+                    calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\
                         filter(db.Data.format == book_format).delete()
-                db.session.commit()
+                calibre_db.session.commit()
             except Exception as e:
                 log.debug(e)
-                db.session.rollback()
+                calibre_db.session.rollback()
         else:
             # book not found
             log.error('Book with id "%s" could not be deleted: not found', book_id)
@@ -244,11 +247,9 @@ def delete_book(book_id, book_format):
 
 
 def render_edit_book(book_id):
-    db.update_title_sort(config)
-    cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
-    book = db.session.query(db.Books)\
-        .filter(db.Books.id == book_id).filter(common_filters()).first()
-
+    calibre_db.update_title_sort(config)
+    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    book = calibre_db.get_filtered_book(book_id)
     if not book:
         flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))
@@ -256,7 +257,7 @@ def render_edit_book(book_id):
     for lang in book.languages:
         lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
 
-    book = order_authors(book)
+    book = calibre_db.order_authors(book)
 
     author_names = []
     for authr in book.authors:
@@ -264,19 +265,25 @@ def render_edit_book(book_id):
 
     # Option for showing convertbook button
     valid_source_formats=list()
-    if config.config_ebookconverter == 2:
+    allowed_conversion_formats = list()
+    kepub_possible=None
+    if config.config_converterpath:
         for file in book.data:
             if file.format.lower() in constants.EXTENSIONS_CONVERT:
                 valid_source_formats.append(file.format.lower())
+    if config.config_kepubifypath and 'epub' in [file.format.lower() for file in book.data]:
+        kepub_possible = True
+        if not config.config_converterpath:
+            valid_source_formats.append('epub')
 
     # Determine what formats don't already exist
+    if config.config_converterpath:
         allowed_conversion_formats = constants.EXTENSIONS_CONVERT.copy()
         for file in book.data:
-        try:
+            if file.format.lower() in allowed_conversion_formats:
                 allowed_conversion_formats.remove(file.format.lower())
-        except Exception:
-            log.warning('%s already removed from list.', file.format.lower())
-
+    if kepub_possible:
+        allowed_conversion_formats.append('kepub')
     return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
                                  title=_(u"edit metadata"), page="editbook",
                                  conversion_formats=allowed_conversion_formats,
@@ -293,7 +300,7 @@ def edit_book_ratings(to_save, book):
         ratingx2 = int(float(to_save["rating"]) * 2)
         if ratingx2 != old_rating:
             changed = True
-            is_rating = db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first()
+            is_rating = calibre_db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first()
             if is_rating:
                 book.ratings.append(is_rating)
             else:
@@ -307,15 +314,59 @@ def edit_book_ratings(to_save, book):
             changed = True
     return changed
 
+def edit_book_tags(tags, book):
+    input_tags = tags.split(',')
+    input_tags = list(map(lambda it: it.strip(), input_tags))
+    # if input_tags[0] !="": ??
+    return modify_database_object(input_tags, book.tags, db.Tags, calibre_db.session, 'tags')
 
-def edit_book_languages(to_save, book):
-    input_languages = to_save["languages"].split(',')
+
+def edit_book_series(series, book):
+    input_series = [series.strip()]
+    input_series = [x for x in input_series if x != '']
+    return modify_database_object(input_series, book.series, db.Series, calibre_db.session, 'series')
+
+
+def edit_book_series_index(series_index, book):
+    # Add default series_index to book
+    modif_date = False
+    series_index = series_index or '1'
+    #if series_index == '':
+    #    series_index = '1'
+    if book.series_index != series_index:
+        book.series_index = series_index
+        modif_date = True
+    return modif_date
+
+# Handle book comments/description
+def edit_book_comments(comments, book):
+    modif_date = False
+    if len(book.comments):
+        if book.comments[0].text != comments:
+            book.comments[0].text = comments
+            modif_date = True
+    else:
+        if comments:
+            book.comments.append(db.Comments(text=comments, book=book.id))
+            modif_date = True
+    return modif_date
+
+
+def edit_book_languages(languages, book, upload=False):
+    input_languages = languages.split(',')
     unknown_languages = []
     input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages)
     for l in unknown_languages:
         log.error('%s is not a valid language', l)
-        flash(_(u"%(langname)s is not a valid language", langname=l), category="error")
-    return modify_database_object(list(input_l), book.languages, db.Languages, db.session, 'languages')
+        flash(_(u"%(langname)s is not a valid language", langname=l), category="warning")
+    # ToDo: Not working correct
+    if upload and len(input_l) == 1:
+        # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
+        # the book it's language is set to the filter language
+        if input_l[0] != current_user.filter_language() and current_user.filter_language() != "all":
+            input_l[0] = calibre_db.session.query(db.Languages). \
+                filter(db.Languages.lang_code == current_user.filter_language()).first()
+    return modify_database_object(input_l, book.languages, db.Languages, calibre_db.session, 'languages')
 
 
 def edit_book_publisher(to_save, book):
@@ -323,15 +374,15 @@ def edit_book_publisher(to_save, book):
     if to_save["publisher"]:
         publisher = to_save["publisher"].rstrip().strip()
         if len(book.publishers) == 0 or (len(book.publishers) > 0 and publisher != book.publishers[0].name):
-            changed |= modify_database_object([publisher], book.publishers, db.Publishers, db.session, 'publisher')
+            changed |= modify_database_object([publisher], book.publishers, db.Publishers, calibre_db.session, 'publisher')
     elif len(book.publishers):
-        changed |= modify_database_object([], book.publishers, db.Publishers, db.session, 'publisher')
+        changed |= modify_database_object([], book.publishers, db.Publishers, calibre_db.session, 'publisher')
     return changed
 
 
 def edit_cc_data(book_id, book, to_save):
     changed = False
-    cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     for c in cc:
         cc_string = "custom_column_" + str(c.id)
         if not c.is_multiple:
@@ -354,12 +405,12 @@ def edit_cc_data(book_id, book, to_save):
                             else:
                                 del_cc = getattr(book, cc_string)[0]
                                 getattr(book, cc_string).remove(del_cc)
-                                db.session.delete(del_cc)
+                                calibre_db.session.delete(del_cc)
                                 changed = True
                         else:
                             cc_class = db.cc_classes[c.id]
                             new_cc = cc_class(value=to_save[cc_string], book=book_id)
-                            db.session.add(new_cc)
+                            calibre_db.session.add(new_cc)
                             changed = True
 
                 else:
@@ -371,18 +422,18 @@ def edit_cc_data(book_id, book, to_save):
                             del_cc = getattr(book, cc_string)[0]
                             getattr(book, cc_string).remove(del_cc)
                             if len(del_cc.books) == 0:
-                                db.session.delete(del_cc)
+                                calibre_db.session.delete(del_cc)
                                 changed = True
                         cc_class = db.cc_classes[c.id]
-                        new_cc = db.session.query(cc_class).filter(
+                        new_cc = calibre_db.session.query(cc_class).filter(
                             cc_class.value == to_save[cc_string].strip()).first()
                         # if no cc val is found add it
                         if new_cc is None:
                             new_cc = cc_class(value=to_save[cc_string].strip())
-                            db.session.add(new_cc)
+                            calibre_db.session.add(new_cc)
                             changed = True
-                            db.session.flush()
-                            new_cc = db.session.query(cc_class).filter(
+                            calibre_db.session.flush()
+                            new_cc = calibre_db.session.query(cc_class).filter(
                                 cc_class.value == to_save[cc_string].strip()).first()
                         # add cc value to book
                         getattr(book, cc_string).append(new_cc)
@@ -392,12 +443,15 @@ def edit_cc_data(book_id, book, to_save):
                     del_cc = getattr(book, cc_string)[0]
                     getattr(book, cc_string).remove(del_cc)
                     if not del_cc.books or len(del_cc.books) == 0:
-                        db.session.delete(del_cc)
+                        calibre_db.session.delete(del_cc)
                         changed = True
         else:
             input_tags = to_save[cc_string].split(',')
             input_tags = list(map(lambda it: it.strip(), input_tags))
-            changed |= modify_database_object(input_tags, getattr(book, cc_string), db.cc_classes[c.id], db.session,
+            changed |= modify_database_object(input_tags,
+                                              getattr(book, cc_string),
+                                              db.cc_classes[c.id],
+                                              calibre_db.session,
                                               'custom')
     return changed
 
@@ -435,17 +489,22 @@ def upload_single_file(request, book, book_id):
                 return redirect(url_for('web.show_book', book_id=book.id))
 
             file_size = os.path.getsize(saved_filename)
-            is_format = db.session.query(db.Data).filter(db.Data.book == book_id).\
-                filter(db.Data.format == file_ext.upper()).first()
+            is_format = calibre_db.get_book_format(book_id, file_ext.upper())
 
             # Format entry already exists, no need to update the database
             if is_format:
                 log.warning('Book format %s already existing', file_ext.upper())
             else:
+                try:
                     db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
-                db.session.add(db_format)
-                db.session.commit()
-                db.update_title_sort(config)
+                    calibre_db.session.add(db_format)
+                    calibre_db.session.commit()
+                    calibre_db.update_title_sort(config)
+                except OperationalError as e:
+                    calibre_db.session.rollback()
+                    log.error('Database error: %s', e)
+                    flash(_(u"Database error: %(error)s.", error=e), category="error")
+                    return redirect(url_for('web.show_book', book_id=book.id))
 
             # Queue uploader info
             uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
@@ -454,7 +513,7 @@ def upload_single_file(request, book, book_id):
 
             return uploader.process(
                 saved_filename, *os.path.splitext(requested_file.filename),
-                rarExcecutable=config.config_rarfile_location)
+                rarExecutable=config.config_rarfile_location)
 
 
 def upload_cover(request, book):
@@ -481,9 +540,8 @@ def edit_book(book_id):
         return render_edit_book(book_id)
 
     # create the function for sorting...
-    db.update_title_sort(config)
-    book = db.session.query(db.Books)\
-        .filter(db.Books.id == book_id).filter(common_filters()).first()
+    calibre_db.update_title_sort(config)
+    book = calibre_db.get_filtered_book(book_id)
 
     # Book not found
     if not book:
@@ -514,13 +572,13 @@ def edit_book(book_id):
         if input_authors == ['']:
             input_authors = [_(u'Unknown')]  # prevent empty Author
 
-        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author')
+        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
 
         # Search for each author if author is in database, if not, authorname and sorted authorname is generated new
         # everything then is assembled for sorted author field in database
         sort_authors_list = list()
         for inp in input_authors:
-            stored_author = db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
             if not stored_author:
                 stored_author = helper.get_sorted_author(inp)
             else:
@@ -548,33 +606,21 @@ def edit_book(book_id):
                 else:
                     flash(error, category="error")
 
-            if book.series_index != to_save["series_index"]:
-                book.series_index = to_save["series_index"]
-                modif_date = True
+            # Add default series_index to book
+            modif_date |= edit_book_series_index(to_save["series_index"], book)
 
             # Handle book comments/description
-            if len(book.comments):
-                if book.comments[0].text != to_save["description"]:
-                    book.comments[0].text = to_save["description"]
-                    modif_date = True
-            else:
-                if to_save["description"]:
-                    book.comments.append(db.Comments(text=to_save["description"], book=book.id))
-                    modif_date = True
+            modif_date |= edit_book_comments(to_save["description"], book)
 
                     # Handle identifiers
             input_identifiers = identifier_list(to_save, book)
-            modif_date |= modify_identifiers(input_identifiers, book.identifiers, db.session)
+            modif_date |= modify_identifiers(input_identifiers, book.identifiers, calibre_db.session)
 
             # Handle book tags
-            input_tags = to_save["tags"].split(',')
-            input_tags = list(map(lambda it: it.strip(), input_tags))
-            modif_date |= modify_database_object(input_tags, book.tags, db.Tags, db.session, 'tags')
+            modif_date |= edit_book_tags(to_save['tags'], book)
 
             # Handle book series
-            input_series = [to_save["series"].strip()]
-            input_series = [x for x in input_series if x != '']
-            modif_date |= modify_database_object(input_series, book.series, db.Series, db.session, 'series')
+            modif_date |= edit_book_series(to_save["series"], book)
 
             if to_save["pubdate"]:
                 try:
@@ -588,7 +634,7 @@ def edit_book(book_id):
             modif_date |= edit_book_publisher(to_save, book)
 
             # handle book languages
-            modif_date |= edit_book_languages(to_save, book)
+            modif_date |= edit_book_languages(to_save['languages'], book)
 
             # handle book ratings
             modif_date |= edit_book_ratings(to_save, book)
@@ -598,7 +644,7 @@ def edit_book(book_id):
 
             if modif_date:
                 book.last_modified = datetime.utcnow()
-            db.session.commit()
+            calibre_db.session.commit()
             if config.config_use_google_drive:
                 gdriveutils.updateGdriveCalibreFromLocal()
             if "detail_view" in to_save:
@@ -607,12 +653,12 @@ def edit_book(book_id):
                 flash(_("Metadata successfully updated"), category="success")
                 return render_edit_book(book_id)
         else:
-            db.session.rollback()
+            calibre_db.session.rollback()
             flash(error, category="error")
             return render_edit_book(book_id)
     except Exception as e:
         log.exception(e)
-        db.session.rollback()
+        calibre_db.session.rollback()
         flash(_("Error editing book, please check logfile for details"), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
 
@@ -652,9 +698,11 @@ def upload():
         abort(404)
     if request.method == 'POST' and 'btn-upload' in request.files:
         for requested_file in request.files.getlist("btn-upload"):
+            try:
+                modif_date = False
                 # create the function for sorting...
-            db.update_title_sort(config)
-            db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
+                calibre_db.update_title_sort(config)
+                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
 
                 # check if file extension is correct
                 if '.' in requested_file.filename:
@@ -678,21 +726,44 @@ def upload():
                     return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
                 title = meta.title
                 authr = meta.author
-            tags = meta.tags
-            series = meta.series
-            series_index = meta.series_id
-            title_dir = helper.get_valid_filename(title)
-            author_dir = helper.get_valid_filename(authr)
-            filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir)
-            saved_filename = os.path.join(filepath, title_dir + meta.extension.lower())
 
                 if title != _(u'Unknown') and authr != _(u'Unknown'):
-                entry = helper.check_exists_book(authr, title)
+                    entry = calibre_db.check_exists_book(authr, title)
                     if entry:
                         log.info("Uploaded book probably exists in library")
                         flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
                             + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")
 
+                # handle authors
+                input_authors = authr.split('&')
+                # handle_authors(input_authors)
+                input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
+                # we have all author names now
+                if input_authors == ['']:
+                    input_authors = [_(u'Unknown')]  # prevent empty Author
+
+                sort_authors_list=list()
+                db_author = None
+                for inp in input_authors:
+                    stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+                    if not stored_author:
+                        if not db_author:
+                            db_author = db.Authors(inp, helper.get_sorted_author(inp), "")
+                            calibre_db.session.add(db_author)
+                            calibre_db.session.commit()
+                        sort_author = helper.get_sorted_author(inp)
+                    else:
+                        if not db_author:
+                            db_author = stored_author
+                        sort_author = stored_author.sort
+                    sort_authors_list.append(sort_author) # helper.get_sorted_author(sort_author))
+                sort_authors = ' & '.join(sort_authors_list)
+
+                title_dir = helper.get_valid_filename(title)
+                author_dir = helper.get_valid_filename(db_author.name)
+                filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir)
+                saved_filename = os.path.join(filepath, title_dir + meta.extension.lower())
+
                 # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
                 if not os.path.exists(filepath):
                     try:
@@ -715,116 +786,84 @@ def upload():
                              os.path.join(filepath, "cover.jpg"))
                 else:
                     has_cover = 1
-                try:
-                    copyfile(meta.cover, os.path.join(filepath, "cover.jpg"))
-                    os.unlink(meta.cover)
-                except OSError as e:
-                    log.error("Failed to move cover file %s: %s", os.path.join(filepath, "cover.jpg"), e)
-                    flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=os.path.join(filepath, "cover.jpg"),
-                            error=e),
-                          category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-
-            # handle authors
-            is_author = db.session.query(db.Authors).filter(db.Authors.name == authr).first()
-            if is_author:
-                db_author = is_author
-            else:
-                db_author = db.Authors(authr, helper.get_sorted_author(authr), "")
-                db.session.add(db_author)
-
-            # handle series
-            db_series = None
-            is_series = db.session.query(db.Series).filter(db.Series.name == series).first()
-            if is_series:
-                db_series = is_series
-            elif series != '':
-                db_series = db.Series(series, "")
-                db.session.add(db_series)
-
-            # add language actually one value in list
-            input_language = meta.languages
-            db_language = None
-            if input_language != "":
-                input_language = isoLanguages.get(name=input_language).part3
-                hasLanguage = db.session.query(db.Languages).filter(db.Languages.lang_code == input_language).first()
-                if hasLanguage:
-                    db_language = hasLanguage
-                else:
-                    db_language = db.Languages(input_language)
-                    db.session.add(db_language)
-
-            # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
-            # the book it's language is set to the filter language
-            if db_language != current_user.filter_language() and current_user.filter_language() != "all":
-                db_language = db.session.query(db.Languages).\
-                    filter(db.Languages.lang_code == current_user.filter_language()).first()
 
                 # combine path and normalize path from windows systems
                 path = os.path.join(author_dir, title_dir).replace('\\', '/')
                 # Calibre adds books with utc as timezone
-            db_book = db.Books(title, "", db_author.sort, datetime.utcnow(), datetime(101, 1, 1),
-                               series_index, datetime.utcnow(), path, has_cover, db_author, [], db_language)
-            db_book.authors.append(db_author)
-            if db_series:
-                db_book.series.append(db_series)
-            if db_language is not None:
-                db_book.languages.append(db_language)
-            file_size = os.path.getsize(saved_filename)
-            db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir)
+                db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
+                                   '1', datetime.utcnow(), path, has_cover, db_author, [], "")
+
+                modif_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
+                                                     'author')
+
+                # Add series_index to book
+                modif_date |= edit_book_series_index(meta.series_id, db_book)
+
+                # add languages
+                modif_date |= edit_book_languages(meta.languages, db_book, upload=True)
 
                 # handle tags
-            input_tags = tags.split(',')
-            input_tags = list(map(lambda it: it.strip(), input_tags))
-            if input_tags[0] !="":
-                modify_database_object(input_tags, db_book.tags, db.Tags, db.session, 'tags')
+                modif_date |= edit_book_tags(meta.tags, db_book)
+
+                # handle series
+                modif_date |= edit_book_series(meta.series, db_book)
+
+                # Add file to book
+                file_size = os.path.getsize(saved_filename)
+                db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir)
+                db_book.data.append(db_data)
+                calibre_db.session.add(db_book)
 
                 # flush content, get db_book.id available
-            db_book.data.append(db_data)
-            db.session.add(db_book)
-            db.session.flush()
+                calibre_db.session.flush()
+
+                # Comments needs book id therfore only possiblw after flush
+                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
 
-            # add comment
                 book_id = db_book.id
-            upload_comment = Markup(meta.description).unescape()
-            if upload_comment != "":
-                db.session.add(db.Comments(upload_comment, book_id))
+                title = db_book.title
+
+                error = helper.update_dir_stucture(book_id, config.config_calibre_dir, input_authors[0])
+
+                # move cover to final directory, including book id
+                if has_cover:
+                    try:
+                        new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
+                        copyfile(meta.cover, new_coverpath)
+                        os.unlink(meta.cover)
+                    except OSError as e:
+                        log.error("Failed to move cover file %s: %s", new_coverpath, e)
+                        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
+                                error=e),
+                              category="error")
 
                 # save data to database, reread data
-            db.session.commit()
-            db.update_title_sort(config)
+                calibre_db.session.commit()
+                #calibre_db.setup_db(config, ub.app_DB_path)
                 # Reread book. It's important not to filter the result, as it could have language which hide it from
                 # current users view (tags are not stored/extracted from metadata and could also be limited)
-            book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-            # upload book to gdrive if nesseccary and add "(bookid)" to folder name
-            if config.config_use_google_drive:
-                gdriveutils.updateGdriveCalibreFromLocal()
-            error = helper.update_dir_stucture(book.id, config.config_calibre_dir)
-            db.session.commit()
+                #book = calibre_db.get_book(book_id)
                 if config.config_use_google_drive:
                     gdriveutils.updateGdriveCalibreFromLocal()
                 if error:
                     flash(error, category="error")
-            uploadText=_(u"File %(file)s uploaded", file=book.title)
+                uploadText=_(u"File %(file)s uploaded", file=title)
                 worker.add_upload(current_user.nickname,
-                "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
+                    "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>")
 
-            # create data for displaying display Full language name instead of iso639.part3language
-            if db_language is not None:
-                book.languages[0].language_name = _(meta.languages)
-            author_names = []
-            for author in db_book.authors:
-                author_names.append(author.name)
                 if len(request.files.getlist("btn-upload")) < 2:
                     if current_user.role_edit() or current_user.role_admin():
-                    resp = {"location": url_for('editbook.edit_book', book_id=db_book.id)}
+                        resp = {"location": url_for('editbook.edit_book', book_id=book_id)}
                         return Response(json.dumps(resp), mimetype='application/json')
                     else:
-                    resp = {"location": url_for('web.show_book', book_id=db_book.id)}
+                        resp = {"location": url_for('web.show_book', book_id=book_id)}
                         return Response(json.dumps(resp), mimetype='application/json')
+            except OperationalError as e:
+                calibre_db.session.rollback()
+                log.error("Database error: %s", e)
+                flash(_(u"Database error: %(error)s.", error=e), category="error")
         return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
-
 @editbook.route("/admin/book/convert/<int:book_id>", methods=['POST'])
 @login_required_if_no_ano
 @edit_required

cps/gdrive.py

@@ -39,7 +39,7 @@ try:
 except ImportError:
     pass
 
-from . import logger, gdriveutils, config, db
+from . import logger, gdriveutils, config, ub, calibre_db
 from .web import admin_required
 
 
@@ -145,7 +145,8 @@ def on_received_watch_confirmation():
                         dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
                     else:
                         dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode()
-                    if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != hashlib.md5(dbpath):
+                    if not response['deleted'] and response['file']['title'] == 'metadata.db' \
+                       and response['file']['md5Checksum'] != hashlib.md5(dbpath):
                         tmpDir = tempfile.gettempdir()
                         log.info('Database file updated')
                         copyfile(dbpath, os.path.join(tmpDir, "metadata.db_" + str(current_milli_time())))
@@ -154,7 +155,7 @@ def on_received_watch_confirmation():
                         log.info('Setting up new DB')
                         # prevent error on windows, as os.rename does on exisiting files
                         move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath)
-                        db.setup_db(config)
+                        calibre_db.setup_db(config, ub.app_DB_path)
             except Exception as e:
                 log.exception(e)
         updateMetaData()

cps/helper.py

@@ -23,7 +23,6 @@ import os
 import io
 import json
 import mimetypes
-import random
 import re
 import shutil
 import time
@@ -42,6 +41,7 @@ from flask_login import current_user
 from sqlalchemy.sql.expression import true, false, and_, or_, text, func
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash
+from . import calibre_db
 
 try:
     from urllib.parse import quote
@@ -74,8 +74,8 @@ log = logger.create()
 
 # Convert existing book entry to new format
 def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first()
+    book = calibre_db.get_book(book_id)
+    data = calibre_db.get_book_format(book.id, old_book_format)
     if not data:
         error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
         log.error("convert_book_format: %s", error_message)
@@ -142,9 +142,10 @@ def check_send_to_kindle(entry):
     """
     if len(entry.data):
         bookformats = list()
-        if config.config_ebookconverter == 0:
+        if not config.config_converterpath:
             # no converter - only for mobi and pdf formats
             for ele in iter(entry.data):
+                if ele.uncompressed_size < config.mail_size:
                     if 'MOBI' in ele.format:
                         bookformats.append({'format': 'Mobi',
                                             'convert': 0,
@@ -160,6 +161,7 @@ def check_send_to_kindle(entry):
         else:
             formats = list()
             for ele in iter(entry.data):
+                if ele.uncompressed_size < config.mail_size:
                     formats.append(ele.format)
             if 'MOBI' in formats:
                 bookformats.append({'format': 'Mobi',
@@ -173,14 +175,13 @@ def check_send_to_kindle(entry):
                 bookformats.append({'format': 'Pdf',
                                     'convert': 0,
                                     'text': _('Send %(format)s to Kindle', format='Pdf')})
-            if config.config_ebookconverter >= 1:
+            if config.config_converterpath:
                 if 'EPUB' in formats and not 'MOBI' in formats:
                     bookformats.append({'format': 'Mobi',
                                         'convert':1,
                                         'text': _('Convert %(orig)s to %(format)s and send to Kindle',
                                                   orig='Epub',
                                                   format='Mobi')})
-            if config.config_ebookconverter == 2:
                 if 'AZW3' in formats and not 'MOBI' in formats:
                     bookformats.append({'format': 'Mobi',
                                         'convert': 2,
@@ -211,7 +212,7 @@ def check_read_formats(entry):
 # 3: If Pdf file is existing, it's directly send to kindle email
 def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
     """Send email with attachments"""
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
 
     if convert == 1:
         # returns None if success, otherwise errormessage
@@ -317,13 +318,13 @@ def delete_book_file(book, calibrepath, book_format=None):
                 return True, None
             else:
                 log.error("Deleting book %s failed, book path not valid: %s", book.id, book.path)
-                return False, _("Deleting book %(id)s failed, book path not valid: %(path)s",
+                return True, _("Deleting book %(id)s, book path not valid: %(path)s",
                                      id=book.id,
                                      path=book.path)
 
 
 def update_dir_structure_file(book_id, calibrepath, first_author):
-    localbook = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    localbook = calibre_db.get_book(book_id)
     path = os.path.join(calibrepath, localbook.path)
 
     authordir = localbook.path.split('/')[0]
@@ -382,7 +383,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author):
 
 def update_dir_structure_gdrive(book_id, first_author):
     error = False
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
     path = book.path
 
     authordir = book.path.split('/')[0]
@@ -493,18 +494,17 @@ def get_cover_on_failure(use_generic_cover):
 
 
 def get_book_cover(book_id):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters(allow_show_archived=True)).first()
+    book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     return get_book_cover_internal(book, use_generic_cover_on_failure=True)
 
 
 def get_book_cover_with_uuid(book_uuid,
                              use_generic_cover_on_failure=True):
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     return get_book_cover_internal(book, use_generic_cover_on_failure)
 
 
-def get_book_cover_internal(book,
-                            use_generic_cover_on_failure):
+def get_book_cover_internal(book, use_generic_cover_on_failure):
     if book and book.has_cover:
         if config.config_use_google_drive:
             try:
@@ -594,7 +594,8 @@ def save_cover(img, book_path):
         return save_cover_from_filestorage(os.path.join(config.config_calibre_dir, book_path), "cover.jpg", img)
 
 
-def do_download_file(book, book_format, data, headers):
+
+def do_download_file(book, book_format, client, data, headers):
     if config.config_use_google_drive:
         startTime = time.time()
         df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format)
@@ -608,6 +609,10 @@ def do_download_file(book, book_format, data, headers):
         if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)):
             # ToDo: improve error handling
             log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format))
+
+        if client == "kobo" and book_format == "kepub":
+            headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
+
         response = make_response(send_from_directory(filename, data.name + "." + book_format))
         # ToDo Check headers parameter
         for element in headers:
@@ -629,11 +634,12 @@ def check_unrar(unrarLocation):
             unrarLocation = unrarLocation.encode(sys.getfilesystemencoding())
         unrarLocation = [unrarLocation]
         for lines in process_wait(unrarLocation):
-            value = re.search('UNRAR (.*) freeware', lines)
+            value = re.search('UNRAR (.*) freeware', lines, re.IGNORECASE)
             if value:
                 version = value.group(1)
                 log.debug("unrar version %s", version)
-    except OSError as err:
+                break
+    except (OSError, UnicodeDecodeError) as err:
         log.exception(err)
         return _('Error excecuting UnRar')
 
@@ -719,66 +725,12 @@ def render_task_status(tasklist):
     return renderedtasklist
 
 
-# Language and content filters for displaying in the UI
-def common_filters(allow_show_archived=False):
-    if not allow_show_archived:
-        archived_books = (
-            ub.session.query(ub.ArchivedBook)
-            .filter(ub.ArchivedBook.user_id == int(current_user.id))
-            .filter(ub.ArchivedBook.is_archived == True)
-            .all()
-        )
-        archived_book_ids = [archived_book.book_id for archived_book in archived_books]
-        archived_filter = db.Books.id.notin_(archived_book_ids)
-    else:
-        archived_filter = true()
-
-    if current_user.filter_language() != "all":
-        lang_filter = db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())
-    else:
-        lang_filter = true()
-    negtags_list = current_user.list_denied_tags()
-    postags_list = current_user.list_allowed_tags()
-    neg_content_tags_filter = false() if negtags_list == [''] else db.Books.tags.any(db.Tags.name.in_(negtags_list))
-    pos_content_tags_filter = true() if postags_list == [''] else db.Books.tags.any(db.Tags.name.in_(postags_list))
-    if config.config_restricted_column:
-        pos_cc_list = current_user.allowed_column_value.split(',')
-        pos_content_cc_filter = true() if pos_cc_list == [''] else \
-            getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\
-                    any(db.cc_classes[config.config_restricted_column].value.in_(pos_cc_list))
-        neg_cc_list = current_user.denied_column_value.split(',')
-        neg_content_cc_filter = false() if neg_cc_list == [''] else \
-            getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\
-                    any(db.cc_classes[config.config_restricted_column].value.in_(neg_cc_list))
-    else:
-        pos_content_cc_filter = true()
-        neg_content_cc_filter = false()
-    return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
-                pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
-
-
 def tags_filters():
     negtags_list = current_user.list_denied_tags()
     postags_list = current_user.list_allowed_tags()
     neg_content_tags_filter = false() if negtags_list == [''] else db.Tags.name.in_(negtags_list)
     pos_content_tags_filter = true() if postags_list == [''] else db.Tags.name.in_(postags_list)
     return and_(pos_content_tags_filter, ~neg_content_tags_filter)
-    # return ~(false()) if postags_list == [''] else db.Tags.in_(postags_list)
-
-
-# Creates for all stored languages a translated speaking name in the array for the UI
-def speaking_language(languages=None):
-    if not languages:
-        languages = db.session.query(db.Languages).join(db.books_languages_link).join(db.Books)\
-            .filter(common_filters())\
-            .group_by(text('books_languages_link.lang_code')).all()
-    for lang in languages:
-        try:
-            cur_l = LC.parse(lang.lang_code)
-            lang.name = cur_l.get_language_name(get_locale())
-        except UnknownLocaleError:
-            lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
-    return languages
 
 
 # checks if domain is in database (including wildcards)
@@ -795,93 +747,28 @@ def check_valid_domain(domain_text):
     return not len(result)
 
 
-# Orders all Authors in the list according to authors sort
-def order_authors(entry):
-    sort_authors = entry.author_sort.split('&')
-    authors_ordered = list()
-    error = False
-    for auth in sort_authors:
-        # ToDo: How to handle not found authorname
-        result = db.session.query(db.Authors).filter(db.Authors.sort == auth.lstrip().strip()).first()
-        if not result:
-            error = True
-            break
-        authors_ordered.append(result)
-    if not error:
-        entry.authors = authors_ordered
-    return entry
-
-
-# Fill indexpage with all requested data from database
-def fill_indexpage(page, database, db_filter, order, *join):
-    return fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
-
-
-def fill_indexpage_with_archived_books(page, database, db_filter, order, allow_show_archived, *join):
-    if current_user.show_detail_random():
-        randm = db.session.query(db.Books).filter(common_filters(allow_show_archived))\
-            .order_by(func.random()).limit(config.config_random_books)
-    else:
-        randm = false()
-    off = int(int(config.config_books_per_page) * (page - 1))
-    query = db.session.query(database).join(*join, isouter=True).\
-        filter(db_filter).\
-        filter(common_filters(allow_show_archived))
-    pagination = Pagination(page, config.config_books_per_page,
-                            len(query.all()))
-    entries = query.order_by(*order).offset(off).limit(config.config_books_per_page).all()
-    for book in entries:
-        book = order_authors(book)
-    return entries, randm, pagination
-
-
-def get_typeahead(database, query, replace=('', ''), tag_filter=true()):
-    query = query or ''
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    entries = db.session.query(database).filter(tag_filter).\
-        filter(func.lower(database.name).ilike("%" + query + "%")).all()
-    json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
-    return json_dumps
-
-
-# read search results from calibre-database and return it (function is used for feed and simple search
-def get_search_results(term):
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = list()
-    authorterms = re.split("[, ]+", term)
-    for authorterm in authorterms:
-        q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%")))
-
-    db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + term + "%"))
-
-    return db.session.query(db.Books).filter(common_filters()).filter(
-        or_(db.Books.tags.any(func.lower(db.Tags.name).ilike("%" + term + "%")),
-            db.Books.series.any(func.lower(db.Series.name).ilike("%" + term + "%")),
-            db.Books.authors.any(and_(*q)),
-            db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + term + "%")),
-            func.lower(db.Books.title).ilike("%" + term + "%")
-            )).order_by(db.Books.sort).all()
-
-
-def get_cc_columns():
-    tmpcc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
-    if config.config_columns_to_ignore:
+def get_cc_columns(filter_config_custom_read=False):
+    tmpcc = calibre_db.session.query(db.Custom_Columns)\
+        .filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     cc = []
-        for col in tmpcc:
+    r = None
+    if config.config_columns_to_ignore:
         r = re.compile(config.config_columns_to_ignore)
-            if not r.match(col.name):
+
+    for col in tmpcc:
+        if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id:
+            continue
+        if r and r.match(col.name):
+            continue
         cc.append(col)
-    else:
-        cc = tmpcc
+
     return cc
 
-
-def get_download_link(book_id, book_format):
+def get_download_link(book_id, book_format, client):
     book_format = book_format.split(".")[0]
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id)
     if book:
-        data1 = db.session.query(db.Data).filter(db.Data.book == book.id)\
-            .filter(db.Data.format == book_format.upper()).first()
+        data1 = data = calibre_db.get_book_format(book.id, book_format.upper())
     else:
         abort(404)
     if data1:
@@ -896,28 +783,6 @@ def get_download_link(book_id, book_format):
         headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
         headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % (
             quote(file_name.encode('utf-8')), book_format, quote(file_name.encode('utf-8')), book_format)
-        return do_download_file(book, book_format, data1, headers)
+        return do_download_file(book, book_format, client, data1, headers)
     else:
         abort(404)
-
-
-def check_exists_book(authr, title):
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = list()
-    authorterms = re.split(r'\s*&\s*', authr)
-    for authorterm in authorterms:
-        q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%")))
-
-    return db.session.query(db.Books).filter(
-        and_(db.Books.authors.any(and_(*q)),
-             func.lower(db.Books.title).ilike("%" + title + "%")
-             )).first()
-
-############### Database Helper functions
-
-
-def lcase(s):
-    try:
-        return unidecode.unidecode(s.lower())
-    except Exception as e:
-        log.exception(e)

cps/isoLanguages.py

@@ -57,12 +57,12 @@ def get_language_name(locale, lang_code):
 
 def get_language_codes(locale, language_names, remainder=None):
     language_names = set(x.strip().lower() for x in language_names if x)
-
+    languages = list()
     for k, v in get_language_names(locale).items():
         v = v.lower()
         if v in language_names:
+            languages.append(k)
             language_names.remove(v)
-            yield k
-
     if remainder is not None:
         remainder.extend(language_names)
+    return languages

cps/jinjia.py

@@ -111,10 +111,3 @@ def timestamptodate(date, fmt=None):
 @jinjia.app_template_filter('yesno')
 def yesno(value, yes, no):
     return yes if value else no
-
-
-'''@jinjia.app_template_filter('canread')
-def canread(ext):
-    if isinstance(ext, db.Data):
-        ext = ext.format
-    return ext.lower() in EXTENSIONS_READER'''

cps/kobo.py

@@ -48,7 +48,7 @@ from sqlalchemy.sql.expression import and_, or_
 from sqlalchemy.exc import StatementError
 import requests
 
-from . import config, logger, kobo_auth, db, helper, shelf as shelf_lib, ub
+from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub
 from .services import SyncToken as SyncToken
 from .web import download_required
 from .kobo_auth import requires_kobo_auth
@@ -143,7 +143,7 @@ def HandleSyncRequest():
 
     # We reload the book database so that the user get's a fresh view of the library
     # in case of external changes (e.g: adding a book through Calibre).
-    db.reconnect_db(config)
+    calibre_db.reconnect_db(config, ub.app_DB_path)
 
     archived_books = (
         ub.session.query(ub.ArchivedBook)
@@ -170,7 +170,7 @@ def HandleSyncRequest():
     # It looks like it's treating the db.Books.last_modified field as a string and may fail
     # the comparison because of the +00:00 suffix.
     changed_entries = (
-        db.session.query(db.Books)
+        calibre_db.session.query(db.Books)
         .join(db.Data)
         .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified,
                     db.Books.id.in_(recently_restored_or_archived_books)))
@@ -207,7 +207,7 @@ def HandleSyncRequest():
                      ub.KoboReadingState.user_id == current_user.id,
                      ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements))))
     for kobo_reading_state in changed_reading_states.all():
-        book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none()
+        book = calibre_db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none()
         if book:
             sync_results.append({
                 "ChangedReadingState": {
@@ -256,7 +256,7 @@ def HandleMetadataRequest(book_uuid):
     if not current_app.wsgi_app.is_proxied:
         log.debug('Kobo: Received unproxied request, changed request port to server port')
     log.info("Kobo library metadata request received for book %s" % book_uuid)
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -474,7 +474,7 @@ def add_items_to_shelf(items, shelf):
                 items_unknown_to_calibre.append(item)
                 continue
 
-            book = db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
+            book = calibre_db.get_book_by_uuid(item["RevisionId"])
             if not book:
                 items_unknown_to_calibre.append(item)
                 continue
@@ -545,7 +545,7 @@ def HandleTagRemoveItem(tag_id):
                 items_unknown_to_calibre.append(item)
                 continue
 
-            book = db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
+            book = calibre_db.get_book_by_uuid(item["RevisionId"])
             if not book:
                 items_unknown_to_calibre.append(item)
                 continue
@@ -613,7 +613,7 @@ def create_kobo_tag(shelf):
         "Type": "UserTag"
     }
     for book_shelf in shelf.books:
-        book = db.session.query(db.Books).filter(db.Books.id == book_shelf.book_id).one_or_none()
+        book = calibre_db.get_book(book_shelf.book_id)
         if not book:
             log.info(u"Book (id: %s) in BookShelf (id: %s) not found in book database",  book_shelf.book_id, shelf.id)
             continue
@@ -629,7 +629,7 @@ def create_kobo_tag(shelf):
 @kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
 @login_required
 def HandleStateRequest(book_uuid):
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -804,7 +804,7 @@ def TopLevelEndpoint():
 @login_required
 def HandleBookDeletionRequest(book_uuid):
     log.info("Kobo book deletion request received for book %s" % book_uuid)
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()

cps/logger.py

@@ -82,7 +82,6 @@ def _absolute_log_file(log_file, default_log_file):
         if not os.path.dirname(log_file):
             log_file = os.path.join(_CONFIG_DIR, log_file)
         return os.path.abspath(log_file)
-
     return default_log_file
 
 
@@ -115,7 +114,7 @@ def setup(log_file, log_level=None):
     if previous_handler:
         # if the log_file has not changed, don't create a new handler
         if getattr(previous_handler, 'baseFilename', None) == log_file:
-            return
+            return "" if log_file == DEFAULT_LOG_FILE else log_file
         logging.debug("logging to %s level %s", log_file, r.level)
 
     if log_file == LOG_TO_STDERR or log_file == LOG_TO_STDOUT:
@@ -132,12 +131,14 @@ def setup(log_file, log_level=None):
             if log_file == DEFAULT_LOG_FILE:
                 raise
             file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=50000, backupCount=2)
+            log_file = ""
     file_handler.setFormatter(FORMATTER)
 
     for h in r.handlers:
         r.removeHandler(h)
         h.close()
     r.addHandler(file_handler)
+    return "" if log_file == DEFAULT_LOG_FILE else log_file
 
 
 def create_access_log(log_file, log_name, formatter):
@@ -150,11 +151,18 @@ def create_access_log(log_file, log_name, formatter):
     access_log = logging.getLogger(log_name)
     access_log.propagate = False
     access_log.setLevel(logging.INFO)
-
+    try:
         file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2)
+    except IOError:
+        if log_file == DEFAULT_ACCESS_LOG:
+            raise
+        file_handler = RotatingFileHandler(DEFAULT_ACCESS_LOG, maxBytes=50000, backupCount=2)
+        log_file = ""
+
     file_handler.setFormatter(formatter)
     access_log.addHandler(file_handler)
-    return access_log
+    return access_log, \
+           "" if _absolute_log_file(log_file, DEFAULT_ACCESS_LOG) == DEFAULT_ACCESS_LOG else log_file
 
 
 # Enable logging of smtp lib debug output

cps/opds.py

@@ -30,10 +30,10 @@ from flask_login import current_user
 from sqlalchemy.sql.expression import func, text, or_, and_
 from werkzeug.security import check_password_hash
 
-from . import constants, logger, config, db, ub, services, get_locale, isoLanguages
-from .helper import fill_indexpage, get_download_link, get_book_cover, speaking_language
+from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages
+from .helper import get_download_link, get_book_cover
 from .pagination import Pagination
-from .web import common_filters, get_search_results, render_read_books, download_required
+from .web import render_read_books, download_required
 from flask_babel import gettext as _
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
@@ -100,7 +100,7 @@ def feed_normal_search():
 @requires_basic_auth_if_no_ano
 def feed_new():
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books, True, [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
@@ -108,7 +108,7 @@ def feed_new():
 @opds.route("/opds/discover")
 @requires_basic_auth_if_no_ano
 def feed_discover():
-    entries = db.session.query(db.Books).filter(common_filters()).order_by(func.random())\
+    entries = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(func.random())\
         .limit(config.config_books_per_page)
     pagination = Pagination(1, config.config_books_per_page, int(config.config_books_per_page))
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
@@ -118,7 +118,7 @@ def feed_discover():
 @requires_basic_auth_if_no_ano
 def feed_best_rated():
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
                                                         [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
@@ -133,16 +133,13 @@ def feed_hot():
     hot_books = all_books.offset(off).limit(config.config_books_per_page)
     entries = list()
     for book in hot_books:
-        downloadBook = db.session.query(db.Books).filter(db.Books.id == book.Downloads.book_id).first()
+        downloadBook = calibre_db.get_book(book.Downloads.book_id)
         if downloadBook:
             entries.append(
-                db.session.query(db.Books).filter(common_filters())
-                .filter(db.Books.id == book.Downloads.book_id).first()
+                calibre_db.get_filtered_book(book.Downloads.book_id)
             )
         else:
             ub.delete_download(book.Downloads.book_id)
-            # ub.session.query(ub.Downloads).filter(book.Downloads.book_id == ub.Downloads.book_id).delete()
-            # ub.session.commit()
     numBooks = entries.__len__()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1),
                             config.config_books_per_page, numBooks)
@@ -153,11 +150,13 @@ def feed_hot():
 @requires_basic_auth_if_no_ano
 def feed_authorindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Authors).join(db.books_authors_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).limit(config.config_books_per_page)\
+    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_authors_link.author'))\
+        .order_by(db.Authors.sort).limit(config.config_books_per_page)\
         .offset(off)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Authors).all()))
+                            len(calibre_db.session.query(db.Authors).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_author', pagination=pagination)
 
 
@@ -165,7 +164,7 @@ def feed_authorindex():
 @requires_basic_auth_if_no_ano
 def feed_author(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.authors.any(db.Authors.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -176,11 +175,14 @@ def feed_author(book_id):
 @requires_basic_auth_if_no_ano
 def feed_publisherindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Publishers).join(db.books_publishers_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.sort)\
+    entries = calibre_db.session.query(db.Publishers)\
+        .join(db.books_publishers_link)\
+        .join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(text('books_publishers_link.publisher'))\
+        .order_by(db.Publishers.sort)\
         .limit(config.config_books_per_page).offset(off)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Publishers).all()))
+                            len(calibre_db.session.query(db.Publishers).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_publisher', pagination=pagination)
 
 
@@ -188,7 +190,7 @@ def feed_publisherindex():
 @requires_basic_auth_if_no_ano
 def feed_publisher(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.publishers.any(db.Publishers.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -199,10 +201,16 @@ def feed_publisher(book_id):
 @requires_basic_auth_if_no_ano
 def feed_categoryindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).offset(off).limit(config.config_books_per_page)
+    entries = calibre_db.session.query(db.Tags)\
+        .join(db.books_tags_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_tags_link.tag'))\
+        .order_by(db.Tags.name)\
+        .offset(off)\
+        .limit(config.config_books_per_page)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Tags).all()))
+                            len(calibre_db.session.query(db.Tags).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_category', pagination=pagination)
 
 
@@ -210,7 +218,7 @@ def feed_categoryindex():
 @requires_basic_auth_if_no_ano
 def feed_category(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.tags.any(db.Tags.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -221,10 +229,15 @@ def feed_category(book_id):
 @requires_basic_auth_if_no_ano
 def feed_seriesindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_series_link.series')).order_by(db.Series.sort).offset(off).all()
+    entries = calibre_db.session.query(db.Series)\
+        .join(db.books_series_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_series_link.series'))\
+        .order_by(db.Series.sort)\
+        .offset(off).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Series).all()))
+                            len(calibre_db.session.query(db.Series).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_series', pagination=pagination)
 
 
@@ -232,7 +245,7 @@ def feed_seriesindex():
 @requires_basic_auth_if_no_ano
 def feed_series(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.series.any(db.Series.id == book_id),
                                                         [db.Books.series_index])
@@ -243,10 +256,13 @@ def feed_series(book_id):
 @requires_basic_auth_if_no_ano
 def feed_ratingindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
+    entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                (db.Ratings.rating / 2).label('name')) \
-        .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \
-        .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
+        .join(db.books_ratings_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_ratings_link.rating'))\
+        .order_by(db.Ratings.rating).all()
 
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(entries))
@@ -260,7 +276,7 @@ def feed_ratingindex():
 @requires_basic_auth_if_no_ano
 def feed_ratings(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.ratings.any(db.Ratings.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -271,8 +287,10 @@ def feed_ratings(book_id):
 @requires_basic_auth_if_no_ano
 def feed_formatindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Data).join(db.Books).filter(common_filters()) \
-        .group_by(db.Data.format).order_by(db.Data.format).all()
+    entries = calibre_db.session.query(db.Data).join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(db.Data.format)\
+        .order_by(db.Data.format).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(entries))
 
@@ -286,7 +304,7 @@ def feed_formatindex():
 @requires_basic_auth_if_no_ano
 def feed_format(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.data.any(db.Data.format == book_id.upper()),
                                                         [db.Books.timestamp.desc()])
@@ -299,13 +317,13 @@ def feed_format(book_id):
 def feed_languagesindex():
     off = request.args.get("offset") or 0
     if current_user.filter_language() == u"all":
-        languages = speaking_language()
+        languages = calibre_db.speaking_language()
     else:
         try:
             cur_l = LC.parse(current_user.filter_language())
         except UnknownLocaleError:
             cur_l = None
-        languages = db.session.query(db.Languages).filter(
+        languages = calibre_db.session.query(db.Languages).filter(
             db.Languages.lang_code == current_user.filter_language()).all()
         if cur_l:
             languages[0].name = cur_l.get_language_name(get_locale())
@@ -320,7 +338,7 @@ def feed_languagesindex():
 @requires_basic_auth_if_no_ano
 def feed_languages(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.languages.any(db.Languages.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -356,7 +374,7 @@ def feed_shelf(book_id):
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by(
             ub.BookShelf.order.asc()).all()
         for book in books_in_shelf:
-            cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+            cur_book = calibre_db.get_book(book.book_id)
             result.append(cur_book)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(result))
@@ -367,14 +385,18 @@ def feed_shelf(book_id):
 @requires_basic_auth_if_no_ano
 @download_required
 def opds_download_link(book_id, book_format):
-    return get_download_link(book_id, book_format.lower())
+    if "Kobo" in request.headers.get('User-Agent'):
+        client = "kobo"
+    else:
+        client = ""
+    return get_download_link(book_id, book_format.lower(), client)
 
 
 @opds.route("/ajax/book/<string:uuid>/<library>")
 @opds.route("/ajax/book/<string:uuid>", defaults={'library': ""})
 @requires_basic_auth_if_no_ano
 def get_metadata_calibre_companion(uuid, library):
-    entry = db.session.query(db.Books).filter(db.Books.uuid.like("%" + uuid + "%")).first()
+    entry = calibre_db.session.query(db.Books).filter(db.Books.uuid.like("%" + uuid + "%")).first()
     if entry is not None:
         js = render_template('json.txt', entry=entry)
         response = make_response(js)
@@ -386,8 +408,7 @@ def get_metadata_calibre_companion(uuid, library):
 
 def feed_search(term):
     if term:
-        term = term.strip().lower()
-        entries = get_search_results(term)
+        entries = calibre_db.get_search_results(term)
         entriescount = len(entries) if len(entries) > 0 else 1
         pagination = Pagination(1, entriescount, entriescount)
         return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination)

cps/server.py

@@ -72,7 +72,11 @@ class WebServer(object):
         if config.config_access_log:
             log_name = "gevent.access" if _GEVENT else "tornado.access"
             formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
-            self.access_logger = logger.create_access_log(config.config_access_logfile, log_name, formatter)
+            self.access_logger, logfile = logger.create_access_log(config.config_access_logfile, log_name, formatter)
+            if logfile != config.config_access_logfile:
+                log.warning("Accesslog path %s not valid, falling back to default", config.config_access_logfile)
+                config.config_access_logfile = logfile
+                config.save()
         else:
             if not _GEVENT:
                 logger.get('tornado.access').disabled = True
@@ -196,6 +200,9 @@ class WebServer(object):
     def stop(self, restart=False):
         from . import updater_thread
         updater_thread.stop()
+        from . import calibre_db
+        calibre_db.stop()
+
 
         log.info("webserver stop (restart=%s)", restart)
         self.restart = restart

cps/services/goodreads_support.py

@@ -20,7 +20,10 @@ from __future__ import division, print_function, unicode_literals
 import time
 from functools import reduce
 
-from goodreads.client import GoodreadsClient
+try:
+    from goodreads.client import GoodreadsClient
+except ImportError:
+    from betterreads.client import GoodreadsClient
 
 try: import Levenshtein
 except ImportError: Levenshtein = False
@@ -95,6 +98,10 @@ def get_other_books(author_info, library_books=None):
     for book in author_info.books:
         if book.isbn in identifiers:
             continue
+        if isinstance(book.gid, int):
+            if book.gid in identifiers:
+                continue
+        else:
             if book.gid["#text"] in identifiers:
                 continue
 

cps/shelf.py

@@ -28,9 +28,8 @@ from flask_babel import gettext as _
 from flask_login import login_required, current_user
 from sqlalchemy.sql.expression import func
 
-from . import logger, ub, searched_ids, db
+from . import logger, ub, searched_ids, db, calibre_db
 from .web import render_title_template
-from .helper import common_filters
 
 
 shelf = Blueprint('shelf', __name__)
@@ -320,11 +319,11 @@ def show_shelf(shelf_type, shelf_id):
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\
             .order_by(ub.BookShelf.order.asc()).all()
         for book in books_in_shelf:
-            cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
+            cur_book = calibre_db.get_filtered_book(book.book_id)
             if cur_book:
                 result.append(cur_book)
             else:
-                cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+                cur_book = calibre_db.get_book(book.book_id)
                 if not cur_book:
                     log.info('Not existing book %s in %s deleted', book.book_id, shelf)
                     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete()
@@ -356,7 +355,7 @@ def order_shelf(shelf_id):
         books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
             .order_by(ub.BookShelf.order.asc()).all()
         for book in books_in_shelf2:
-            cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
+            cur_book = calibre_db.get_filtered_book(book.book_id)
             if cur_book:
                 result.append({'title': cur_book.title,
                                'id': cur_book.id,
@@ -364,7 +363,7 @@ def order_shelf(shelf_id):
                                'series': cur_book.series,
                                'series_index': cur_book.series_index})
             else:
-                cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+                cur_book = calibre_db.get_book(book.book_id)
                 result.append({'title': _('Hidden Book'),
                                'id': cur_book.id,
                                'author': [],

cps/static/css/caliBlur_override.css

@@ -0,0 +1,17 @@
+body.serieslist.grid-view div.container-fluid>div>div.col-sm-10:before{
+    display: none;
+}
+
+.cover .badge{
+    position: absolute;
+    top: 0;
+    right: 0;
+    background-color: #cc7b19;
+    border-radius: 0;
+    padding: 0 8px;
+    box-shadow: 0 0 4px rgba(0,0,0,.6);
+    line-height: 24px;
+}
+.cover{
+    box-shadow: 0 0 4px rgba(0,0,0,.6);
+}

cps/static/css/style.css

@@ -109,7 +109,7 @@ a { color: #45b29d; }
 
 .container-fluid .book .cover img {
   border: 1px solid #fff;
-  box-sizeing: border-box;
+  box-sizing: border-box;
   height: 100%;
   bottom: 0;
   position: absolute;
@@ -165,7 +165,7 @@ span.glyphicon.glyphicon-tags {
 
 .container-fluid .single .cover img {
   border: 1px solid #fff;
-  box-sizeing: border-box;
+  box-sizing: border-box;
   -webkit-box-shadow: 0 5px 8px -6px #777;
   -moz-box-shadow: 0 5px 8px -6px #777;
   box-shadow: 0 5px 8px -6px #777;
@@ -174,6 +174,12 @@ span.glyphicon.glyphicon-tags {
 .navbar-default .navbar-toggle .icon-bar {background-color: #000; }
 .navbar-default .navbar-toggle {border-color: #000; }
 .cover { margin-bottom: 10px; }
+.cover .badge{
+   position: absolute;
+   top: 2px;
+   left: 2px;
+   background-color: #777;
+}
 .cover-height { max-height: 100px;}
 
 .col-sm-2 a .cover-small {
@@ -207,7 +213,7 @@ span.glyphicon.glyphicon-tags {
 .panel-body {background-color: #f5f5f5; }
 .spinner {margin: 0 41%; }
 .spinner2 {margin: 0 41%; }
-
+.intend-form { margin-left:20px; }
 table .bg-dark-danger {background-color: #d9534f; color: #fff; }
 table .bg-dark-danger a {color: #fff; }
 table .bg-dark-danger:hover {background-color: #c9302c; }
@@ -296,3 +302,4 @@ div.log {
   white-space: nowrap;
   padding: 0.5em;
 }
+

cps/static/js/filter_grid.js

@@ -0,0 +1,54 @@
+/* This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+ *    Copyright (C) 2018 OzzieIsaacs
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+var $list = $("#list").isotope({
+    itemSelector: ".book",
+    layoutMode: "fitRows",
+    getSortData: {
+        title: ".title",
+    }
+});
+
+$("#desc").click(function() {
+    $list.isotope({
+        sortBy: "name",
+        sortAscending: true
+    });
+    return;
+});
+
+$("#asc").click(function() {
+    $list.isotope({
+        sortBy: "name",
+        sortAscending: false
+    });
+    return;
+});
+
+$("#all").click(function() {
+    // go through all elements and make them visible
+    $list.isotope({ filter: function() {
+        return true;
+      } })
+});
+
+$(".char").click(function() {
+    var character = this.innerText;
+    $list.isotope({ filter: function() {
+        return this.attributes["data-id"].value.charAt(0).toUpperCase() == character;
+      } })
+});

cps/static/js/main.js

@@ -323,4 +323,17 @@ $(function() {
         $(".discover .row").isotope("layout");
     });
 
+    $(".update-view").click(function(e) {
+        var target = $(this).data("target");
+        var view = $(this).data("view");
+
+        e.preventDefault();
+        e.stopPropagation();
+        var data = {};
+        data[target] = view;
+        console.debug("Updating view data: ", data);
+        $.post( "/ajax/view", data).done(function( ) {
+            location.reload();
+        });
+    });
 });

cps/subproc_wrapper.py

@@ -22,7 +22,7 @@ import os
 import subprocess
 
 
-def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE):
+def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE, newlines=True):
     # Linux py2.7 encode as list without quotes no empty element for parameters
     # linux py3.x no encode and as list without quotes no empty element for parameters
     # windows py2.7 encode as string with quotes empty element for parameters is okay
@@ -41,12 +41,13 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro
         else:
             exc_command = [x for x in command]
 
-    return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=True, env=env)
+    return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=newlines, env=env)
 
 
 def process_wait(command, serr=subprocess.PIPE):
     # Run command, wait for process to terminate, and return an iterator over lines of its output.
-    p = process_open(command, serr=serr)
+    newlines = os.name != 'nt'
+    p = process_open(command, serr=serr, newlines=newlines)
     p.wait()
     for line in p.stdout.readlines():
         if isinstance(line, bytes):

cps/templates/admin.html

@@ -13,11 +13,14 @@
             <th>{{_('E-mail Address')}}</th>
             <th>{{_('Send to Kindle E-mail Address')}}</th>
             <th>{{_('Downloads')}}</th>
-            <th class="hidden-xs">{{_('Admin')}}</th>
-            <th class="hidden-xs">{{_('Download')}}</th>
-            <th class="hidden-xs">{{_('View Books')}}</th>
-            <th class="hidden-xs">{{_('Upload')}}</th>
-            <th class="hidden-xs">{{_('Edit')}}</th>
+            <th class="hidden-xs ">{{_('Admin')}}</th>
+            <th class="hidden-xs hidden-sm">{{_('Password')}}</th>
+            <th class="hidden-xs hidden-sm">{{_('Upload')}}</th>
+            <th class="hidden-xs hidden-sm">{{_('Download')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('View Books')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('Edit')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('Delete')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('Public Shelf')}}</th>
         </tr>
         {% for user in allUser %}
           {% if not user.role_anonymous() or config.config_anonbrowse %}
@@ -27,10 +30,13 @@
             <td>{{user.kindle_mail}}</td>
             <td>{{user.downloads.count()}}</td>
             <td class="hidden-xs">{{ display_bool_setting(user.role_admin()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_download()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_viewer()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_upload()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_edit()) }}</td>
+            <td class="hidden-xs hidden-sm">{{ display_bool_setting(user.role_passwd()) }}</td>
+            <td class="hidden-xs hidden-sm">{{ display_bool_setting(user.role_upload()) }}</td>
+            <td class="hidden-xs hidden-sm">{{ display_bool_setting(user.role_download()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_viewer()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_edit()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_delete_books()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_edit_shelfs()) }}</td>
           </tr>
           {% endif %}
         {% endfor %}

cps/templates/book_edit.html

@@ -86,7 +86,7 @@
     </div>
     <div class="form-group">
       <label for="series_index">{{_('Series ID')}}</label>
-      <input type="number" step="0.01" min="0" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
+      <input type="number" step="0.01" min="0" placeholder="1" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
     </div>
     <div class="form-group">
       <label for="rating">{{_('Rating')}}</label>

cps/templates/book_table.html

@@ -0,0 +1,59 @@
+{% extends "layout.html" %}
+{% block body %}
+<h1 class="{{page}}">{{_(title)}}</h1>
+
+    <div class="filterheader hidden-xs hidden-sm">
+      {% if entries.__len__() %}
+       {% if data == 'author' %}
+        <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
+       {% endif %}
+      {% endif %}
+      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      {% if charlist|length %}
+      <button id="all" class="btn btn-primary">{{_('All')}}</button>
+      {% endif %}
+      <div class="btn-group character" role="group">
+        {% for char in charlist%}
+        <button class="btn btn-primary char">{{char.char}}</button>
+        {% endfor %}
+      </div>
+
+      {% if title == "Series" %}
+      <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="grid">Grid</button>
+      {% endif %}
+    </div>
+  <div class="container">
+    <div id="list" class="col-xs-12 col-sm-6">
+    {% for entry in entries %}
+      {% if loop.index0 == (loop.length/2+loop.length%2)|int and loop.length > 20 %}
+        </div>
+        <div id="second" class="col-xs-12 col-sm-6">
+      {% endif %}
+      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
+        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
+        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort='new', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].id )}}{% endif %}">
+          {% if entry.name %}
+          <div class="rating">
+          {% for number in range(entry.name) %}
+            <span class="glyphicon glyphicon-star good"></span>
+            {% if loop.last and loop.index < 5 %}
+              {% for numer in range(5 - loop.index) %}
+                <span class="glyphicon glyphicon-star"></span>
+              {% endfor %}
+            {% endif %}
+          {% endfor %}
+          </div>
+          {% else %}
+          {% if entry.format %}
+            {{entry.format}}
+          {% else %}
+            {{entry[0].name}}{% endif %}{% endif %}</a></div>
+      </div>
+    {% endfor %}
+    </div>
+  </div>
+{% endblock %}
+{% block js %}
+<script src="{{ url_for('static', filename='js/filter_list.js') }}"></script>
+{% endblock %}

cps/templates/config_edit.html

@@ -3,7 +3,7 @@
 <div class="discover">
   <h2>{{title}}</h2>
   <form role="form" method="POST" autocomplete="off">
-<div class="panel-group">
+<div class="panel-group col-md-10 col-lg-6">
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
@@ -15,9 +15,12 @@
     </div>
     <div id="collapseOne" class="panel-collapse collapse in">
       <div class="panel-body">
-    <div class="form-group required">
-      <label for="config_calibre_dir">{{_('Location of Calibre Database')}}</label>
-      <input type="text" class="form-control" name="config_calibre_dir" id="config_calibre_dir" value="{% if config.config_calibre_dir != None %}{{ config.config_calibre_dir }}{% endif %}" autocomplete="off">
+       <div class="form-group required input-group">
+        <label for="config_calibre_dir" class="sr-only">{{_('Location of Calibre Database')}}</label>
+        <input type="text" class="form-control" id="config_calibre_dir" name="config_calibre_dir" value="{% if config.config_calibre_dir != None %}{{ config.config_calibre_dir }}{% endif %}" autocomplete="off">
+        <span class="input-group-btn">
+          <button type="button" id="library_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+        </span>
       </div>
     {% if feature_support['gdrive']  %}
     <div class="form-group required">
@@ -87,21 +90,25 @@
           <label for="config_port">{{_('Server Port')}}</label>
           <input type="number" min="1" max="65535" class="form-control" name="config_port" id="config_port" value="{% if config.config_port != None %}{{ config.config_port }}{% endif %}" autocomplete="off" required>
         </div>
-        <div class="form-group">
         <label for="config_certfile">{{_('SSL certfile location (leave it empty for non-SSL Servers)')}}</label>
-          <input type="text" class="form-control" name="config_certfile" id="config_certfile" value="{% if config.config_certfile != None %}{{ config.config_certfile }}{% endif %}" autocomplete="off">
+         <div class="form-group input-group">
+          <input type="text" class="form-control" id="config_certfile" name="config_certfile" value="{% if config.config_certfile != None %}{{ config.config_certfile }}{% endif %}" autocomplete="off">
+          <span class="input-group-btn">
+            <button type="button" id="certfile_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+          </span>
         </div>
-        <div class="form-group">
-          <label for="config_keyfile">{{_('SSL Keyfile location (leave it empty for non-SSL Servers)')}}</label>
-          <input type="text" class="form-control" name="config_keyfile" id="config_keyfile" value="{% if config.config_keyfile != None %}{{ config.config_keyfile }}{% endif %}" autocomplete="off">
+          <label for="config_calibre_dir" >{{_('SSL Keyfile location (leave it empty for non-SSL Servers)')}}</label>
+         <div class="form-group input-group">
+          <input type="text" class="form-control" id="config_keyfile" name="config_keyfile" value="{% if config.config_keyfile != None %}{{ config.config_keyfile }}{% endif %}" autocomplete="off">
+          <span class="input-group-btn">
+            <button type="button" id="keyfile_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+          </span>
         </div>
         <div class="form-group">
           <label for="config_updatechannel">{{_('Update Channel')}}</label>
             <select name="config_updatechannel" id="config_updatechannel" class="form-control">
               <option value="0" {% if config.config_updatechannel == 0 %}selected{% endif %}>{{_('Stable')}}</option>
-                    <!--option value="1" {% if config.config_updatechannel == 1 %}selected{% endif %}>{{_('Stable (Automatic)')}}</option-->
               <option value="2" {% if config.config_updatechannel == 2 %}selected{% endif %}>{{_('Nightly')}}</option>
-                    <!--option-- value="3" {% if config.config_updatechannel == 3 %}selected{% endif %}>{{_('Nightly (Automatic)')}}</option-->
             </select>
         </div>
       </div>
@@ -154,17 +161,29 @@
     <div id="collapsefive" class="panel-collapse collapse">
       <div class="panel-body">
     <div class="form-group">
-        <input type="checkbox" id="config_uploading" name="config_uploading" {% if config.config_uploading %}checked{% endif %}>
+        <input type="checkbox" id="config_uploading" data-control="upload_settings" name="config_uploading" {% if config.config_uploading %}checked{% endif %}>
         <label for="config_uploading">{{_('Enable Uploads')}}</label>
     </div>
+    <div data-related="upload_settings">
+      <div class="form-group">
+        <label for="config_upload_formats">{{_('Allowed Upload Fileformats')}}</label>
+        <input type="text" class="form-control" name="config_upload_formats" id="config_upload_formats" value="{% if config.config_upload_formats != None %}{{ config.config_upload_formats }}{% endif %}" autocomplete="off">
+      </div>
+    </div>
     <div class="form-group">
         <input type="checkbox" id="config_anonbrowse" name="config_anonbrowse" {% if config.config_anonbrowse %}checked{% endif %}>
         <label for="config_anonbrowse">{{_('Enable Anonymous Browsing')}}</label>
     </div>
     <div class="form-group">
-        <input type="checkbox" id="config_public_reg" name="config_public_reg" {% if config.config_public_reg %}checked{% endif %}>
+        <input type="checkbox" id="config_public_reg" data-control="register_settings" name="config_public_reg" {% if config.config_public_reg %}checked{% endif %}>
         <label for="config_public_reg">{{_('Enable Public Registration')}}</label>
     </div>
+    <div data-related="register_settings">
+      <div class="form-group intend-form">
+        <input type="checkbox" id="config_register_email" name="config_register_email" {% if config.config_register_email %}checked{% endif %}>
+        <label for="config_register_email">{{_('Use E-Mail as Username')}}</label>
+      </div>
+    </div>
     <div class="form-group">
       <input type="checkbox" id="config_remote_login" name="config_remote_login" {% if config.config_remote_login %}checked{% endif %}>
       <label for="config_remote_login">{{_('Enable Magic Link Remote Login')}}</label>
@@ -326,28 +345,31 @@
     </div>
     <div id="collapseeight" class="panel-collapse collapse">
       <div class="panel-body">
-          <div class="form-group">
-              <div><input type="radio" name="config_ebookconverter" id="converter0" value="0" {% if config.config_ebookconverter == 0 %}checked{% endif %}>
-              <label for="converter0">{{_('No Converter')}}</label></div>
-              <div><input type="radio" name="config_ebookconverter" id="converter1" value="1" {% if config.config_ebookconverter == 1 %}checked{% endif %}>
-              <label for="converter1">{{_('Use Kindlegen')}}</label></div>
-              <div><input type="radio" name="config_ebookconverter" id="converter2" value="2" {% if config.config_ebookconverter == 2 %}checked{% endif %}>
-              <label for="converter2">{{_('Use calibre\'s ebook converter')}}</label></div>
+           <label for="config_converterpath">{{_('Path to Calibre E-Book Converter')}}</label>
+           <div class="form-group input-group">
+            <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
+            <span class="input-group-btn">
+              <button type="button" id="converter_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+            </span>
            </div>
-          <div data-related="calibre">
            <div class="form-group">
-                <label for="config_calibre">{{_('E-Book converter settings')}}</label>
+              <label for="config_calibre">{{_('Calibre E-Book Converter Settings')}}</label>
               <input type="text" class="form-control" id="config_calibre" name="config_calibre" value="{% if config.config_calibre != None %}{{ config.config_calibre }}{% endif %}" autocomplete="off">
            </div>
-            <div class="form-group">
-                <label for="config_calibre">{{_('Path to convertertool')}}</label>
-                <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
-            </div>
+        <label for="config_kepubifypath">{{_('Path to Kepubify E-Book Converter')}}</label>
+           <div class="form-group input-group">
+            <input type="text" class="form-control" id="config_kepubifypath" name="config_kepubifypath" value="{% if config.config_kepubifypath != None %}{{ config.config_kepubifypath }}{% endif %}" autocomplete="off">
+            <span class="input-group-btn">
+              <button type="button" id="kepubify_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+            </span>
           </div>
         {% if feature_support['rar'] %}
-            <div class="form-group">
             <label for="config_rarfile_location">{{_('Location of Unrar binary')}}</label>
-                <input type="text" class="form-control" name="config_rarfile_location" id="config_rarfile_location" value="{% if config.config_rarfile_location != None %}{{ config.config_rarfile_location }}{% endif %}" autocomplete="off">
+           <div class="form-group input-group">
+            <input type="text" class="form-control" id="config_rarfile_location" name="config_rarfile_location" value="{% if config.config_rarfile_location != None %}{{ config.config_rarfile_location }}{% endif %}" autocomplete="off">
+            <span class="input-group-btn">
+              <button type="button" id="unrar_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+            </span>
           </div>
         {% endif %}
       </div>

cps/templates/config_view_edit.html

@@ -6,7 +6,7 @@
 {% block body %}
 <div class="discover">
   <h2>{{title}}</h2>
-  <form role="form" method="POST" autocomplete="off">
+  <form role="form" method="POST" autocomplete="off" class="col-md-10 col-lg-6">
 <div class="panel-group">
   <div class="panel panel-default">
     <div class="panel-heading">

cps/templates/detail.html

@@ -202,6 +202,7 @@
             </label>
           </form>
           </p>
+          {% if g.user.check_visibility(32768) %}
           <p>
             <form id="archived_form" action="{{ url_for('web.toggle_archived', book_id=entry.id)}}" method="POST">
               <label class="block-label">
@@ -210,6 +211,7 @@
               </label>
             </form>
           </p>
+          {% endif %}
         </div>
       {% endif %}
 

cps/templates/email_edit.html

@@ -6,14 +6,14 @@
 {% block body %}
 <div class="discover">
   <h1>{{title}}</h1>
-  <form role="form" method="POST">
+  <form role="form" class="col-md-10 col-lg-6" method="POST">
     <div class="form-group">
       <label for="mail_server">{{_('SMTP Hostname')}}</label>
       <input type="text" class="form-control" name="mail_server" id="mail_server" value="{{content.mail_server}}">
     </div>
     <div class="form-group">
       <label for="mail_port">{{_('SMTP Port')}}</label>
-      <input type="text" class="form-control" name="mail_port" id="mail_port" value="{{content.mail_port}}">
+      <input type="number" min="1" max="65535" step="1" class="form-control" name="mail_port" id="mail_port" value="{% if content.mail_port != None %}{{ content.mail_port }}{% endif %}" autocomplete="off">
     </div>
     <div class="form-group">
       <label for="mail_use_ssl">{{_('Encryption')}}</label>
@@ -35,11 +35,19 @@
       <label for="mail_from">{{_('From E-mail')}}</label>
       <input type="text" class="form-control" name="mail_from" id="mail_from" value="{{content.mail_from}}">
     </div>
+    <label for="mail_size">{{_('Attachment Size Limit')}}</label>
+     <div class="form-group input-group">
+      <input type="number" min="1" max="600" step="1" class="form-control" name="mail_size" id="mail_size" value="{% if content.mail_size != None %}{{ (content.mail_size / 1024 / 1024)|int }}{% endif %}">
+      <span class="input-group-btn">
+        <button type="button" id="attachement_size" class="btn btn-default" disabled>MB</button>
+      </span>
+    </div>
     <button type="submit" name="submit" value="submit" class="btn btn-default">{{_('Save')}}</button>
     <button type="submit" name="test" value="test" class="btn btn-default">{{_('Save and Send Test E-mail')}}</button>
     <a href="{{ url_for('admin.admin') }}" id="back" class="btn btn-default">{{_('Cancel')}}</a>
   </form>
     {% if g.allow_registration %}
+  <div class="col-md-10 col-lg-6">
     <h2>{{_('Allowed Domains (Whitelist)')}}</h2>
     <form id="domain_add_allow" action="{{ url_for('admin.add_domain',allow=1)}}" method="POST">
     <div class="form-group required">
@@ -74,6 +82,7 @@
     </div>
     <button id="domain_deny_submit" class="btn btn-default">{{_('Add')}}</button>
     </form>
+    </div>
 
     {% endif %}
 </div>

cps/templates/grid.html

@@ -0,0 +1,49 @@
+{% extends "layout.html" %}
+{% block body %}
+<h1 class="{{page}}">{{_(title)}}</h1>
+
+    <div class="filterheader hidden-xs hidden-sm">
+      {% if entries.__len__() %}
+       {% if entries[0][0].sort %}
+        <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
+       {% endif %}
+      {% endif %}
+      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      {% if charlist|length %}
+      <button id="all" class="btn btn-primary">{{_('All')}}</button>
+      {% endif %}
+      <div class="btn-group character" role="group">
+        {% for char in charlist%}
+        <button class="btn btn-primary char">{{char.char}}</button>
+        {% endfor %}
+      </div>
+
+        <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="list">List</button>
+    </div>
+
+    {% if entries[0] %}
+        <div id="list" class="row">
+          {% for entry in entries %}
+              <div class="col-sm-3 col-lg-2 col-xs-6 book sortable" {% if entry[0].sort %}data-name="{{entry[0].series[0].name}}"{% endif %} data-id="{% if entry[0].series[0].name %}{{entry[0].series[0].name}}{% endif %}">
+                  <div class="cover">
+                      <a href="{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].series[0].id )}}">
+                          <img src="{{ url_for('web.get_cover', book_id=entry[0].id) }}" alt="{{ entry[0].name }}"/>
+                          <span class="badge">{{entry.count}}</span>
+                      </a>
+                  </div>
+                  <div class="meta">
+                      <a href="{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].series[0].id )}}">
+                          <p class="title">{{entry[0].series[0].name|shortentitle}}</p>
+                      </a>
+                  </div>
+              </div>
+        {% endfor %}
+        </div>
+    {% endif %}
+
+
+{% endblock %}
+{% block js %}
+<script src="{{ url_for('static', filename='js/filter_grid.js') }}"></script>
+{% endblock %}

cps/templates/layout.html

@@ -17,6 +17,7 @@
     <link href="{{ url_for('static', filename='css/upload.css') }}" rel="stylesheet" media="screen">
     {% if g.current_theme == 1 %}
        <link href="{{ url_for('static', filename='css/caliBlur.min.css') }}" rel="stylesheet" media="screen">
+       <link href="{{ url_for('static', filename='css/caliBlur_override.css') }}" rel="stylesheet" media="screen">
     {% endif %}
     <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
     <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
@@ -25,7 +26,7 @@
       <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
     <![endif]-->
   </head>
-  <body class="{{ page }}" data-text="{{_('Home')}}" data-textback="{{_('Back')}}">
+  <body class="{{ page }} {{ bodyClass }}" data-text="{{_('Home')}}" data-textback="{{_('Back')}}">
     <!-- Static navbar -->
     <div class="navbar navbar-default navbar-static-top" role="navigation">
       <div class="container-fluid">

cps/templates/list.html

@@ -18,6 +18,10 @@
         <button class="btn btn-primary char">{{char.char}}</button>
         {% endfor %}
       </div>
+
+      {% if data == "series" %}
+      <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="grid">Grid</button>
+      {% endif %}
     </div>
   <div class="container">
     <div id="list" class="col-xs-12 col-sm-6">

cps/templates/register.html

@@ -3,10 +3,12 @@
 <div class="well col-sm-6 col-sm-offset-2">
   <h2 style="margin-top: 0">{{_('Register New Account')}}</h2>
   <form method="POST" role="form">
+    {% if not config.config_register_email %}
     <div class="form-group required">
       <label for="nickname">{{_('Username')}}</label>
       <input type="text" class="form-control" id="nickname" name="nickname" placeholder="{{_('Choose a username')}}" required>
     </div>
+    {% endif %}
     <div class="form-group required">
       <label for="email">{{_('E-mail Address')}}</label>
       <input type="email" class="form-control" id="email" name="email" placeholder="{{_('Your email address')}}" required>

cps/templates/search_form.html

@@ -1,6 +1,6 @@
 {% extends "layout.html" %}
 {% block body %}
-<div class="col-sm-8">
+<div class="col-md-10 col-lg-6">
   <form role="form" id="search" action="{{ url_for('web.advanced_search') }}" method="GET">
     <div class="form-group">
       <label for="book_title">{{_('Book Title')}}</label>

cps/templates/user_edit.html

@@ -3,6 +3,7 @@
 <div class="discover">
   <h1>{{title}}</h1>
   <form role="form" method="POST" autocomplete="off">
+    <div class="col-md-10 col-lg-8">
     {% if new_user or ( g.user and content.nickname != "Guest" and g.user.role_admin() ) %}
     <div class="form-group required">
       <label for="nickname">{{_('Username')}}</label>
@@ -65,6 +66,7 @@
       <div class="btn btn-danger" id="config_delete_kobo_token" data-toggle="modal" data-target="#modalDeleteToken" data-remote="false" {% if not content.remote_auth_token.first() %} style="display: none;" {% endif %}>{{_('Delete')}}</div>
     </div>
     {% endif %}
+    </div>
     <div class="col-sm-6">
         {% for element in sidebar %}
           {% if element['config_show'] %}

cps/ub.py

@@ -49,6 +49,7 @@ from . import constants
 
 
 session = None
+app_DB_path = None
 Base = declarative_base()
 
 
@@ -107,6 +108,11 @@ def get_sidebar_config(kwargs=None):
         {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived",
          "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived",
          "show_text": _('Show archived books'), "config_show": content})
+    '''sidebar.append(
+        {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_list', "id": "list",
+         "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list",
+         "show_text": _('Show Books List'), "config_show": content})'''
+
     return sidebar
 
 
@@ -211,6 +217,7 @@ class User(UserBase, Base):
     denied_column_value = Column(String, default="")
     allowed_column_value = Column(String, default="")
     remote_auth_token = relationship('RemoteAuthToken', backref='user', lazy='dynamic')
+    series_view = Column(String(10), default="list")
 
 
 if oauth_support:
@@ -251,6 +258,7 @@ class Anonymous(AnonymousUserMixin, UserBase):
         self.allowed_tags = data.allowed_tags
         self.denied_column_value = data.denied_column_value
         self.allowed_column_value = data.allowed_column_value
+        self.series_view = data.series_view
 
     def role_admin(self):
         return False
@@ -447,7 +455,7 @@ class RemoteAuthToken(Base):
 
 
 # Migrate database to current version, has to be updated after every database change. Currently migration from
-# everywhere to curent should work. Migration is done by checking if relevant coloums are existing, and than adding
+# everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding
 # rows with SQL commands
 def migrate_Database(session):
     engine = session.bind
@@ -557,6 +565,12 @@ def migrate_Database(session):
         conn.execute("ALTER TABLE user ADD column `denied_column_value` DEFAULT ''")
         conn.execute("ALTER TABLE user ADD column `allowed_column_value` DEFAULT ''")
         session.commit()
+    try:
+        session.query(exists().where(User.series_view)).scalar()
+    except exc.OperationalError:
+        conn = engine.connect()
+        conn.execute("ALTER TABLE user ADD column `series_view` VARCHAR(10) DEFAULT 'list'")
+
     if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() \
         is None:
         create_anonymous_user(session)
@@ -568,7 +582,7 @@ def migrate_Database(session):
         # Create new table user_id and copy contents of table user into it
         conn = engine.connect()
         conn.execute("CREATE TABLE user_id (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,"
-                     " nickname VARCHAR(64),"
+                     "nickname VARCHAR(64),"
                      "email VARCHAR(120),"
                      "role SMALLINT,"
                      "password VARCHAR,"
@@ -576,10 +590,11 @@ def migrate_Database(session):
                      "locale VARCHAR(2),"
                      "sidebar_view INTEGER,"
                      "default_language VARCHAR(3),"
+                     "series_view VARCHAR(10),"
                      "UNIQUE (nickname),"
                      "UNIQUE (email))")
         conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale,"
-                     "sidebar_view, default_language) "
+                     "sidebar_view, default_language, series_view) "
                      "SELECT id, nickname, email, role, password, kindle_mail, locale,"
                      "sidebar_view, default_language FROM user")
         # delete old user table and rename new user_id table to user:
@@ -616,8 +631,7 @@ def delete_download(book_id):
     session.query(Downloads).filter(book_id == Downloads.book_id).delete()
     session.commit()
 
-
-# Generate user Guest (translated text), as anoymous user, no rights
+# Generate user Guest (translated text), as anonymous user, no rights
 def create_anonymous_user(session):
     user = User()
     user.nickname = "Guest"
@@ -651,7 +665,9 @@ def create_admin_user(session):
 def init_db(app_db_path):
     # Open session for database connection
     global session
+    global app_DB_path
 
+    app_DB_path = app_db_path
     engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False)
 
     Session = sessionmaker()

cps/uploader.py

@@ -40,7 +40,7 @@ try:
     from wand.exceptions import PolicyError
     use_generic_pdf_cover = False
 except (ImportError, RuntimeError) as e:
-    log.debug('cannot import Image, generating pdf covers for pdf uploads will not work: %s', e)
+    log.debug('Cannot import Image, generating pdf covers for pdf uploads will not work: %s', e)
     use_generic_pdf_cover = True
 
 try:
@@ -48,21 +48,21 @@ try:
     from PyPDF2 import __version__ as PyPdfVersion
     use_pdf_meta = True
 except ImportError as e:
-    log.debug('cannot import PyPDF2, extracting pdf metadata will not work: %s', e)
+    log.debug('Cannot import PyPDF2, extracting pdf metadata will not work: %s', e)
     use_pdf_meta = False
 
 try:
     from . import epub
     use_epub_meta = True
 except ImportError as e:
-    log.debug('cannot import epub, extracting epub metadata will not work: %s', e)
+    log.debug('Cannot import epub, extracting epub metadata will not work: %s', e)
     use_epub_meta = False
 
 try:
     from . import fb2
     use_fb2_meta = True
 except ImportError as e:
-    log.debug('cannot import fb2, extracting fb2 metadata will not work: %s', e)
+    log.debug('Cannot import fb2, extracting fb2 metadata will not work: %s', e)
     use_fb2_meta = False
 
 try:
@@ -70,20 +70,17 @@ try:
     from PIL import __version__ as PILversion
     use_PIL = True
 except ImportError as e:
-    log.debug('cannot import Pillow, using png and webp images as cover will not work: %s', e)
+    log.debug('Cannot import Pillow, using png and webp images as cover will not work: %s', e)
     use_PIL = False
 
-__author__ = 'lemmsh'
-
 
 def process(tmp_file_path, original_file_name, original_file_extension, rarExecutable):
-    """Get the metadata for tmp_file_path."""
     meta = None
     extension_upper = original_file_extension.upper()
     try:
         if ".PDF" == extension_upper:
             meta = pdf_meta(tmp_file_path, original_file_name, original_file_extension)
-        elif ".EPUB" == extension_upper and use_epub_meta is True:
+        elif extension_upper in [".KEPUB", ".EPUB"] and use_epub_meta is True:
             meta = epub.get_epub_info(tmp_file_path, original_file_name, original_file_extension)
         elif ".FB2" == extension_upper and use_fb2_meta is True:
             meta = fb2.get_fb2_info(tmp_file_path, original_file_extension)
@@ -182,7 +179,7 @@ def get_versions():
     else:
         PILVersion = u'not installed'
     if comic.use_comic_meta:
-        ComicVersion = u'installed'
+        ComicVersion = comic.comic_version or u'installed'
     else:
         ComicVersion = u'not installed'
     return {'Image Magick': IVersion,

cps/web.py

@@ -37,9 +37,9 @@ from flask import Blueprint
 from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for
 from flask_babel import gettext as _
 from flask_login import login_user, logout_user, login_required, current_user
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
 from sqlalchemy.sql.expression import text, func, true, false, not_, and_, or_
-from werkzeug.exceptions import default_exceptions
+from werkzeug.exceptions import default_exceptions, InternalServerError
 from sqlalchemy.sql.functions import coalesce
 try:
     from werkzeug.exceptions import FailedDependency
@@ -48,13 +48,13 @@ except ImportError:
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash, check_password_hash
 
-from . import constants, logger, isoLanguages, services, worker
+from . import constants, logger, isoLanguages, services, worker, cli
 from . import searched_ids, lm, babel, db, ub, config, get_locale, app
+from . import calibre_db
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
-from .helper import common_filters, get_search_results, fill_indexpage, fill_indexpage_with_archived_books, \
-    speaking_language, check_valid_domain, order_authors, get_typeahead, render_task_status, json_serial, \
+from .helper import check_valid_domain, render_task_status, json_serial, \
     get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
-    send_registration_mail, check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password
+    send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password
 from .pagination import Pagination
 from .redirect import redirect_back
 
@@ -124,6 +124,21 @@ if feature_support['ldap']:
         log.debug('LDAP server not accessible while trying to login to opds feed')
         return error_http(FailedDependency())
 
+# @app.errorhandler(InvalidRequestError)
+#@app.errorhandler(OperationalError)
+#def handle_db_exception(e):
+#    db.session.rollback()
+#    log.error('Database request error: %s',e)
+#    return internal_error(InternalServerError(e))
+
+@app.after_request
+def add_security_headers(resp):
+    # resp.headers['Content-Security-Policy']= "script-src 'self' https://www.googleapis.com https://api.douban.com https://comicvine.gamespot.com;"
+    resp.headers['X-Content-Type-Options'] = 'nosniff'
+    resp.headers['X-Frame-Options'] = 'SAMEORIGIN'
+    resp.headers['X-XSS-Protection'] = '1; mode=block'
+    # resp.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains'
+    return resp
 
 web = Blueprint('web', __name__)
 log = logger.create()
@@ -423,21 +438,24 @@ def toggle_read(book_id):
         ub.session.commit()
     else:
         try:
-            db.update_title_sort(config)
-            book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+            calibre_db.update_title_sort(config)
+            book = calibre_db.get_filtered_book(book_id)
             read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
             if len(read_status):
                 read_status[0].value = not read_status[0].value
-                db.session.commit()
+                calibre_db.session.commit()
             else:
                 cc_class = db.cc_classes[config.config_read_column]
                 new_cc = cc_class(value=1, book=book_id)
-                db.session.add(new_cc)
-                db.session.commit()
+                calibre_db.session.add(new_cc)
+                calibre_db.session.commit()
         except KeyError:
             log.error(u"Custom Column No.%d is not exisiting in calibre database", config.config_read_column)
-    return ""
+        except OperationalError as e:
+            calibre_db.session.rollback()
+            log.error(u"Read status could not set: %e", e)
 
+    return ""
 
 @web.route("/ajax/togglearchived/<int:book_id>", methods=['POST'])
 @login_required
@@ -455,11 +473,30 @@ def toggle_archived(book_id):
     return ""
 
 
+@web.route("/ajax/view", methods=["POST"])
+@login_required
+def update_view():
+    to_save = request.form.to_dict()
+    allowed_view = ['grid', 'list']
+    if "series_view" in to_save and to_save["series_view"] in allowed_view:
+        current_user.series_view = to_save["series_view"]
+    else:
+        log.error("Invalid request received: %r %r", request, to_save)
+        return "Invalid request", 400
+
+    try:
+        ub.session.commit()
+    except InvalidRequestError:
+        log.error("Invalid request received: %r ", request, )
+        return "Invalid request", 400
+    return "", 200
+
+
 '''
 @web.route("/ajax/getcomic/<int:book_id>/<book_format>/<int:page>")
 @login_required
 def get_comic_book(book_id, book_format, page):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
     if not book:
         return "", 204
     else:
@@ -513,25 +550,25 @@ def get_comic_book(book_id, book_format, page):
 @web.route("/get_authors_json", methods=['GET'])
 @login_required_if_no_ano
 def get_authors_json():
-    return get_typeahead(db.Authors, request.args.get('q'), ('|', ','))
+    return calibre_db.get_typeahead(db.Authors, request.args.get('q'), ('|', ','))
 
 
 @web.route("/get_publishers_json", methods=['GET'])
 @login_required_if_no_ano
 def get_publishers_json():
-    return get_typeahead(db.Publishers, request.args.get('q'), ('|', ','))
+    return calibre_db.get_typeahead(db.Publishers, request.args.get('q'), ('|', ','))
 
 
 @web.route("/get_tags_json", methods=['GET'])
 @login_required_if_no_ano
 def get_tags_json():
-    return get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters())
+    return calibre_db.get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters())
 
 
 @web.route("/get_series_json", methods=['GET'])
 @login_required_if_no_ano
 def get_series_json():
-    return get_typeahead(db.Series, request.args.get('q'))
+    return calibre_db.get_typeahead(db.Series, request.args.get('q'))
 
 
 @web.route("/get_languages_json", methods=['GET'])
@@ -552,8 +589,8 @@ def get_languages_json():
 @login_required_if_no_ano
 def get_matching_tags():
     tag_dict = {'tags': []}
-    q = db.session.query(db.Books)
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
+    q = calibre_db.session.query(db.Books)
+    calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
     author_input = request.args.get('author_name') or ''
     title_input = request.args.get('book_title') or ''
     include_tag_inputs = request.args.getlist('include_tag') or ''
@@ -583,7 +620,7 @@ def get_matching_tags():
 @web.route('/page/<int:page>')
 @login_required_if_no_ano
 def index(page):
-    entries, random, pagination = fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
+    entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                  title=_(u"Recently Added Books"), page="root")
 
@@ -610,7 +647,9 @@ def books_list(data, sort, book_id, page):
 
     if data == "rated":
         if current_user.check_visibility(constants.SIDEBAR_BEST_RATED):
-            entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
+            entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                    db.Books,
+                                                                    db.Books.ratings.any(db.Ratings.rating > 9),
                                                                     order)
             return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                          id=book_id, title=_(u"Top Rated Books"), page="rated")
@@ -618,7 +657,7 @@ def books_list(data, sort, book_id, page):
             abort(404)
     elif data == "discover":
         if current_user.check_visibility(constants.SIDEBAR_RANDOM):
-            entries, __, pagination = fill_indexpage(page, db.Books, True, [func.randomblob(2)])
+            entries, __, pagination = calibre_db.fill_indexpage(page, db.Books, True, [func.randomblob(2)])
             pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page)
             return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id,
                                          title=_(u"Discover (Random Books)"), page="discover")
@@ -647,7 +686,7 @@ def books_list(data, sort, book_id, page):
     elif data == "archived":
         return render_archived_books(page, order)
     else:
-        entries, random, pagination = fill_indexpage(page, db.Books, True, order)
+        entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, order)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                      title=_(u"Books"), page="newest")
 
@@ -655,7 +694,7 @@ def books_list(data, sort, book_id, page):
 def render_hot_books(page):
     if current_user.check_visibility(constants.SIDEBAR_HOT):
         if current_user.show_detail_random():
-            random = db.session.query(db.Books).filter(common_filters()) \
+            random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \
                 .order_by(func.random()).limit(config.config_random_books)
         else:
             random = false()
@@ -665,7 +704,7 @@ def render_hot_books(page):
         hot_books = all_books.offset(off).limit(config.config_books_per_page)
         entries = list()
         for book in hot_books:
-            downloadBook = db.session.query(db.Books).filter(common_filters()).filter(
+            downloadBook = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter(
                 db.Books.id == book.Downloads.book_id).first()
             if downloadBook:
                 entries.append(downloadBook)
@@ -682,15 +721,18 @@ def render_hot_books(page):
 
 
 def render_author_books(page, author_id, order):
-    entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == author_id),
+    entries, __, pagination = calibre_db.fill_indexpage(page,
+                                                        db.Books,
+                                                        db.Books.authors.any(db.Authors.id == author_id),
                                                         [order[0], db.Series.name, db.Books.series_index],
-                                             db.books_series_link, db.Series)
+                                                        db.books_series_link,
+                                                        db.Series)
     if entries is None or not len(entries):
         flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"),
               category="error")
         return redirect(url_for("web.index"))
 
-    author = db.session.query(db.Authors).get(author_id)
+    author = calibre_db.session.query(db.Authors).get(author_id)
     author_name = author.name.replace('|', ',')
 
     author_info = None
@@ -705,12 +747,14 @@ def render_author_books(page, author_id, order):
 
 
 def render_publisher_books(page, book_id, order):
-    publisher = db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
+    publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
     if publisher:
-        entries, random, pagination = fill_indexpage(page, db.Books,
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
                                                                 db.Books.publishers.any(db.Publishers.id == book_id),
                                                                 [db.Series.name, order[0], db.Books.series_index],
-                                                     db.books_series_link, db.Series)
+                                                                db.books_series_link,
+                                                                db.Series)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
                                      title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher")
     else:
@@ -718,9 +762,11 @@ def render_publisher_books(page, book_id, order):
 
 
 def render_series_books(page, book_id, order):
-    name = db.session.query(db.Series).filter(db.Series.id == book_id).first()
+    name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books, db.Books.series.any(db.Series.id == book_id),
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.series.any(db.Series.id == book_id),
                                                                 [db.Books.series_index, order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
                                      title=_(u"Series: %(serie)s", serie=name.name), page="series")
@@ -729,8 +775,10 @@ def render_series_books(page, book_id, order):
 
 
 def render_ratings_books(page, book_id, order):
-    name = db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
-    entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.id == book_id),
+    name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
+    entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                            db.Books,
+                                                            db.Books.ratings.any(db.Ratings.id == book_id),
                                                             [db.Books.timestamp.desc(), order[0]])
     if name and name.rating <= 10:
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
@@ -740,9 +788,10 @@ def render_ratings_books(page, book_id, order):
 
 
 def render_formats_books(page, book_id, order):
-    name = db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first()
+    name = calibre_db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books,
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
                                                                 db.Books.data.any(db.Data.format == book_id.upper()),
                                                                 [db.Books.timestamp.desc(), order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
@@ -752,9 +801,11 @@ def render_formats_books(page, book_id, order):
 
 
 def render_category_books(page, book_id, order):
-    name = db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
+    name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books, db.Books.tags.any(db.Tags.id == book_id),
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.tags.any(db.Tags.id == book_id),
                                                                 [order[0], db.Series.name, db.Books.series_index],
                                                                 db.books_series_link, db.Series)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
@@ -772,21 +823,29 @@ def render_language_books(page, name, order):
             lang_name = _(isoLanguages.get(part3=name).name)
         except KeyError:
             abort(404)
-    entries, random, pagination = fill_indexpage(page, db.Books, db.Books.languages.any(db.Languages.lang_code == name),
+    entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                            db.Books,
+                                                            db.Books.languages.any(db.Languages.lang_code == name),
                                                             [db.Books.timestamp.desc(), order[0]])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
                                  title=_(u"Language: %(name)s", name=lang_name), page="language")
 
 
+'''@web.route("/table")
+@login_required_if_no_ano
+def books_table():
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
+                                 title=_(u"Language: %(name)s", name=lang_name), page="language")'''
+
 @web.route("/author")
 @login_required_if_no_ano
 def author_list():
     if current_user.check_visibility(constants.SIDEBAR_AUTHOR):
-        entries = db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
-            .join(db.books_authors_link).join(db.Books).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
+            .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).all()
-        charlist = db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \
-            .join(db.books_authors_link).join(db.Books).filter(common_filters()) \
+        charlist = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \
+            .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all()
         for entry in entries:
             entry.Authors.name = entry.Authors.name.replace('|', ',')
@@ -800,11 +859,11 @@ def author_list():
 @login_required_if_no_ano
 def publisher_list():
     if current_user.check_visibility(constants.SIDEBAR_PUBLISHER):
-        entries = db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
-            .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
+            .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.name).all()
-        charlist = db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
-            .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \
+        charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
+            .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Publishers"), page="publisherlist", data="publisher")
@@ -816,14 +875,25 @@ def publisher_list():
 @login_required_if_no_ano
 def series_list():
     if current_user.check_visibility(constants.SIDEBAR_SERIES):
-        entries = db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
-            .join(db.books_series_link).join(db.Books).filter(common_filters()) \
+        if current_user.series_view == 'list':
+            entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
-        charlist = db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
-            .join(db.books_series_link).join(db.Books).filter(common_filters()) \
+            charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
             return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                          title=_(u"Series"), page="serieslist", data="series")
+        else:
+            entries = calibre_db.session.query(db.Books, func.count('books_series_link').label('count')) \
+                .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters()) \
+                .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
+            charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
+                .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
+
+            return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist,
+                                         title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view")
     else:
         abort(404)
 
@@ -832,9 +902,9 @@ def series_list():
 @login_required_if_no_ano
 def ratings_list():
     if current_user.check_visibility(constants.SIDEBAR_RATING):
-        entries = db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
+        entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                    (db.Ratings.rating / 2).label('name')) \
-            .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"Ratings list"), page="ratingslist", data="ratings")
@@ -846,8 +916,10 @@ def ratings_list():
 @login_required_if_no_ano
 def formats_list():
     if current_user.check_visibility(constants.SIDEBAR_FORMAT):
-        entries = db.session.query(db.Data, func.count('data.book').label('count'), db.Data.format.label('format')) \
-            .join(db.Books).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Data,
+                                           func.count('data.book').label('count'),
+                                           db.Data.format.label('format')) \
+            .join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(db.Data.format).order_by(db.Data.format).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"File formats list"), page="formatslist", data="formats")
@@ -861,20 +933,20 @@ def language_overview():
     if current_user.check_visibility(constants.SIDEBAR_LANGUAGE):
         charlist = list()
         if current_user.filter_language() == u"all":
-            languages = speaking_language()
+            languages = calibre_db.speaking_language()
             # ToDo: generate first character list for languages
         else:
             try:
                 cur_l = LC.parse(current_user.filter_language())
             except UnknownLocaleError:
                 cur_l = None
-            languages = db.session.query(db.Languages).filter(
+            languages = calibre_db.session.query(db.Languages).filter(
                 db.Languages.lang_code == current_user.filter_language()).all()
             if cur_l:
                 languages[0].name = cur_l.get_language_name(get_locale())
             else:
                 languages[0].name = _(isoLanguages.get(part3=languages[0].lang_code).name)
-        lang_counter = db.session.query(db.books_languages_link,
+        lang_counter = calibre_db.session.query(db.books_languages_link,
                                         func.count('books_languages_link.book').label('bookcount')).group_by(
             text('books_languages_link.lang_code')).all()
         return render_title_template('languages.html', languages=languages, lang_counter=lang_counter,
@@ -888,11 +960,11 @@ def language_overview():
 @login_required_if_no_ano
 def category_list():
     if current_user.check_visibility(constants.SIDEBAR_CATEGORY):
-        entries = db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
-            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
+            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(calibre_db.common_filters()) \
             .group_by(text('books_tags_link.tag')).all()
-        charlist = db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \
-            .join(db.books_tags_link).join(db.Books).filter(common_filters()) \
+        charlist = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \
+            .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Categories"), page="catlist", data="category")
@@ -912,21 +984,21 @@ def get_tasks_status():
     return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks")
 
 
-# ################################### Search functions ################################################################
-
 @app.route("/reconnect")
 def reconnect():
-    db.reconnect_db(config)
+    db.reconnect_db(config, ub.app_DB_path)
     return json.dumps({})
 
 
+# ################################### Search functions ################################################################
+
+
 @web.route("/search", methods=["GET"])
 @login_required_if_no_ano
 def search():
     term = request.args.get("query")
     if term:
-        term.strip().lower()
-        entries = get_search_results(term)
+        entries = calibre_db.get_search_results(term)
         ids = list()
         for element in entries:
             ids.append(element.id)
@@ -948,9 +1020,9 @@ def search():
 @login_required_if_no_ano
 def advanced_search():
     # Build custom columns names
-    cc = get_cc_columns()
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = db.session.query(db.Books).filter(common_filters()).order_by(db.Books.sort)
+    cc = get_cc_columns(filter_config_custom_read=True)
+    calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
+    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(db.Books.sort)
 
     include_tag_inputs = request.args.getlist('include_tag')
     exclude_tag_inputs = request.args.getlist('exclude_tag')
@@ -1003,13 +1075,13 @@ def advanced_search():
                                                format='medium', locale=get_locale())])
             except ValueError:
                 pub_start = u""
-        tag_names = db.session.query(db.Tags).filter(db.Tags.id.in_(include_tag_inputs)).all()
+        tag_names = calibre_db.session.query(db.Tags).filter(db.Tags.id.in_(include_tag_inputs)).all()
         searchterm.extend(tag.name for tag in tag_names)
-        serie_names = db.session.query(db.Series).filter(db.Series.id.in_(include_series_inputs)).all()
+        serie_names = calibre_db.session.query(db.Series).filter(db.Series.id.in_(include_series_inputs)).all()
         searchterm.extend(serie.name for serie in serie_names)
-        language_names = db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all()
+        language_names = calibre_db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all()
         if language_names:
-            language_names = speaking_language(language_names)
+            language_names = calibre_db.speaking_language(language_names)
         searchterm.extend(language.name for language in language_names)
         if rating_high:
             searchterm.extend([_(u"Rating <= %(rating)s", rating=rating_high)])
@@ -1064,13 +1136,16 @@ def advanced_search():
         # search custom culumns
         for c in cc:
             custom_query = request.args.get('custom_column_' + str(c.id))
-            if custom_query:
+            if custom_query != '' and custom_query is not None:
                 if c.datatype == 'bool':
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         db.cc_classes[c.id].value == (custom_query == "True")))
-                elif c.datatype == 'int':
+                elif c.datatype == 'int' or c.datatype == 'float':
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         db.cc_classes[c.id].value == custom_query))
+                elif c.datatype == 'rating':
+                    q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
+                        db.cc_classes[c.id].value == int(custom_query) * 2))
                 else:
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%")))
@@ -1082,15 +1157,26 @@ def advanced_search():
         return render_title_template('search.html', adv_searchterm=searchterm,
                                      entries=q, title=_(u"search"), page="search")
     # prepare data for search-form
-    tags = db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters()) \
-        .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).all()
-    series = db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters()) \
-        .group_by(text('books_series_link.series')).order_by(db.Series.name).filter(common_filters()).all()
-    extensions = db.session.query(db.Data).join(db.Books).filter(common_filters()) \
-        .group_by(db.Data.format).order_by(db.Data.format).all()
-
+    tags = calibre_db.session.query(db.Tags)\
+        .join(db.books_tags_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_tags_link.tag'))\
+        .order_by(db.Tags.name).all()
+    series = calibre_db.session.query(db.Series)\
+        .join(db.books_series_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_series_link.series'))\
+        .order_by(db.Series.name)\
+        .filter(calibre_db.common_filters()).all()
+    extensions = calibre_db.session.query(db.Data)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(db.Data.format)\
+        .order_by(db.Data.format).all()
     if current_user.filter_language() == u"all":
-        languages = speaking_language()
+        languages = calibre_db.speaking_language()
     else:
         languages = None
     return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions,
@@ -1100,30 +1186,35 @@ def advanced_search():
 def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs):
     order = order or []
     if not config.config_read_column:
-        readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\
-            .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all()
-        readBookIds = [x.book_id for x in readBooks]
         if are_read:
-            db_filter = db.Books.id.in_(readBookIds)
+            db_filter = and_(ub.ReadBook.user_id == int(current_user.id),
+                             ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
         else:
-            db_filter = ~db.Books.id.in_(readBookIds)
-        entries, random, pagination = fill_indexpage(page, db.Books, db_filter, order)
+            db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db_filter,
+                                                                order,
+                                                                ub.ReadBook, db.Books.id==ub.ReadBook.book_id)
     else:
         try:
             if are_read:
                 db_filter = db.cc_classes[config.config_read_column].value == True
             else:
                 db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
-            # book_count = db.session.query(func.count(db.Books.id)).filter(common_filters()).filter(db_filter).scalar()
-            entries, random, pagination = fill_indexpage(page, db.Books,
+            entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                    db.Books,
                                                                     db_filter,
                                                                     order,
                                                                     db.cc_classes[config.config_read_column])
         except KeyError:
             log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
-            book_count = 0
-
-
+            if not as_xml:
+                flash(_("Custom Column No.%(column)d is not existing in calibre database",
+                        column=config.config_read_column),
+                      category="error")
+                return redirect(url_for("web.index"))
+            # ToDo: Handle error Case for opds
     if as_xml:
         return entries, pagination
     else:
@@ -1149,7 +1240,10 @@ def render_archived_books(page, order):
 
     archived_filter = db.Books.id.in_(archived_book_ids)
 
-    entries, random, pagination = fill_indexpage_with_archived_books(page, db.Books, archived_filter, order,
+    entries, random, pagination = calibre_db.fill_indexpage_with_archived_books(page,
+                                                                                db.Books,
+                                                                                archived_filter,
+                                                                                order,
                                                                                 allow_show_archived=True)
 
     name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
@@ -1172,9 +1266,8 @@ def get_cover(book_id):
 @viewer_required
 def serve_book(book_id, book_format, anyname):
     book_format = book_format.split(".")[0]
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == book_format.upper()) \
-        .first()
+    book = calibre_db.get_book(book_id)
+    data = calibre_db.get_book_format(book.id, book_format.upper())
     log.info('Serving book: %s', data.name)
     if config.config_use_google_drive:
         headers = Headers()
@@ -1190,7 +1283,12 @@ def serve_book(book_id, book_format, anyname):
 @login_required_if_no_ano
 @download_required
 def download_link(book_id, book_format, anyname):
-    return get_download_link(book_id, book_format.lower())
+    if "Kobo" in request.headers.get('User-Agent'):
+        client = "kobo"
+    else:
+        client=""
+
+    return get_download_link(book_id, book_format, client)
 
 
 @web.route('/send/<int:book_id>/<book_format>/<int:convert>')
@@ -1231,30 +1329,33 @@ def register():
 
     if request.method == "POST":
         to_save = request.form.to_dict()
-        if not to_save["nickname"] or not to_save["email"]:
+        if config.config_register_email:
+            nickname = to_save["email"]
+        else:
+            nickname = to_save["nickname"]
+        if not nickname or not to_save["email"]:
             flash(_(u"Please fill out all fields!"), category="error")
             return render_title_template('register.html', title=_(u"register"), page="register")
 
-        existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"]
+
+        existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == nickname
                                                          .lower()).first()
         existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()).first()
         if not existing_user and not existing_email:
             content = ub.User()
-            # content.password = generate_password_hash(to_save["password"])
             if check_valid_domain(to_save["email"]):
-                content.nickname = to_save["nickname"]
+                content.nickname = nickname
                 content.email = to_save["email"]
                 password = generate_random_password()
                 content.password = generate_password_hash(password)
                 content.role = config.config_default_role
                 content.sidebar_view = config.config_default_show
-                # content.mature_content = bool(config.config_default_show & constants.MATURE_CONTENT)
                 try:
                     ub.session.add(content)
                     ub.session.commit()
                     if feature_support['oauth']:
                         register_user_with_oauth(content)
-                    send_registration_mail(to_save["email"], to_save["nickname"], password)
+                    send_registration_mail(to_save["email"], nickname, password)
                 except Exception:
                     ub.session.rollback()
                     flash(_(u"An unknown error occurred. Please try again later."), category="error")
@@ -1448,7 +1549,7 @@ def token_verified():
 @login_required
 def profile():
     downloads = list()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = babel.list_translations() + [LC('en')]
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     if feature_support['oauth']:
@@ -1457,9 +1558,9 @@ def profile():
         oauth_status = None
 
     for book in current_user.downloads:
-        downloadBook = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+        downloadBook = calibre_db.get_book(book.book_id)
         if downloadBook:
-            downloads.append(db.session.query(db.Books).filter(db.Books.id == book.book_id).first())
+            downloads.append(downloadBook)
         else:
             ub.delete_download(book.book_id)
     if request.method == "POST":
@@ -1538,7 +1639,7 @@ def profile():
 @login_required_if_no_ano
 @viewer_required
 def read_book(book_id, book_format):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id)
     if not book:
         flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error")
         log.debug(u"Error opening eBook. File does not exist or file is not accessible:")
@@ -1562,7 +1663,7 @@ def read_book(book_id, book_format):
     else:
         for fileExt in constants.EXTENSIONS_AUDIO:
             if book_format.lower() == fileExt:
-                entries = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+                entries = calibre_db.get_filtered_book(book_id)
                 log.debug(u"Start mp3 listening for %d", book_id)
                 return render_title_template('listenmp3.html', mp3file=book_id, audioformat=book_format.lower(),
                                              title=_(u"Read a Book"), entry=entries, bookmark=bookmark)
@@ -1588,8 +1689,7 @@ def read_book(book_id, book_format):
 @web.route("/book/<int:book_id>")
 @login_required_if_no_ano
 def show_book(book_id):
-    entries = db.session.query(db.Books).filter(and_(db.Books.id == book_id,
-                                                     common_filters(allow_show_archived=True))).first()
+    entries = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     if entries:
         for index in range(0, len(entries.languages)):
             try:
@@ -1598,7 +1698,7 @@ def show_book(book_id):
             except UnknownLocaleError:
                 entries.languages[index].language_name = _(
                     isoLanguages.get(part3=entries.languages[index].lang_code).name)
-        cc = get_cc_columns()
+        cc = get_cc_columns(filter_config_custom_read=True)
         book_in_shelfs = []
         shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all()
         for entry in shelfs:
@@ -1629,7 +1729,7 @@ def show_book(book_id):
 
         entries.tags = sort(entries.tags, key=lambda tag: tag.name)
 
-        entries = order_authors(entries)
+        entries = calibre_db.order_authors(entries)
 
         kindle_list = check_send_to_kindle(entries)
         reader_list = check_read_formats(entries)

cps/worker.py

@@ -24,6 +24,12 @@ import smtplib
 import socket
 import time
 import threading
+try:
+    import queue
+except ImportError:
+    import Queue as queue
+from glob import glob
+from shutil import copyfile
 from datetime import datetime
 
 try:
@@ -43,9 +49,10 @@ from email.utils import make_msgid
 from email.generator import Generator
 from flask_babel import gettext as _
 
-from . import logger, config, db, gdriveutils
+from . import calibre_db, db
+from . import logger, config
 from .subproc_wrapper import process_open
-
+from . import gdriveutils
 
 log = logger.create()
 
@@ -187,6 +194,8 @@ class WorkerThread(threading.Thread):
         self.UIqueue = list()
         self.asyncSMTP = None
         self.id = 0
+        self.db_queue = queue.Queue()
+        calibre_db.add_queue(self.db_queue)
         self.doLock = threading.Lock()
 
     # Main thread loop starting the different tasks
@@ -273,7 +282,7 @@ class WorkerThread(threading.Thread):
         index = self.current
         self.doLock.release()
         file_path = self.queue[index]['file_path']
-        bookid = self.queue[index]['bookid']
+        book_id = self.queue[index]['bookid']
         format_old_ext = u'.' + self.queue[index]['settings']['old_book_format'].lower()
         format_new_ext = u'.' + self.queue[index]['settings']['new_book_format'].lower()
 
@@ -281,34 +290,74 @@ class WorkerThread(threading.Thread):
         # if it does - mark the conversion task as complete and return a success
         # this will allow send to kindle workflow to continue to work
         if os.path.isfile(file_path + format_new_ext):
-            log.info("Book id %d already converted to %s", bookid, format_new_ext)
-            cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first()
+            log.info("Book id %d already converted to %s", book_id, format_new_ext)
+            cur_book = calibre_db.get_book(book_id)
             self.queue[index]['path'] = file_path
             self.queue[index]['title'] = cur_book.title
             self._handleSuccess()
             return file_path + format_new_ext
         else:
-            log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", bookid, format_new_ext)
+            log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
+                     book_id,
+                     format_new_ext)
 
-        # check if converter-executable is existing
+        if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
+            check, error_message = self._convert_kepubify(file_path,
+                                                          format_old_ext,
+                                                          format_new_ext,
+                                                          index)
+        else:
+            # check if calibre converter-executable is existing
             if not os.path.exists(config.config_converterpath):
                 # ToDo Text is not translated
-            self._handleError(u"Convertertool %s not found" % config.config_converterpath)
+                self._handleError(_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
+                return
+            check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, index)
+
+        if check == 0:
+            cur_book = calibre_db.get_book(book_id)
+            if os.path.isfile(file_path + format_new_ext):
+                # self.db_queue.join()
+                new_format = db.Data(name=cur_book.data[0].name,
+                                         book_format=self.queue[index]['settings']['new_book_format'].upper(),
+                                         book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
+                task = {'task':'add_format','id': book_id, 'format': new_format}
+                self.db_queue.put(task)
+                # To Do how to handle error?
+
+                '''cur_book.data.append(new_format)
+                try:
+                    # db.session.merge(cur_book)
+                    calibre_db.session.commit()
+                except OperationalError as e:
+                    calibre_db.session.rollback()
+                    log.error("Database error: %s", e)
+                    self._handleError(_(u"Database error: %(error)s.", error=e))
+                    return'''
+
+                self.queue[index]['path'] = cur_book.path
+                self.queue[index]['title'] = cur_book.title
+                if config.config_use_google_drive:
+                    os.remove(file_path + format_old_ext)
+                self._handleSuccess()
+                return file_path + format_new_ext
+            else:
+                error_message = format_new_ext.upper() + ' format not found on disk'
+        log.info("ebook converter failed with error while converting book")
+        if not error_message:
+            error_message = 'Ebook converter failed with unknown error'
+        self._handleError(error_message)
         return
 
+
+    def _convert_calibre(self, file_path, format_old_ext, format_new_ext, index):
         try:
-            # check which converter to use kindlegen is "1"
-            if format_old_ext == '.epub' and format_new_ext == '.mobi':
-                if config.config_ebookconverter == 1:
-                    command = [config.config_converterpath, file_path + u'.epub']
-                    quotes = [1]
-            if config.config_ebookconverter == 2:
             # Linux py2.7 encode as list without quotes no empty element for parameters
             # linux py3.x no encode and as list without quotes no empty element for parameters
             # windows py2.7 encode as string with quotes empty element for parameters is okay
             # windows py 3.x no encode and as string with quotes empty element for parameters is okay
             # separate handling for windows and linux
-                quotes = [1,2]
+            quotes = [1, 2]
             command = [config.config_converterpath, (file_path + format_old_ext),
                        (file_path + format_new_ext)]
             quotes_index = 3
@@ -318,23 +367,11 @@ class WorkerThread(threading.Thread):
                     command.append(param)
                     quotes.append(quotes_index)
                     quotes_index += 1
-            p = process_open(command, quotes)
-            # p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
-        except OSError as e:
-            self._handleError(_(u"Ebook-converter failed: %(error)s", error=e))
-            return
 
-        if config.config_ebookconverter == 1:
-            nextline = p.communicate()[0]
-            # Format of error message (kindlegen translates its output texts):
-            # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
-            conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
-            # If error occoures, store error message for logfile
-            if conv_error:
-                error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
-                                  error=conv_error.group(1), message=conv_error.group(2).strip())
-            log.debug("convert_kindlegen: %s", nextline)
-        else:
+            p = process_open(command, quotes)
+        except OSError as e:
+            return 1, _(u"Ebook-converter failed: %(error)s", error=e)
+
         while p.poll() is None:
             nextline = p.stdout.readline()
             if os.name == 'nt' and sys.version_info < (3, 0):
@@ -350,39 +387,48 @@ class WorkerThread(threading.Thread):
         # process returncode
         check = p.returncode
         calibre_traceback = p.stderr.readlines()
+        error_message = ""
         for ele in calibre_traceback:
             if sys.version_info < (3, 0):
                 ele = ele.decode('utf-8')
             log.debug(ele.strip('\n'))
             if not ele.startswith('Traceback') and not ele.startswith('  File'):
                 error_message = "Calibre failed with error: %s" % ele.strip('\n')
+        return check, error_message
 
-        # kindlegen returncodes
-        # 0 = Info(prcgen):I1036: Mobi file built successfully
-        # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
-        # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
-        if (check < 2 and config.config_ebookconverter == 1) or \
-            (check == 0 and config.config_ebookconverter == 2):
-            cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first()
-            if os.path.isfile(file_path + format_new_ext):
-                new_format = db.Data(name=cur_book.data[0].name,
-                                         book_format=self.queue[index]['settings']['new_book_format'].upper(),
-                                         book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
-                cur_book.data.append(new_format)
-                db.session.commit()
-                self.queue[index]['path'] = cur_book.path
-                self.queue[index]['title'] = cur_book.title
-                if config.config_use_google_drive:
-                    os.remove(file_path + format_old_ext)
-                self._handleSuccess()
-                return file_path + format_new_ext
+
+    def _convert_kepubify(self, file_path, format_old_ext, format_new_ext, index):
+        quotes = [1, 3]
+        command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)]
+        try:
+            p = process_open(command, quotes)
+        except OSError as e:
+            return 1, _(u"Kepubify-converter failed: %(error)s", error=e)
+        self.UIqueue[index]['progress'] = '1 %'
+        while True:
+            nextline = p.stdout.readlines()
+            nextline = [x.strip('\n') for x in nextline if x != '\n']
+            if sys.version_info < (3, 0):
+                nextline = [x.decode('utf-8') for x in nextline]
+            for line in nextline:
+                log.debug(line)
+            if p.poll() is not None:
+                break
+
+        # ToD Handle
+        # process returncode
+        check = p.returncode
+
+        # move file
+        if check == 0:
+            converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub"))
+            if len(converted_file) == 1:
+                copyfile(converted_file[0], (file_path + format_new_ext))
+                os.unlink(converted_file[0])
             else:
-                error_message = format_new_ext.upper() + ' format not found on disk'
-        log.info("ebook converter failed with error while converting book")
-        if not error_message:
-            error_message = 'Ebook converter failed with unknown error'
-        self._handleError(error_message)
-        return
+                return 1, _(u"Converted file not found or more than one file in folder %(folder)s",
+                            folder=os.path.dirname(file_path))
+        return check, None
 
 
     def add_convert(self, file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
@@ -533,10 +579,6 @@ class WorkerThread(threading.Thread):
         self.UIqueue[index]['formRuntime'] = datetime.now() - self.queue[index]['starttime']
 
 
-_worker = WorkerThread()
-_worker.start()
-
-
 def get_taskstatus():
     return _worker.get_taskstatus()
 
@@ -551,3 +593,7 @@ def add_upload(user_name, taskMessage):
 
 def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
     return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail)
+
+
+_worker = WorkerThread()
+_worker.start()

optional-requirements.txt

@@ -31,7 +31,7 @@ rarfile>=2.7
 
 # other
 natsort>=2.2.0,<7.1.0
-git+https://github.com/OzzieIsaacs/comicapi.git@15dff9ce4e1ffed29ba4a2feadfcdb6bed00bcad#egg=comicapi
+git+https://github.com/OzzieIsaacs/comicapi.git@3e15b950b72724b1b8ca619c36580b5fbaba9784#egg=comicapi
 
 #Kobo integration
 jsonschema>=3.2.0,<3.3.0