merge remote

cps/__init__.py

@@ -127,7 +127,7 @@ def get_locale():
     user = getattr(g, 'user', None)
     # user = None
     if user is not None and hasattr(user, "locale"):
-        if user.nickname != 'Guest':   # if the account is the guest account bypass the config lang settings
+        if user.name != 'Guest':   # if the account is the guest account bypass the config lang settings
             return user.locale
 
     preferred = list()

cps/admin.py

@@ -35,13 +35,15 @@ from flask import Blueprint, flash, redirect, url_for, abort, request, make_resp
 from flask_login import login_required, current_user, logout_user, confirm_login
 from flask_babel import gettext as _
 from sqlalchemy import and_
+from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError
 from sqlalchemy.sql.expression import func, or_
 
 from . import constants, logger, helper, services
 from .cli import filepicker
 from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
-from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
+from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
+    valid_email, check_username
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .render_template import render_title_template, get_sidebar_config
 from . import debug_info
@@ -57,12 +59,12 @@ feature_support = {
         'ldap': bool(services.ldap),
         'goodreads': bool(services.goodreads_support),
         'kobo':  bool(services.kobo),
-        'updater': constants.UPDATER_AVAILABLE
+        'updater': constants.UPDATER_AVAILABLE,
+        'gmail': bool(services.gmail)
     }
 
 try:
-    # pylint: disable=unused-import
+    import rarfile  # pylint: disable=unused-import
-    import rarfile
     feature_support['rar'] = True
 except (ImportError, SyntaxError):
     feature_support['rar'] = False
@@ -150,7 +152,7 @@ def shutdown():
         else:
             showtext['text'] = _(u'Performing shutdown of server, please close window')
         # stop gevent/tornado server
-        web_server.stop(task == 0)
+        web_server.stop(task==0)
         return json.dumps(showtext)
 
     if task == 2:
@@ -185,10 +187,10 @@ def admin():
         else:
             commit = version['version']
 
-    all_user = ub.session.query(ub.User).all()
+    allUser = ub.session.query(ub.User).all()
     email_settings = config.get_mail_settings()
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
-    return render_title_template("admin.html", allUser=all_user, email=email_settings, config=config, commit=commit,
+    return render_title_template("admin.html", allUser=allUser, email=email_settings, config=config, commit=commit,
                                  feature_support=feature_support, kobo_support=kobo_support,
                                  title=_(u"Admin page"), page="admin")
 
@@ -214,6 +216,173 @@ def view_configuration():
                                  restrictColumns=restrict_columns,
                                  title=_(u"UI Configuration"), page="uiconfig")
 
+@admi.route("/admin/usertable")
+@login_required
+@admin_required
+def edit_user_table():
+    visibility = current_user.view_settings.get('useredit', {})
+    languages = calibre_db.speaking_language()
+    translations = babel.list_translations() + [LC('en')]
+    allUser = ub.session.query(ub.User)
+    if not config.config_anonbrowse:
+        allUser = allUser.filter(ub.User.role.op('&')(constants.ROLE_ANONYMOUS) != constants.ROLE_ANONYMOUS)
+
+    return render_title_template("user_table.html",
+                                 users=allUser.all(),
+                                 translations=translations,
+                                 languages=languages,
+                                 visiblility=visibility,
+                                 all_roles=constants.ALL_ROLES,
+                                 sidebar_settings=constants.sidebar_settings,
+                                 title=_(u"Edit Users"),
+                                 page="usertable")
+
+@admi.route("/ajax/listusers")
+@login_required
+@admin_required
+def list_users():
+    off = request.args.get("offset") or 0
+    limit = request.args.get("limit") or 10
+    search = request.args.get("search")
+    all_user = ub.session.query(ub.User)
+    if not config.config_anonbrowse:
+        all_user = all_user.filter(ub.User.role.op('&')(constants.ROLE_ANONYMOUS) != constants.ROLE_ANONYMOUS)
+    total_count = all_user.count()
+    if search:
+        users = all_user.filter(or_(func.lower(ub.User.name).ilike("%" + search + "%"),
+                                    func.lower(ub.User.kindle_mail).ilike("%" + search + "%"),
+                                    func.lower(ub.User.email).ilike("%" + search + "%")))\
+            .offset(off).limit(limit).all()
+        filtered_count = len(users)
+    else:
+        users = all_user.offset(off).limit(limit).all()
+        filtered_count = total_count
+
+    for user in users:
+        if user.default_language == "all":
+            user.default = _("all")
+        else:
+            user.default = LC.parse(user.default_language).get_language_name(get_locale())
+
+    table_entries = {'totalNotFiltered': total_count, 'total': filtered_count, "rows": users}
+    js_list = json.dumps(table_entries, cls=db.AlchemyEncoder)
+    response = make_response(js_list)
+    response.headers["Content-Type"] = "application/json; charset=utf-8"
+    return response
+
+@admi.route("/ajax/deleteuser")
+@login_required
+@admin_required
+def delete_user():
+    # ToDo User delete check also not last one
+    return ""
+
+@admi.route("/ajax/getlocale")
+@login_required
+@admin_required
+def table_get_locale():
+    locale = babel.list_translations() + [LC('en')]
+    ret = list()
+    current_locale = get_locale()
+    for loc in locale:
+        ret.append({'value': str(loc), 'text': loc.get_language_name(current_locale)})
+    return json.dumps(ret)
+
+
+@admi.route("/ajax/getdefaultlanguage")
+@login_required
+@admin_required
+def table_get_default_lang():
+    languages = calibre_db.speaking_language()
+    ret = list()
+    ret.append({'value':'all','text':_('Show All')})
+    for lang in languages:
+        ret.append({'value': lang.lang_code, 'text': lang.name})
+    return json.dumps(ret)
+
+
+@admi.route("/ajax/editlistusers/<param>", methods=['POST'])
+@login_required
+@admin_required
+def edit_list_user(param):
+    vals = request.form.to_dict(flat=False)
+    all_user = ub.session.query(ub.User)
+    if not config.config_anonbrowse:
+        all_user = all_user.filter(ub.User.role.op('&')(constants.ROLE_ANONYMOUS) != constants.ROLE_ANONYMOUS)
+    # only one user is posted
+    if "pk" in vals:
+        users = [all_user.filter(ub.User.id == vals['pk'][0]).one_or_none()]
+    else:
+        if "pk[]" in vals:
+            users = all_user.filter(ub.User.id.in_(vals['pk[]'])).all()
+        else:
+            return ""
+    if 'field_index' in vals:
+        vals['field_index'] = vals['field_index'][0]
+    if 'value' in vals:
+        vals['value'] = vals['value'][0]
+    else:
+        return ""
+    for user in users:
+        try:
+            vals['value'] = vals['value'].strip()
+            if param == 'name':
+                if user.name == "Guest":
+                    raise Exception(_("Guest Name can't be changed"))
+                user.name = check_username(vals['value'])
+            elif param =='email':
+                user.email = check_email(vals['value'])
+            elif param == 'kindle_mail':
+                user.kindle_mail = valid_email(vals['value']) if vals['value'] else ""
+            elif param == 'role':
+                if vals['value'] == 'true':
+                    user.role |= int(vals['field_index'])
+                else:
+                    if int(vals['field_index']) == constants.ROLE_ADMIN:
+                        if not ub.session.query(ub.User).\
+                               filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
+                                      ub.User.id != user.id).count():
+                            return _(u"No admin user remaining, can't remove admin role", nick=user.name), 400
+                    user.role &= ~int(vals['field_index'])
+            elif param == 'sidebar_view':
+                if vals['value'] == 'true':
+                    user.sidebar_view |= int(vals['field_index'])
+                else:
+                    user.sidebar_view &= ~int(vals['field_index'])
+            elif param == 'denied_tags':
+                    user.denied_tags = vals['value']
+            elif param == 'allowed_tags':
+                    user.allowed_tags = vals['value']
+            elif param == 'allowed_column_value':
+                user.allowed_column_value = vals['value']
+            elif param == 'denied_column_value':
+                user.denied_column_value = vals['value']
+            elif param == 'locale':
+                user.locale = vals['value']
+            elif param == 'default_language':
+                user.default_language = vals['value']
+        except Exception as ex:
+            return str(ex), 400
+    ub.session_commit()
+    return ""
+
+
+@admi.route("/ajax/user_table_settings", methods=['POST'])
+@login_required
+@admin_required
+def update_table_settings():
+    current_user.view_settings['useredit'] = json.loads(request.data)
+    try:
+        try:
+            flag_modified(current_user, "view_settings")
+        except AttributeError:
+            pass
+        ub.session.commit()
+    except (InvalidRequestError, OperationalError):
+        log.error("Invalid request received: {}".format(request))
+        return "Invalid request", 400
+    return ""
+
 
 @admi.route("/admin/viewconfig", methods=["POST"])
 @login_required
@@ -262,6 +431,14 @@ def load_dialogtexts(element_id):
         texts["main"] = _('Do you really want to delete this user?')
     elif element_id == "delete_shelf":
         texts["main"] = _('Are you sure you want to delete this shelf?')
+    elif element_id == "select_locale":
+        texts["main"] = _('Are you sure you want to change locales of selected user(s)?')
+    elif element_id == "select_default_language":
+        texts["main"] = _('Are you sure you want to change visible book languages for selected user(s)?')
+    elif element_id == "role":
+        texts["main"] = _('Are you sure you want to change the selected role for the selected user(s)?')
+    elif element_id == "sidebar_view":
+        texts["main"] = _('Are you sure you want to change the selected visibility restrictions for the selected user(s)?')
     return json.dumps(texts)
 
 
@@ -348,7 +525,7 @@ def edit_restriction(res_type, user_id):
             elementlist = usr.list_allowed_tags()
             elementlist[int(element['id'][1:])] = element['Element']
             usr.allowed_tags = ','.join(elementlist)
-            ub.session_commit("Changed allowed tags of user {} to {}".format(usr.nickname, usr.allowed_tags))
+            ub.session_commit("Changed allowed tags of user {} to {}".format(usr.name, usr.allowed_tags))
         if res_type == 3:  # CColumn per user
             if isinstance(user_id, int):
                 usr = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()
@@ -357,7 +534,7 @@ def edit_restriction(res_type, user_id):
             elementlist = usr.list_allowed_column_values()
             elementlist[int(element['id'][1:])] = element['Element']
             usr.allowed_column_value = ','.join(elementlist)
-            ub.session_commit("Changed allowed columns of user {} to {}".format(usr.nickname, usr.allowed_column_value))
+            ub.session_commit("Changed allowed columns of user {} to {}".format(usr.name, usr.allowed_column_value))
     if element['id'].startswith('d'):
         if res_type == 0:  # Tags as template
             elementlist = config.list_denied_tags()
@@ -377,7 +554,7 @@ def edit_restriction(res_type, user_id):
             elementlist = usr.list_denied_tags()
             elementlist[int(element['id'][1:])] = element['Element']
             usr.denied_tags = ','.join(elementlist)
-            ub.session_commit("Changed denied tags of user {} to {}".format(usr.nickname, usr.denied_tags))
+            ub.session_commit("Changed denied tags of user {} to {}".format(usr.name, usr.denied_tags))
         if res_type == 3:  # CColumn per user
             if isinstance(user_id, int):
                 usr = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()
@@ -386,7 +563,7 @@ def edit_restriction(res_type, user_id):
             elementlist = usr.list_denied_column_values()
             elementlist[int(element['id'][1:])] = element['Element']
             usr.denied_column_value = ','.join(elementlist)
-            ub.session_commit("Changed denied columns of user {} to {}".format(usr.nickname, usr.denied_column_value))
+            ub.session_commit("Changed denied columns of user {} to {}".format(usr.name, usr.denied_column_value))
     return ""
 
 
@@ -433,10 +610,10 @@ def add_restriction(res_type, user_id):
             usr = current_user
         if 'submit_allow' in element:
             usr.allowed_tags = restriction_addition(element, usr.list_allowed_tags)
-            ub.session_commit("Changed allowed tags of user {} to {}".format(usr.nickname, usr.list_allowed_tags))
+            ub.session_commit("Changed allowed tags of user {} to {}".format(usr.name, usr.list_allowed_tags))
         elif 'submit_deny' in element:
             usr.denied_tags = restriction_addition(element, usr.list_denied_tags)
-            ub.session_commit("Changed denied tags of user {} to {}".format(usr.nickname, usr.list_denied_tags))
+            ub.session_commit("Changed denied tags of user {} to {}".format(usr.name, usr.list_denied_tags))
     if res_type == 3:  # CustomC per user
         if isinstance(user_id, int):
             usr = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()
@@ -444,11 +621,11 @@ def add_restriction(res_type, user_id):
             usr = current_user
         if 'submit_allow' in element:
             usr.allowed_column_value = restriction_addition(element, usr.list_allowed_column_values)
-            ub.session_commit("Changed allowed columns of user {} to {}".format(usr.nickname,
+            ub.session_commit("Changed allowed columns of user {} to {}".format(usr.name,
                                                                                 usr.list_allowed_column_values))
         elif 'submit_deny' in element:
             usr.denied_column_value = restriction_addition(element, usr.list_denied_column_values)
-            ub.session_commit("Changed denied columns of user {} to {}".format(usr.nickname,
+            ub.session_commit("Changed denied columns of user {} to {}".format(usr.name,
                                                                                usr.list_denied_column_values))
     return ""
 
@@ -480,10 +657,10 @@ def delete_restriction(res_type, user_id):
             usr = current_user
         if element['id'].startswith('a'):
             usr.allowed_tags = restriction_deletion(element, usr.list_allowed_tags)
-            ub.session_commit("Deleted allowed tags of user {}: {}".format(usr.nickname, usr.list_allowed_tags))
+            ub.session_commit("Deleted allowed tags of user {}: {}".format(usr.name, usr.list_allowed_tags))
         elif element['id'].startswith('d'):
             usr.denied_tags = restriction_deletion(element, usr.list_denied_tags)
-            ub.session_commit("Deleted denied tags of user {}: {}".format(usr.nickname, usr.list_allowed_tags))
+            ub.session_commit("Deleted denied tags of user {}: {}".format(usr.name, usr.list_allowed_tags))
     elif res_type == 3:  # Columns per user
         if isinstance(user_id, int):
             usr = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()
@@ -491,12 +668,12 @@ def delete_restriction(res_type, user_id):
             usr = current_user
         if element['id'].startswith('a'):
             usr.allowed_column_value = restriction_deletion(element, usr.list_allowed_column_values)
-            ub.session_commit("Deleted allowed columns of user {}: {}".format(usr.nickname,
+            ub.session_commit("Deleted allowed columns of user {}: {}".format(usr.name,
                                                                               usr.list_allowed_column_values))
 
         elif element['id'].startswith('d'):
             usr.denied_column_value = restriction_deletion(element, usr.list_denied_column_values)
-            ub.session_commit("Deleted denied columns of user {}: {}".format(usr.nickname,
+            ub.session_commit("Deleted denied columns of user {}: {}".format(usr.name,
                                                                              usr.list_denied_column_values))
     return ""
 
@@ -602,7 +779,6 @@ def pathchooser():
         folders = []
 
     files = []
-    # locale = get_locale()
     for f in folders:
         try:
             data = {"name": f, "fullpath": os.path.join(cwd, f)}
@@ -730,13 +906,35 @@ def _configuration_logfile_helper(to_save, gdrive_error):
     return reboot_required, None
 
 
-def _configuration_ldap_check(reboot_required, to_save, gdrive_error):
+def _configuration_ldap_helper(to_save, gdrive_error):
+    reboot_required = False
+    reboot_required |= _config_string(to_save, "config_ldap_provider_url")
+    reboot_required |= _config_int(to_save, "config_ldap_port")
+    reboot_required |= _config_int(to_save, "config_ldap_authentication")
+    reboot_required |= _config_string(to_save, "config_ldap_dn")
+    reboot_required |= _config_string(to_save, "config_ldap_serv_username")
+    reboot_required |= _config_string(to_save, "config_ldap_user_object")
+    reboot_required |= _config_string(to_save, "config_ldap_group_object_filter")
+    reboot_required |= _config_string(to_save, "config_ldap_group_members_field")
+    reboot_required |= _config_string(to_save, "config_ldap_member_user_object")
+    reboot_required |= _config_checkbox(to_save, "config_ldap_openldap")
+    reboot_required |= _config_int(to_save, "config_ldap_encryption")
+    reboot_required |= _config_string(to_save, "config_ldap_cacert_path")
+    reboot_required |= _config_string(to_save, "config_ldap_cert_path")
+    reboot_required |= _config_string(to_save, "config_ldap_key_path")
+    _config_string(to_save, "config_ldap_group_name")
+    if to_save.get("config_ldap_serv_password", "") != "":
+        reboot_required |= 1
+        config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8')
+    config.save()
+
     if not config.config_ldap_provider_url \
-       or not config.config_ldap_port \
+        or not config.config_ldap_port \
-       or not config.config_ldap_dn \
+        or not config.config_ldap_dn \
-       or not config.config_ldap_user_object:
+        or not config.config_ldap_user_object:
         return reboot_required, _configuration_result(_('Please Enter a LDAP Provider, '
                                                         'Port, DN and User Object Identifier'), gdrive_error)
+
     if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS:
         if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE:
             if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password):
@@ -746,14 +944,6 @@ def _configuration_ldap_check(reboot_required, to_save, gdrive_error):
             if not config.config_ldap_serv_username:
                 return reboot_required, _configuration_result('Please Enter a LDAP Service Account', gdrive_error)
 
-    if config.config_ldap_group_object_filter:
-        if config.config_ldap_group_object_filter.count("%s") != 1:
-            return reboot_required, \
-                   _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'),
-                                         gdrive_error)
-        if config.config_ldap_group_object_filter.count("(") != config.config_ldap_group_object_filter.count(")"):
-            return reboot_required, _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'),
-                                                          gdrive_error)
     if config.config_ldap_group_object_filter:
         if config.config_ldap_group_object_filter.count("%s") != 1:
             return reboot_required, \
@@ -771,7 +961,7 @@ def _configuration_ldap_check(reboot_required, to_save, gdrive_error):
         return reboot_required, _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'),
                                                       gdrive_error)
 
-    if "ldap_import_user_filter" in to_save and to_save["ldap_import_user_filter"] == '0':
+    if to_save.get("ldap_import_user_filter") == '0':
         config.config_ldap_member_user_object = ""
     else:
         if config.config_ldap_member_user_object.count("%s") != 1:
@@ -793,31 +983,6 @@ def _configuration_ldap_check(reboot_required, to_save, gdrive_error):
     return reboot_required, None
 
 
-def _configuration_ldap_helper(to_save, gdrive_error):
-    reboot_required = False
-    reboot_required |= _config_string(to_save, "config_ldap_provider_url")
-    reboot_required |= _config_int(to_save, "config_ldap_port")
-    reboot_required |= _config_int(to_save, "config_ldap_authentication")
-    reboot_required |= _config_string(to_save, "config_ldap_dn")
-    reboot_required |= _config_string(to_save, "config_ldap_serv_username")
-    reboot_required |= _config_string(to_save, "config_ldap_user_object")
-    reboot_required |= _config_string(to_save, "config_ldap_group_object_filter")
-    reboot_required |= _config_string(to_save, "config_ldap_group_members_field")
-    reboot_required |= _config_string(to_save, "config_ldap_member_user_object")
-    reboot_required |= _config_checkbox(to_save, "config_ldap_openldap")
-    reboot_required |= _config_int(to_save, "config_ldap_encryption")
-    reboot_required |= _config_string(to_save, "config_ldap_cacert_path")
-    reboot_required |= _config_string(to_save, "config_ldap_cert_path")
-    reboot_required |= _config_string(to_save, "config_ldap_key_path")
-    _config_string(to_save, "config_ldap_group_name")
-    if "config_ldap_serv_password" in to_save and to_save["config_ldap_serv_password"] != "":
-        reboot_required |= 1
-        config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8')
-    config.save()
-
-    return _configuration_ldap_check(reboot_required, to_save, gdrive_error)
-
-
 def _configuration_update_helper(configured):
     reboot_required = False
     db_change = False
@@ -921,8 +1086,8 @@ def _configuration_update_helper(configured):
         if config.config_use_google_drive and is_gdrive_ready() and not os.path.exists(metadata_db):
             gdriveutils.downloadFile(None, "metadata.db", metadata_db)
             db_change = True
-    except Exception as e:
+    except Exception as ex:
-        return _configuration_result('%s' % e, gdrive_error, configured)
+        return _configuration_result('%s' % ex, gdrive_error, configured)
 
     if db_change:
         if not calibre_db.setup_db(config, ub.app_DB_path):
@@ -974,7 +1139,6 @@ def _configuration_result(error_flash=None, gdrive_error=None, configured=True):
 
 def _handle_new_user(to_save, content, languages, translations, kobo_support):
     content.default_language = to_save["default_language"]
-    # content.mature_content = "Show_mature_content" in to_save
     content.locale = to_save.get("locale", content.locale)
 
     content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_'))
@@ -982,28 +1146,21 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support):
         content.sidebar_view |= constants.DETAIL_RANDOM
 
     content.role = constants.selected_roles(to_save)
-
-    if not to_save["nickname"] or not to_save["email"] or not to_save["password"]:
-        flash(_(u"Please fill out all fields!"), category="error")
-        return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
-                                     registered_oauth=oauth_check, kobo_support=kobo_support,
-                                     title=_(u"Add new user"))
     content.password = generate_password_hash(to_save["password"])
-    existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower()) \
+    try:
-        .first()
+        if not to_save["name"] or not to_save["email"] or not to_save["password"]:
-    existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
+            log.info("Missing entries on new user")
-        .first()
+            raise Exception(_(u"Please fill out all fields!"))
-    if not existing_user and not existing_email:
+        content.email = check_email(to_save["email"])
-        content.nickname = to_save["nickname"]
+        # Query User name, if not existing, change
-        if config.config_public_reg and not check_valid_domain(to_save["email"]):
+        content.name = check_username(to_save["name"])
-            flash(_(u"E-mail is not from valid domain"), category="error")
+        if to_save.get("kindle_mail"):
-            return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
+            content.kindle_mail = valid_email(to_save["kindle_mail"])
-                                         registered_oauth=oauth_check, kobo_support=kobo_support,
+        if config.config_public_reg and not check_valid_domain(content.email):
-                                         title=_(u"Add new user"))
+            log.info("E-mail: {} for new user is not from valid domain".format(content.email))
-        else:
+            raise Exception(_(u"E-mail is not from valid domain"))
-            content.email = to_save["email"]
+    except Exception as ex:
-    else:
+        flash(str(ex), category="error")
-        flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
         return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
                                      languages=languages, title=_(u"Add new user"), page="newuser",
                                      kobo_support=kobo_support, registered_oauth=oauth_check)
@@ -1014,49 +1171,33 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support):
         content.denied_column_value = config.config_denied_column_value
         ub.session.add(content)
         ub.session.commit()
-        flash(_(u"User '%(user)s' created", user=content.nickname), category="success")
+        flash(_(u"User '%(user)s' created", user=content.name), category="success")
         return redirect(url_for('admin.admin'))
     except IntegrityError:
         ub.session.rollback()
-        flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
+        flash(_(u"Found an existing account for this e-mail address or name."), category="error")
-    except OperationalError:
-        ub.session.rollback()
-        flash(_(u"Settings DB is not Writeable"), category="error")
-
-def delete_user(content):
-    if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
-                                        ub.User.id != content.id).count():
-        ub.session.query(ub.User).filter(ub.User.id == content.id).delete()
-        ub.session_commit()
-        flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success")
-        return redirect(url_for('admin.admin'))
-    else:
-        flash(_(u"No admin user remaining, can't delete user", nick=content.nickname), category="error")
-        return redirect(url_for('admin.admin'))
-
-
-def save_edited_user(content):
-    try:
-        ub.session_commit()
-        flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success")
-    except IntegrityError:
-        ub.session.rollback()
-        flash(_(u"An unknown error occured."), category="error")
     except OperationalError:
         ub.session.rollback()
         flash(_(u"Settings DB is not Writeable"), category="error")
 
 
 def _handle_edit_user(to_save, content, languages, translations, kobo_support):
-    if "delete" in to_save:
+    if to_save.get("delete"):
-        return delete_user(content)
+        if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
+                                            ub.User.id != content.id).count():
+            ub.session.query(ub.User).filter(ub.User.id == content.id).delete()
+            ub.session_commit()
+            flash(_(u"User '%(nick)s' deleted", nick=content.name), category="success")
+            return redirect(url_for('admin.admin'))
+        else:
+            flash(_(u"No admin user remaining, can't delete user", nick=content.name), category="error")
+            return redirect(url_for('admin.admin'))
     else:
         if not ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
                                                 ub.User.id != content.id).count() and 'admin_role' not in to_save:
-            flash(_(u"No admin user remaining, can't remove admin role", nick=content.nickname), category="error")
+            flash(_(u"No admin user remaining, can't remove admin role", nick=content.name), category="error")
             return redirect(url_for('admin.admin'))
-
+        if to_save.get("password"):
-        if "password" in to_save and to_save["password"]:
             content.password = generate_password_hash(to_save["password"])
         anonymous = content.is_anonymous
         content.role = constants.selected_roles(to_save)
@@ -1074,50 +1215,46 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
             elif value not in val and content.check_visibility(value):
                 content.sidebar_view &= ~value
 
-        if "Show_detail_random" in to_save:
+        if to_save.get("Show_detail_random"):
             content.sidebar_view |= constants.DETAIL_RANDOM
         else:
             content.sidebar_view &= ~constants.DETAIL_RANDOM
 
-        if "default_language" in to_save:
+        if to_save.get("default_language"):
             content.default_language = to_save["default_language"]
-        if "locale" in to_save and to_save["locale"]:
+        if to_save.get("locale"):
             content.locale = to_save["locale"]
-        if to_save["email"] and to_save["email"] != content.email:
+        try:
-            existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
+            if to_save.get("email", content.email) != content.email:
-                .first()
+                content.email = check_email(to_save["email"])
-            if not existing_email:
+            # Query User name, if not existing, change
-                content.email = to_save["email"]
+            if to_save.get("name", content.name) != content.name:
-            else:
+                if to_save.get("name") == "Guest":
-                flash(_(u"Found an existing account for this e-mail address."), category="error")
+                    raise Exception(_("Guest Name can't be changed"))
-                return render_title_template("user_edit.html",
+                content.name = check_username(to_save["name"])
-                                             translations=translations,
+            if to_save.get("kindle_mail") != content.kindle_mail:
-                                             languages=languages,
+                content.kindle_mail = valid_email(to_save["kindle_mail"]) if to_save["kindle_mail"] else ""
-                                             mail_configured=config.get_mail_server_configured(),
+        except Exception as ex:
-                                             kobo_support=kobo_support,
+            flash(str(ex), category="error")
-                                             new_user=0,
+            return render_title_template("user_edit.html",
-                                             content=content,
+                                         translations=translations,
-                                             registered_oauth=oauth_check,
+                                         languages=languages,
-                                             title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser")
+                                         mail_configured=config.get_mail_server_configured(),
-        if "nickname" in to_save and to_save["nickname"] != content.nickname:
+                                         kobo_support=kobo_support,
-            # Query User nickname, if not existing, change
+                                         new_user=0,
-            if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar():
+                                         content=content,
-                content.nickname = to_save["nickname"]
+                                         registered_oauth=oauth_check,
-            else:
+                                         title=_(u"Edit User %(nick)s", nick=content.name),
-                flash(_(u"This username is already taken"), category="error")
+                                         page="edituser")
-                return render_title_template("user_edit.html",
+    try:
-                                             translations=translations,
+        ub.session_commit()
-                                             languages=languages,
+        flash(_(u"User '%(nick)s' updated", nick=content.name), category="success")
-                                             mail_configured=config.get_mail_server_configured(),
+    except IntegrityError:
-                                             new_user=0, content=content,
+        ub.session.rollback()
-                                             registered_oauth=oauth_check,
+        flash(_(u"An unknown error occured."), category="error")
-                                             kobo_support=kobo_support,
+    except OperationalError:
-                                             title=_(u"Edit User %(nick)s", nick=content.nickname),
+        ub.session.rollback()
-                                             page="edituser")
+        flash(_(u"Settings DB is not Writeable"), category="error")
-
-        if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail:
-            content.kindle_mail = to_save["kindle_mail"]
-    return save_edited_user(content)
 
 
 @admi.route("/admin/user/new", methods=["GET", "POST"])
@@ -1145,7 +1282,7 @@ def new_user():
 def edit_mailsettings():
     content = config.get_mail_settings()
     return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"),
-                                 page="mailset")
+                                 page="mailset", feature_support=feature_support)
 
 
 @admi.route("/admin/mailsettings", methods=["POST"])
@@ -1153,14 +1290,30 @@ def edit_mailsettings():
 @admin_required
 def update_mailsettings():
     to_save = request.form.to_dict()
+    _config_int(to_save, "mail_server_type")
+    if to_save.get("invalidate"):
+        config.mail_gmail_token = {}
+        try:
+            flag_modified(config, "mail_gmail_token")
+        except AttributeError:
+            pass
+    elif to_save.get("gmail"):
+        try:
+            config.mail_gmail_token = services.gmail.setup_gmail(config.mail_gmail_token)
+            flash(_(u"G-Mail Account Verification Successful"), category="success")
+        except Exception as ex:
+            flash(str(ex), category="error")
+            log.error(ex)
+            return edit_mailsettings()
 
-    _config_string(to_save, "mail_server")
+    else:
-    _config_int(to_save, "mail_port")
+        _config_string(to_save, "mail_server")
-    _config_int(to_save, "mail_use_ssl")
+        _config_int(to_save, "mail_port")
-    _config_string(to_save, "mail_login")
+        _config_int(to_save, "mail_use_ssl")
-    _config_string(to_save, "mail_password")
+        _config_string(to_save, "mail_login")
-    _config_string(to_save, "mail_from")
+        _config_string(to_save, "mail_password")
-    _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024)
+        _config_string(to_save, "mail_from")
+        _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024)
     try:
         config.save()
     except (OperationalError, InvalidRequestError):
@@ -1170,10 +1323,10 @@ def update_mailsettings():
 
     if to_save.get("test"):
         if current_user.email:
-            result = send_test_mail(current_user.email, current_user.nickname)
+            result = send_test_mail(current_user.email, current_user.name)
             if result is None:
-                flash(_(u"Test e-mail queued for sending to %(email)s, please check Tasks for result", email=current_user.email),
+                flash(_(u"Test e-mail queued for sending to %(email)s, please check Tasks for result",
-                      category="info")
+                        email=current_user.email), category="info")
             else:
                 flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error")
         else:
@@ -1189,7 +1342,7 @@ def update_mailsettings():
 @admin_required
 def edit_user(user_id):
     content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()  # type: ub.User
-    if not content or (not config.config_anonbrowse and content.nickname == "Guest"):
+    if not content or (not config.config_anonbrowse and content.name == "Guest"):
         flash(_(u"User not found"), category="error")
         return redirect(url_for('admin.admin'))
     languages = calibre_db.speaking_language()
@@ -1206,7 +1359,8 @@ def edit_user(user_id):
                                  registered_oauth=oauth_check,
                                  mail_configured=config.get_mail_server_configured(),
                                  kobo_support=kobo_support,
-                                 title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser")
+                                 title=_(u"Edit User %(nick)s", nick=content.name),
+                                 page="edituser")
 
 
 @admi.route("/admin/resetpassword/<int:user_id>")
@@ -1328,18 +1482,15 @@ def get_updater_status():
     return ''
 
 
-def create_ldap_user(user, user_data, config):
+def ldap_import_create_user(user, user_data):
-    imported = 0
-    showtext = None
-
     user_login_field = extract_dynamic_field_from_filter(user, config.config_ldap_user_object)
 
     username = user_data[user_login_field][0].decode('utf-8')
     # check for duplicate username
-    if ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first():
+    if ub.session.query(ub.User).filter(func.lower(ub.User.name) == username.lower()).first():
-        # if ub.session.query(ub.User).filter(ub.User.nickname == username).first():
+        # if ub.session.query(ub.User).filter(ub.User.name == username).first():
         log.warning("LDAP User  %s Already in Database", user_data)
-        return imported, showtext
+        return 0, None
 
     kindlemail = ''
     if 'mail' in user_data:
@@ -1350,13 +1501,15 @@ def create_ldap_user(user, user_data, config):
     else:
         log.debug('No Mail Field Found in LDAP Response')
         useremail = username + '@email.com'
-    # check for duplicate email
-    if ub.session.query(ub.User).filter(func.lower(ub.User.email) == useremail.lower()).first():
-        log.warning("LDAP Email %s Already in Database", user_data)
-        return imported, showtext
 
+    try:
+        # check for duplicate email
+        useremail = check_email(useremail)
+    except Exception as ex:
+        log.warning("LDAP Email Error: {}, {}".format(user_data, ex))
+        return 0, None
     content = ub.User()
-    content.nickname = username
+    content.name = username
     content.password = ''  # dummy password which will be replaced by ldap one
     content.email = useremail
     content.kindle_mail = kindlemail
@@ -1369,12 +1522,12 @@ def create_ldap_user(user, user_data, config):
     ub.session.add(content)
     try:
         ub.session.commit()
-        imported = 1
+        return 1, None    # increase no of users
-    except Exception as e:
+    except Exception as ex:
-        log.warning("Failed to create LDAP user: %s - %s", user, e)
+        log.warning("Failed to create LDAP user: %s - %s", user, ex)
         ub.session.rollback()
-        showtext = _(u'Failed to Create at Least One LDAP User')
+        message = _(u'Failed to Create at Least One LDAP User')
-    return imported, showtext
+        return 0, message
 
 
 @admi.route('/import_ldap_users')
@@ -1404,23 +1557,23 @@ def import_ldap_users():
                 query_filter = config.config_ldap_user_object
             try:
                 user_identifier = extract_user_identifier(user, query_filter)
-            except Exception as e:
+            except Exception as ex:
-                log.warning(e)
+                log.warning(ex)
                 continue
         else:
             user_identifier = user
             query_filter = None
         try:
             user_data = services.ldap.get_object_details(user=user_identifier, query_filter=query_filter)
-        except AttributeError as e:
+        except AttributeError as ex:
-            log.debug_or_exception(e)
+            log.debug_or_exception(ex)
             continue
         if user_data:
-            success, txt = create_ldap_user(user, user_data, config)
+            user_count, message = ldap_import_create_user(user, user_data)
-            # In case of error store text for showing it
+            if message:
-            if txt:
+                showtext['text'] = message
-                showtext['text'] = txt
+            else:
-            imported += success
+                imported += user_count
         else:
             log.warning("LDAP User: %s Not Found", user)
             showtext['text'] = _(u'At Least One LDAP User Not Found in Database')

cps/comic.py

@@ -105,8 +105,8 @@ def _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecu
                     if extension in COVER_EXTENSIONS:
                         cover_data = cf.read(name)
                         break
-        except Exception as e:
+        except Exception as ex:
-            log.debug('Rarfile failed with error: %s', e)
+            log.debug('Rarfile failed with error: %s', ex)
     return cover_data
 
 

cps/config_sql.py

@@ -23,7 +23,11 @@ import sys
 
 from sqlalchemy import exc, Column, String, Integer, SmallInteger, Boolean, BLOB, JSON
 from sqlalchemy.exc import OperationalError
-from sqlalchemy.ext.declarative import declarative_base
+try:
+    # Compability with sqlalchemy 2.0
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 
 from . import constants, cli, logger, ub
 
@@ -35,7 +39,7 @@ class _Flask_Settings(_Base):
     __tablename__ = 'flask_settings'
 
     id = Column(Integer, primary_key=True)
-    flask_session_key = Column(BLOB, default="")
+    flask_session_key = Column(BLOB, default=b"")
 
     def __init__(self, key):
         self.flask_session_key = key
@@ -54,6 +58,8 @@ class _Settings(_Base):
     mail_password = Column(String, default='mypassword')
     mail_from = Column(String, default='automailer <mail@example.com>')
     mail_size = Column(Integer, default=25*1024*1024)
+    mail_server_type = Column(SmallInteger, default=0)
+    mail_gmail_token = Column(JSON, default={})
 
     config_calibre_dir = Column(String)
     config_port = Column(Integer, default=constants.DEFAULT_PORT)
@@ -242,15 +248,16 @@ class _ConfigSQL(object):
         return {k:v for k, v in self.__dict__.items() if k.startswith('mail_')}
 
     def get_mail_server_configured(self):
-        return not bool(self.mail_server == constants.DEFAULT_MAIL_SERVER)
+        return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0)
+                    or (self.mail_gmail_token != {} and self.mail_server_type == 1))
 
 
     def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None):
-        '''Possibly updates a field of this object.
+        """Possibly updates a field of this object.
         The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor.
 
         :returns: `True` if the field has changed value
-        '''
+        """
         new_value = dictionary.get(field, default)
         if new_value is None:
             # log.debug("_ConfigSQL set_from_dictionary field '%s' not found", field)
@@ -301,8 +308,11 @@ class _ConfigSQL(object):
                 have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
         constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')]
-        # pylint: disable=access-member-before-definition
+        if os.environ.get('FLASK_DEBUG'):
-        logfile = logger.setup(self.config_logfile, self.config_log_level)
+            logfile = logger.setup(logger.LOG_TO_STDOUT, logger.logging.DEBUG)
+        else:
+            # pylint: disable=access-member-before-definition
+            logfile = logger.setup(self.config_logfile, self.config_log_level)
         if logfile != self.config_logfile:
             log.warning("Log path %s not valid, falling back to default", self.config_logfile)
             self.config_logfile = logfile
@@ -360,10 +370,14 @@ def _migrate_table(session, orm_class):
                     if isinstance(column.default.arg, bool):
                         column_default = ("DEFAULT %r" % int(column.default.arg))
                     else:
-                        column_default = ("DEFAULT %r" % column.default.arg)
+                        column_default = ("DEFAULT '%r'" % column.default.arg)
+                if isinstance(column.type, JSON):
+                    column_type = "JSON"
+                else:
+                    column_type = column.type
                 alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__,
                                                                         column_name,
-                                                                        column.type,
+                                                                        column_type,
                                                                         column_default)
                 log.debug(alter_table)
                 session.execute(alter_table)
@@ -426,12 +440,12 @@ def load_configuration(session):
         session.commit()
     conf = _ConfigSQL(session)
     # Migrate from global restrictions to user based restrictions
-    if bool(conf.config_default_show & constants.MATURE_CONTENT) and conf.config_denied_tags == "":
+    #if bool(conf.config_default_show & constants.MATURE_CONTENT) and conf.config_denied_tags == "":
-        conf.config_denied_tags = conf.config_mature_content_tags
+    #    conf.config_denied_tags = conf.config_mature_content_tags
-        conf.save()
+    #    conf.save()
-        session.query(ub.User).filter(ub.User.mature_content != True). \
+    #    session.query(ub.User).filter(ub.User.mature_content != True). \
-            update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False)
+    #        update({"denied_tags": conf.config_mature_content_tags}, synchronize_session=False)
-        session.commit()
+    #    session.commit()
     return conf
 
 def get_flask_session_key(session):

cps/constants.py

@@ -21,8 +21,10 @@ import sys
 import os
 from collections import namedtuple
 
-# if installed via pip this variable is set to true
+# if installed via pip this variable is set to true (empty file with name .HOMEDIR present)
-HOME_CONFIG = False
+HOME_CONFIG = os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), '.HOMEDIR'))
+
+#In executables updater is not available, so variable is set to False there
 UPDATER_AVAILABLE = True
 
 # Base dir is parent of current file, necessary if called from different folder
@@ -86,6 +88,26 @@ SIDEBAR_ARCHIVED        = 1 << 15
 SIDEBAR_DOWNLOAD        = 1 << 16
 SIDEBAR_LIST            = 1 << 17
 
+sidebar_settings = {
+                "detail_random": DETAIL_RANDOM,
+                "sidebar_language": SIDEBAR_LANGUAGE,
+                "sidebar_series": SIDEBAR_SERIES,
+                "sidebar_category": SIDEBAR_CATEGORY,
+                "sidebar_random": SIDEBAR_RANDOM,
+                "sidebar_author": SIDEBAR_AUTHOR,
+                "sidebar_best_rated": SIDEBAR_BEST_RATED,
+                "sidebar_read_and_unread": SIDEBAR_READ_AND_UNREAD,
+                "sidebar_recent": SIDEBAR_RECENT,
+                "sidebar_sorted": SIDEBAR_SORTED,
+                "sidebar_publisher": SIDEBAR_PUBLISHER,
+                "sidebar_rating": SIDEBAR_RATING,
+                "sidebar_format": SIDEBAR_FORMAT,
+                "sidebar_archived": SIDEBAR_ARCHIVED,
+                "sidebar_download": SIDEBAR_DOWNLOAD,
+                "sidebar_list": SIDEBAR_LIST,
+            }
+
+
 ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_ANONYMOUS
 ADMIN_USER_SIDEBAR      = (SIDEBAR_LIST << 1) - 1
 

cps/db.py

@@ -30,7 +30,13 @@ from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
 from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.orm.collections import InstrumentedList
-from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
+from sqlalchemy.ext.declarative import DeclarativeMeta
+from sqlalchemy.exc import OperationalError
+try:
+    # Compability with sqlalchemy 2.0
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.sql.expression import and_, true, false, text, func, or_
 from sqlalchemy.ext.associationproxy import association_proxy
@@ -326,7 +332,6 @@ class Books(Base):
     has_cover = Column(Integer, default=0)
     uuid = Column(String)
     isbn = Column(String(collation='NOCASE'), default="")
-    # Iccn = Column(String(collation='NOCASE'), default="")
     flags = Column(Integer, nullable=False, default=1)
 
     authors = relationship('Authors', secondary=books_authors_link, backref='books')
@@ -437,12 +442,80 @@ class CalibreDB():
 
         self.instances.add(self)
 
-
     def initSession(self, expire_on_commit=True):
         self.session = self.session_factory()
         self.session.expire_on_commit = expire_on_commit
         self.update_title_sort(self.config)
 
+    @classmethod
+    def setup_db_cc_classes(self, cc):
+        cc_ids = []
+        books_custom_column_links = {}
+        for row in cc:
+            if row.datatype not in cc_exceptions:
+                if row.datatype == 'series':
+                    dicttable = {'__tablename__': 'books_custom_column_' + str(row.id) + '_link',
+                                 'id': Column(Integer, primary_key=True),
+                                 'book': Column(Integer, ForeignKey('books.id'),
+                                                primary_key=True),
+                                 'map_value': Column('value', Integer,
+                                                     ForeignKey('custom_column_' +
+                                                                str(row.id) + '.id'),
+                                                     primary_key=True),
+                                 'extra': Column(Float),
+                                 'asoc': relationship('custom_column_' + str(row.id), uselist=False),
+                                 'value': association_proxy('asoc', 'value')
+                                 }
+                    books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'),
+                                                             (Base,), dicttable)
+                else:
+                    books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link',
+                                                              Base.metadata,
+                                                              Column('book', Integer, ForeignKey('books.id'),
+                                                                     primary_key=True),
+                                                              Column('value', Integer,
+                                                                     ForeignKey('custom_column_' +
+                                                                                str(row.id) + '.id'),
+                                                                     primary_key=True)
+                                                              )
+                cc_ids.append([row.id, row.datatype])
+
+                ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                          'id': Column(Integer, primary_key=True)}
+                if row.datatype == 'float':
+                    ccdict['value'] = Column(Float)
+                elif row.datatype == 'int':
+                    ccdict['value'] = Column(Integer)
+                elif row.datatype == 'bool':
+                    ccdict['value'] = Column(Boolean)
+                else:
+                    ccdict['value'] = Column(String)
+                if row.datatype in ['float', 'int', 'bool']:
+                    ccdict['book'] = Column(Integer, ForeignKey('books.id'))
+                cc_classes[row.id] = type(str('custom_column_' + str(row.id)), (Base,), ccdict)
+
+        for cc_id in cc_ids:
+            if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'):
+                setattr(Books,
+                        'custom_column_' + str(cc_id[0]),
+                        relationship(cc_classes[cc_id[0]],
+                                     primaryjoin=(
+                                         Books.id == cc_classes[cc_id[0]].book),
+                                     backref='books'))
+            elif (cc_id[1] == 'series'):
+                setattr(Books,
+                        'custom_column_' + str(cc_id[0]),
+                        relationship(books_custom_column_links[cc_id[0]],
+                                     backref='books'))
+            else:
+                setattr(Books,
+                        'custom_column_' + str(cc_id[0]),
+                        relationship(cc_classes[cc_id[0]],
+                                     secondary=books_custom_column_links[cc_id[0]],
+                                     backref='books'))
+
+        return cc_classes
+
     @classmethod
     def setup_db(cls, config, app_db_path):
         cls.config = config
@@ -465,84 +538,24 @@ class CalibreDB():
                                        isolation_level="SERIALIZABLE",
                                        connect_args={'check_same_thread': False},
                                        poolclass=StaticPool)
-            cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
+            with cls.engine.begin() as connection:
-            cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
+                connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
+                connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
 
             conn = cls.engine.connect()
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
-        except Exception as e:
+        except Exception as ex:
-            config.invalidate(e)
+            config.invalidate(ex)
             return False
 
         config.db_configured = True
 
         if not cc_classes:
-            cc = conn.execute("SELECT id, datatype FROM custom_columns")
+            try:
-
+                cc = conn.execute("SELECT id, datatype FROM custom_columns")
-            cc_ids = []
+                cls.setup_db_cc_classes(cc)
-            books_custom_column_links = {}
+            except OperationalError as e:
-            for row in cc:
+                log.debug_or_exception(e)
-                if row.datatype not in cc_exceptions:
-                    if row.datatype == 'series':
-                        dicttable = {'__tablename__': 'books_custom_column_' + str(row.id) + '_link',
-                                     'id': Column(Integer, primary_key=True),
-                                     'book': Column(Integer, ForeignKey('books.id'),
-                                                    primary_key=True),
-                                     'map_value': Column('value', Integer,
-                                                         ForeignKey('custom_column_' +
-                                                                    str(row.id) + '.id'),
-                                                         primary_key=True),
-                                     'extra': Column(Float),
-                                     'asoc': relationship('custom_column_' + str(row.id), uselist=False),
-                                     'value': association_proxy('asoc', 'value')
-                                     }
-                        books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'),
-                                                                 (Base,), dicttable)
-                    else:
-                        books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link',
-                                                                  Base.metadata,
-                                                                  Column('book', Integer, ForeignKey('books.id'),
-                                                                         primary_key=True),
-                                                                  Column('value', Integer,
-                                                                         ForeignKey('custom_column_' +
-                                                                                    str(row.id) + '.id'),
-                                                                         primary_key=True)
-                                                                  )
-                    cc_ids.append([row.id, row.datatype])
-
-                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                              'id': Column(Integer, primary_key=True)}
-                    if row.datatype == 'float':
-                        ccdict['value'] = Column(Float)
-                    elif row.datatype == 'int':
-                        ccdict['value'] = Column(Integer)
-                    elif row.datatype == 'bool':
-                        ccdict['value'] = Column(Boolean)
-                    else:
-                        ccdict['value'] = Column(String)
-                    if row.datatype in ['float', 'int', 'bool']:
-                        ccdict['book'] = Column(Integer, ForeignKey('books.id'))
-                    cc_classes[row.id] = type(str('custom_column_' + str(row.id)), (Base,), ccdict)
-
-            for cc_id in cc_ids:
-                if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'):
-                    setattr(Books,
-                            'custom_column_' + str(cc_id[0]),
-                            relationship(cc_classes[cc_id[0]],
-                                         primaryjoin=(
-                                             Books.id == cc_classes[cc_id[0]].book),
-                                         backref='books'))
-                elif (cc_id[1] == 'series'):
-                    setattr(Books,
-                            'custom_column_' + str(cc_id[0]),
-                            relationship(books_custom_column_links[cc_id[0]],
-                                         backref='books'))
-                else:
-                    setattr(Books,
-                            'custom_column_' + str(cc_id[0]),
-                            relationship(cc_classes[cc_id[0]],
-                                         secondary=books_custom_column_links[cc_id[0]],
-                                         backref='books'))
 
         cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
@@ -614,13 +627,18 @@ class CalibreDB():
             randm = self.session.query(Books) \
                 .filter(self.common_filters(allow_show_archived)) \
                 .order_by(func.random()) \
-                .limit(self.config.config_random_books)
+                .limit(self.config.config_random_books).all()
         else:
             randm = false()
         off = int(int(pagesize) * (page - 1))
-        query = self.session.query(database) \
+        query = self.session.query(database)
-            .join(*join, isouter=True) \
+        if len(join) == 3:
-            .filter(db_filter) \
+            query = query.outerjoin(join[0], join[1]).outerjoin(join[2])
+        elif len(join) == 2:
+            query = query.outerjoin(join[0], join[1])
+        elif len(join) == 1:
+            query = query.outerjoin(join[0])
+        query = query.filter(db_filter)\
             .filter(self.common_filters(allow_show_archived))
         entries = list()
         pagination = list()
@@ -628,8 +646,8 @@ class CalibreDB():
             pagination = Pagination(page, pagesize,
                                     len(query.all()))
             entries = query.order_by(*order).offset(off).limit(pagesize).all()
-        except Exception as e:
+        except Exception as ex:
-            log.debug_or_exception(e)
+            log.debug_or_exception(ex)
         #for book in entries:
         #    book = self.order_authors(book)
         return entries, randm, pagination
@@ -773,7 +791,7 @@ class CalibreDB():
 def lcase(s):
     try:
         return unidecode.unidecode(s.lower())
-    except Exception as e:
+    except Exception as ex:
         log = logger.create()
-        log.debug_or_exception(e)
+        log.debug_or_exception(ex)
         return s.lower()

cps/editbooks.py

@@ -36,6 +36,8 @@ except ImportError:
     pass
 
 
+from babel import Locale as LC
+from babel.core import UnknownLocaleError
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
 from flask_babel import gettext as _
 from flask_login import current_user, login_required
@@ -77,17 +79,7 @@ def edit_required(f):
 
     return inner
 
-
+def search_objects_remove(db_book_object, db_type, input_elements):
-# Modifies different Database objects, first check if elements have to be added to database, than check
-# if elements have to be deleted, because they are no longer used
-def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type):
-    # passing input_elements not as a list may lead to undesired results
-    if not isinstance(input_elements, list):
-        raise TypeError(str(input_elements) + " should be passed as a list")
-    changed = False
-    input_elements = [x for x in input_elements if x != '']
-    # we have all input element (authors, series, tags) names now
-    # 1. search for elements to remove
     del_elements = []
     for c_elements in db_book_object:
         found = False
@@ -105,7 +97,10 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
         # if the element was not found in the new list, add it to remove list
         if not found:
             del_elements.append(c_elements)
-    # 2. search for elements that need to be added
+    return del_elements
+
+
+def search_objects_add(db_book_object, db_type, input_elements):
     add_elements = []
     for inp_element in input_elements:
         found = False
@@ -121,64 +116,96 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
                 break
         if not found:
             add_elements.append(inp_element)
-    # if there are elements to remove, we remove them now
+    return add_elements
+
+
+def remove_objects(db_book_object, db_session, del_elements):
+    changed = False
     if len(del_elements) > 0:
         for del_element in del_elements:
             db_book_object.remove(del_element)
             changed = True
             if len(del_element.books) == 0:
                 db_session.delete(del_element)
+    return changed
+
+def add_objects(db_book_object, db_object, db_session, db_type, add_elements):
+    changed = False
+    if db_type == 'languages':
+        db_filter = db_object.lang_code
+    elif db_type == 'custom':
+        db_filter = db_object.value
+    else:
+        db_filter = db_object.name
+    for add_element in add_elements:
+        # check if a element with that name exists
+        db_element = db_session.query(db_object).filter(db_filter == add_element).first()
+        # if no element is found add it
+        # if new_element is None:
+        if db_type == 'author':
+            new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')), "")
+        elif db_type == 'series':
+            new_element = db_object(add_element, add_element)
+        elif db_type == 'custom':
+            new_element = db_object(value=add_element)
+        elif db_type == 'publisher':
+            new_element = db_object(add_element, None)
+        else:  # db_type should be tag or language
+            new_element = db_object(add_element)
+        if db_element is None:
+            changed = True
+            db_session.add(new_element)
+            db_book_object.append(new_element)
+        else:
+            db_element = create_objects_for_addition(db_element, add_element, db_type)
+            changed = True
+            # add element to book
+            changed = True
+            db_book_object.append(db_element)
+    return changed
+
+
+def create_objects_for_addition(db_element, add_element, db_type):
+    if db_type == 'custom':
+        if db_element.value != add_element:
+            db_element.value = add_element  # ToDo: Before new_element, but this is not plausible
+    elif db_type == 'languages':
+        if db_element.lang_code != add_element:
+            db_element.lang_code = add_element
+    elif db_type == 'series':
+        if db_element.name != add_element:
+            db_element.name = add_element
+            db_element.sort = add_element
+    elif db_type == 'author':
+        if db_element.name != add_element:
+            db_element.name = add_element
+            db_element.sort = add_element.replace('|', ',')
+    elif db_type == 'publisher':
+        if db_element.name != add_element:
+            db_element.name = add_element
+            db_element.sort = None
+    elif db_element.name != add_element:
+        db_element.name = add_element
+    return db_element
+
+
+# Modifies different Database objects, first check if elements if elements have to be deleted,
+# because they are no longer used, than check if elements have to be added to database
+def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type):
+    # passing input_elements not as a list may lead to undesired results
+    if not isinstance(input_elements, list):
+        raise TypeError(str(input_elements) + " should be passed as a list")
+    input_elements = [x for x in input_elements if x != '']
+    # we have all input element (authors, series, tags) names now
+    # 1. search for elements to remove
+    del_elements = search_objects_remove(db_book_object, db_type, input_elements)
+    # 2. search for elements that need to be added
+    add_elements = search_objects_add(db_book_object, db_type, input_elements)
+    # if there are elements to remove, we remove them now
+    changed = remove_objects(db_book_object, db_session, del_elements)
     # if there are elements to add, we add them now!
     if len(add_elements) > 0:
-        if db_type == 'languages':
+        changed |= add_objects(db_book_object, db_object, db_session, db_type, add_elements)
-            db_filter = db_object.lang_code
-        elif db_type == 'custom':
-            db_filter = db_object.value
-        else:
-            db_filter = db_object.name
-        for add_element in add_elements:
-            # check if a element with that name exists
-            db_element = db_session.query(db_object).filter(db_filter == add_element).first()
-            # if no element is found add it
-            # if new_element is None:
-            if db_type == 'author':
-                new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')), "")
-            elif db_type == 'series':
-                new_element = db_object(add_element, add_element)
-            elif db_type == 'custom':
-                new_element = db_object(value=add_element)
-            elif db_type == 'publisher':
-                new_element = db_object(add_element, None)
-            else:  # db_type should be tag or language
-                new_element = db_object(add_element)
-            if db_element is None:
-                changed = True
-                db_session.add(new_element)
-                db_book_object.append(new_element)
-            else:
-                if db_type == 'custom':
-                    if db_element.value != add_element:
-                        new_element.value = add_element
-                elif db_type == 'languages':
-                    if db_element.lang_code != add_element:
-                        db_element.lang_code = add_element
-                elif db_type == 'series':
-                    if db_element.name != add_element:
-                        db_element.name = add_element
-                        db_element.sort = add_element
-                elif db_type == 'author':
-                    if db_element.name != add_element:
-                        db_element.name = add_element
-                        db_element.sort = add_element.replace('|', ',')
-                elif db_type == 'publisher':
-                    if db_element.name != add_element:
-                        db_element.name = add_element
-                        db_element.sort = None
-                elif db_element.name != add_element:
-                    db_element.name = add_element
-                # add element to book
-                changed = True
-                db_book_object.append(db_element)
     return changed
 
 
@@ -318,8 +345,8 @@ def delete_book(book_id, book_format, jsonResponse):
                     calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\
                         filter(db.Data.format == book_format).delete()
                 calibre_db.session.commit()
-            except Exception as e:
+            except Exception as ex:
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
                 calibre_db.session.rollback()
         else:
             # book not found
@@ -431,7 +458,7 @@ def edit_book_comments(comments, book):
     return modif_date
 
 
-def edit_book_languages(languages, book, upload=False):
+def edit_book_languages(languages, book, upload=False, invalid=None):
     input_languages = languages.split(',')
     unknown_languages = []
     if not upload:
@@ -440,7 +467,10 @@ def edit_book_languages(languages, book, upload=False):
         input_l = isoLanguages.get_valid_language_codes(get_locale(), input_languages, unknown_languages)
     for l in unknown_languages:
         log.error('%s is not a valid language', l)
-        flash(_(u"%(langname)s is not a valid language", langname=l), category="warning")
+        if isinstance(invalid, list):
+            invalid.append(l)
+        else:
+            flash(_(u"%(langname)s is not a valid language", langname=l), category="warning")
     # ToDo: Not working correct
     if upload and len(input_l) == 1:
         # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
@@ -606,7 +636,7 @@ def upload_single_file(request, book, book_id):
 
             # Queue uploader info
             uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
-            WorkerThread.add(current_user.nickname, TaskUpload(
+            WorkerThread.add(current_user.name, TaskUpload(
                 "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>"))
 
             return uploader.process(
@@ -630,6 +660,46 @@ def upload_cover(request, book):
     return None
 
 
+def handle_title_on_edit(book, book_title):
+    # handle book title
+    book_title = book_title.rstrip().strip()
+    if book.title != book_title:
+        if book_title == '':
+            book_title = _(u'Unknown')
+        book.title = book_title
+        return True
+    return False
+
+
+def handle_author_on_edit(book, author_name, update_stored=True):
+    # handle author(s)
+    input_authors = author_name.split('&')
+    input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
+    # Remove duplicates in authors list
+    input_authors = helper.uniq(input_authors)
+    # we have all author names now
+    if input_authors == ['']:
+        input_authors = [_(u'Unknown')]  # prevent empty Author
+
+    change = modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
+
+    # Search for each author if author is in database, if not, author name and sorted author name is generated new
+    # everything then is assembled for sorted author field in database
+    sort_authors_list = list()
+    for inp in input_authors:
+        stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+        if not stored_author:
+            stored_author = helper.get_sorted_author(inp)
+        else:
+            stored_author = stored_author.sort
+        sort_authors_list.append(helper.get_sorted_author(stored_author))
+    sort_authors = ' & '.join(sort_authors_list)
+    if book.author_sort != sort_authors and update_stored:
+        book.author_sort = sort_authors
+        change = True
+    return input_authors, change
+
+
 @editbook.route("/admin/book/<int:book_id>", methods=['GET', 'POST'])
 @login_required_if_no_ano
 @edit_required
@@ -647,7 +717,6 @@ def edit_book(book_id):
     if request.method != 'POST':
         return render_edit_book(book_id)
 
-
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
 
     # Book not found
@@ -665,41 +734,14 @@ def edit_book(book_id):
         # Update book
         edited_books_id = None
 
-        #handle book title
+        # handle book title
-        if book.title != to_save["book_title"].rstrip().strip():
+        title_change = handle_title_on_edit(book, to_save["book_title"])
-            if to_save["book_title"] == '':
+
-                to_save["book_title"] = _(u'Unknown')
+        input_authors, authorchange = handle_author_on_edit(book, to_save["author_name"])
-            book.title = to_save["book_title"].rstrip().strip()
+        if authorchange or title_change:
             edited_books_id = book.id
             modif_date = True
 
-        # handle author(s)
-        input_authors = to_save["author_name"].split('&')
-        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
-        # Remove duplicates in authors list
-        input_authors = helper.uniq(input_authors)
-        # we have all author names now
-        if input_authors == ['']:
-            input_authors = [_(u'Unknown')]  # prevent empty Author
-
-        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
-
-        # Search for each author if author is in database, if not, authorname and sorted authorname is generated new
-        # everything then is assembled for sorted author field in database
-        sort_authors_list = list()
-        for inp in input_authors:
-            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
-            if not stored_author:
-                stored_author = helper.get_sorted_author(inp)
-            else:
-                stored_author = stored_author.sort
-            sort_authors_list.append(helper.get_sorted_author(stored_author))
-        sort_authors = ' & '.join(sort_authors_list)
-        if book.author_sort != sort_authors:
-            edited_books_id = book.id
-            book.author_sort = sort_authors
-            modif_date = True
-
         if config.config_use_google_drive:
             gdriveutils.updateGdriveCalibreFromLocal()
 
@@ -724,10 +766,8 @@ def edit_book(book_id):
 
             # Add default series_index to book
             modif_date |= edit_book_series_index(to_save["series_index"], book)
-
             # Handle book comments/description
             modif_date |= edit_book_comments(to_save["description"], book)
-
             # Handle identifiers
             input_identifiers = identifier_list(to_save, book)
             modification, warning = modify_identifiers(input_identifiers, book.identifiers, calibre_db.session)
@@ -736,9 +776,16 @@ def edit_book(book_id):
             modif_date |= modification
             # Handle book tags
             modif_date |= edit_book_tags(to_save['tags'], book)
-
             # Handle book series
             modif_date |= edit_book_series(to_save["series"], book)
+            # handle book publisher
+            modif_date |= edit_book_publisher(to_save['publisher'], book)
+            # handle book languages
+            modif_date |= edit_book_languages(to_save['languages'], book)
+            # handle book ratings
+            modif_date |= edit_book_ratings(to_save, book)
+            # handle cc data
+            modif_date |= edit_cc_data(book_id, book, to_save)
 
             if to_save["pubdate"]:
                 try:
@@ -748,18 +795,6 @@ def edit_book(book_id):
             else:
                 book.pubdate = db.Books.DEFAULT_PUBDATE
 
-            # handle book publisher
-            modif_date |= edit_book_publisher(to_save['publisher'], book)
-
-            # handle book languages
-            modif_date |= edit_book_languages(to_save['languages'], book)
-
-            # handle book ratings
-            modif_date |= edit_book_ratings(to_save, book)
-
-            # handle cc data
-            modif_date |= edit_cc_data(book_id, book, to_save)
-
             if modif_date:
                 book.last_modified = datetime.utcnow()
             calibre_db.session.merge(book)
@@ -775,8 +810,8 @@ def edit_book(book_id):
             calibre_db.session.rollback()
             flash(error, category="error")
             return render_edit_book(book_id)
-    except Exception as e:
+    except Exception as ex:
-        log.debug_or_exception(e)
+        log.debug_or_exception(ex)
         calibre_db.session.rollback()
         flash(_("Error editing book, please check logfile for details"), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
@@ -892,6 +927,48 @@ def create_book_on_upload(modif_date, meta):
     calibre_db.session.flush()
     return db_book, input_authors, title_dir
 
+def file_handling_on_upload(requested_file):
+    # check if file extension is correct
+    if '.' in requested_file.filename:
+        file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
+        if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
+            flash(
+                _("File extension '%(ext)s' is not allowed to be uploaded to this server",
+                  ext=file_ext), category="error")
+            return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+    else:
+        flash(_('File to be uploaded must have an extension'), category="error")
+        return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+
+    # extract metadata from file
+    try:
+        meta = uploader.upload(requested_file, config.config_rarfile_location)
+    except (IOError, OSError):
+        log.error("File %s could not saved to temp dir", requested_file.filename)
+        flash(_(u"File %(filename)s could not saved to temp dir",
+                filename=requested_file.filename), category="error")
+        return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+    return meta, None
+
+
+def move_coverfile(meta, db_book):
+    # move cover to final directory, including book id
+    if meta.cover:
+        coverfile = meta.cover
+    else:
+        coverfile = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
+    new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
+    try:
+        copyfile(coverfile, new_coverpath)
+        if meta.cover:
+            os.unlink(meta.cover)
+    except OSError as e:
+        log.error("Failed to move cover file %s: %s", new_coverpath, e)
+        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
+                error=e),
+              category="error")
+
+
 @editbook.route("/upload", methods=["GET", "POST"])
 @login_required_if_no_ano
 @upload_required
@@ -906,30 +983,13 @@ def upload():
                 calibre_db.update_title_sort(config)
                 calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
 
-                # check if file extension is correct
+                meta, error = file_handling_on_upload(requested_file)
-                if '.' in requested_file.filename:
+                if error:
-                    file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
+                    return error
-                    if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
-                        flash(
-                            _("File extension '%(ext)s' is not allowed to be uploaded to this server",
-                              ext=file_ext), category="error")
-                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-                else:
-                    flash(_('File to be uploaded must have an extension'), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-
-                # extract metadata from file
-                try:
-                    meta = uploader.upload(requested_file, config.config_rarfile_location)
-                except (IOError, OSError):
-                    log.error("File %s could not saved to temp dir", requested_file.filename)
-                    flash(_(u"File %(filename)s could not saved to temp dir",
-                            filename= requested_file.filename), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
                 db_book, input_authors, title_dir = create_book_on_upload(modif_date, meta)
 
-                # Comments needs book id therfore only possible after flush
+                # Comments needs book id therefore only possible after flush
                 modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
 
                 book_id = db_book.id
@@ -941,21 +1001,7 @@ def upload():
                                                    meta.file_path,
                                                    title_dir + meta.extension)
 
-                # move cover to final directory, including book id
+                move_coverfile(meta, db_book)
-                if meta.cover:
-                    coverfile = meta.cover
-                else:
-                    coverfile = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
-                new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
-                try:
-                    copyfile(coverfile, new_coverpath)
-                    if meta.cover:
-                        os.unlink(meta.cover)
-                except OSError as e:
-                    log.error("Failed to move cover file %s: %s", new_coverpath, e)
-                    flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
-                            error=e),
-                          category="error")
 
                 # save data to database, reread data
                 calibre_db.session.commit()
@@ -965,7 +1011,7 @@ def upload():
                 if error:
                     flash(error, category="error")
                 uploadText=_(u"File %(file)s uploaded", file=title)
-                WorkerThread.add(current_user.nickname, TaskUpload(
+                WorkerThread.add(current_user.name, TaskUpload(
                     "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>"))
 
                 if len(request.files.getlist("btn-upload")) < 2:
@@ -995,7 +1041,7 @@ def convert_bookformat(book_id):
 
     log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to)
     rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(),
-                                     book_format_to.upper(), current_user.nickname)
+                                     book_format_to.upper(), current_user.name)
 
     if rtn is None:
         flash(_(u"Book successfully queued for converting to %(book_format)s",
@@ -1023,63 +1069,88 @@ def scholar_search(query):
     else:
         return []
 
-
-
 @editbook.route("/ajax/editbooks/<param>", methods=['POST'])
 @login_required_if_no_ano
 @edit_required
 def edit_list_book(param):
     vals = request.form.to_dict()
     book = calibre_db.get_book(vals['pk'])
+    ret = ""
     if param =='series_index':
         edit_book_series_index(vals['value'], book)
+        ret = Response(json.dumps({'success': True, 'newValue': book.series_index}), mimetype='application/json')
     elif param =='tags':
         edit_book_tags(vals['value'], book)
+        ret = Response(json.dumps({'success': True, 'newValue': ', '.join([tag.name for tag in book.tags])}),
+                       mimetype='application/json')
     elif param =='series':
         edit_book_series(vals['value'], book)
+        ret = Response(json.dumps({'success': True, 'newValue':  ', '.join([serie.name for serie in book.series])}),
+                       mimetype='application/json')
     elif param =='publishers':
-        vals['publisher'] = vals['value']
+        edit_book_publisher(vals['value'], book)
-        edit_book_publisher(vals, book)
+        ret =  Response(json.dumps({'success': True,
+                                    'newValue': ', '.join([publisher.name for publisher in book.publishers])}),
+                       mimetype='application/json')
     elif param =='languages':
-        edit_book_languages(vals['value'], book)
+        invalid = list()
+        edit_book_languages(vals['value'], book, invalid=invalid)
+        if invalid:
+            ret = Response(json.dumps({'success': False,
+                                       'msg': 'Invalid languages in request: {}'.format(','.join(invalid))}),
+                           mimetype='application/json')
+        else:
+            lang_names = list()
+            for lang in book.languages:
+                try:
+                    lang_names.append(LC.parse(lang.lang_code).get_language_name(get_locale()))
+                except UnknownLocaleError:
+                    lang_names.append(_(isoLanguages.get(part3=lang.lang_code).name))
+            ret =  Response(json.dumps({'success': True, 'newValue':  ', '.join(lang_names)}),
+                            mimetype='application/json')
     elif param =='author_sort':
         book.author_sort = vals['value']
-    elif param =='title':
+        ret = Response(json.dumps({'success': True, 'newValue':  book.author_sort}),
-        book.title = vals['value']
+                       mimetype='application/json')
+    elif param == 'title':
+        sort = book.sort
+        handle_title_on_edit(book, vals.get('value', ""))
         helper.update_dir_stucture(book.id, config.config_calibre_dir)
+        ret = Response(json.dumps({'success': True, 'newValue':  book.title}),
+                       mimetype='application/json')
     elif param =='sort':
         book.sort = vals['value']
-    # ToDo: edit books
+        ret = Response(json.dumps({'success': True, 'newValue':  book.sort}),
+                       mimetype='application/json')
     elif param =='authors':
-        input_authors = vals['value'].split('&')
+        input_authors, __ = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
-        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
-        modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
-        sort_authors_list = list()
-        for inp in input_authors:
-            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
-            if not stored_author:
-                stored_author = helper.get_sorted_author(inp)
-            else:
-                stored_author = stored_author.sort
-            sort_authors_list.append(helper.get_sorted_author(stored_author))
-        sort_authors = ' & '.join(sort_authors_list)
-        if book.author_sort != sort_authors:
-            book.author_sort = sort_authors
         helper.update_dir_stucture(book.id, config.config_calibre_dir, input_authors[0])
+        ret = Response(json.dumps({'success': True,
+                                   'newValue':  ' & '.join([author.replace('|',',') for author in input_authors])}),
+                       mimetype='application/json')
     book.last_modified = datetime.utcnow()
     calibre_db.session.commit()
-    return ""
+    # revert change for sort if automatic fields link is deactivated
+    if param == 'title' and vals.get('checkT') == "false":
+        book.sort = sort
+        calibre_db.session.commit()
+    return ret
+
 
 @editbook.route("/ajax/sort_value/<field>/<int:bookid>")
 @login_required
 def get_sorted_entry(field, bookid):
-    if field == 'title' or field == 'authors':
+    if field in ['title', 'authors', 'sort', 'author_sort']:
         book = calibre_db.get_filtered_book(bookid)
         if book:
             if field == 'title':
                 return json.dumps({'sort': book.sort})
             elif field == 'authors':
                 return json.dumps({'author_sort': book.author_sort})
+            if field == 'sort':
+                return json.dumps({'sort': book.title})
+            if field == 'author_sort':
+                return json.dumps({'author_sort': book.author})
     return ""
 
 

cps/epub.py

@@ -87,18 +87,29 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     lang = epub_metadata['language'].split('-', 1)[0].lower()
     epub_metadata['language'] = isoLanguages.get_lang3(lang)
 
-    series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns)
+    epub_metadata = parse_epbub_series(ns, tree, epub_metadata)
-    if len(series) > 0:
-        epub_metadata['series'] = series[0]
-    else:
-        epub_metadata['series'] = ''
 
-    series_id = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series_index']/@content", namespaces=ns)
+    coverfile = parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path)
-    if len(series_id) > 0:
-        epub_metadata['series_id'] = series_id[0]
-    else:
-        epub_metadata['series_id'] = '1'
 
+    if not epub_metadata['title']:
+        title = original_file_name
+    else:
+        title = epub_metadata['title']
+
+    return BookMeta(
+        file_path=tmp_file_path,
+        extension=original_file_extension,
+        title=title.encode('utf-8').decode('utf-8'),
+        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
+        cover=coverfile,
+        description=epub_metadata['description'],
+        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
+        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
+        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
+        languages=epub_metadata['language'],
+        publisher="")
+
+def parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path):
     coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
     coverfile = None
     if len(coversection) > 0:
@@ -126,21 +137,18 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
                     coverfile = extractCover(epubZip, filename, "", tmp_file_path)
             else:
                 coverfile = extractCover(epubZip, coversection[0], coverpath, tmp_file_path)
+    return coverfile
 
-    if not epub_metadata['title']:
+def parse_epbub_series(ns, tree, epub_metadata):
-        title = original_file_name
+    series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns)
+    if len(series) > 0:
+        epub_metadata['series'] = series[0]
     else:
-        title = epub_metadata['title']
+        epub_metadata['series'] = ''
 
-    return BookMeta(
+    series_id = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series_index']/@content", namespaces=ns)
-        file_path=tmp_file_path,
+    if len(series_id) > 0:
-        extension=original_file_extension,
+        epub_metadata['series_id'] = series_id[0]
-        title=title.encode('utf-8').decode('utf-8'),
+    else:
-        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
+        epub_metadata['series_id'] = '1'
-        cover=coverfile,
+    return epub_metadata
-        description=epub_metadata['description'],
-        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
-        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
-        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
-        languages=epub_metadata['language'],
-        publisher="")

cps/gdrive.py

@@ -155,6 +155,6 @@ def on_received_watch_confirmation():
                 # prevent error on windows, as os.rename does on existing files, also allow cross hdd move
                 move(os.path.join(tmp_dir, "tmp_metadata.db"), dbpath)
                 calibre_db.reconnect_db(config, ub.app_DB_path)
-    except Exception as e:
+    except Exception as ex:
-        log.debug_or_exception(e)
+        log.debug_or_exception(ex)
     return ''

cps/gdriveutils.py

@@ -28,7 +28,11 @@ from sqlalchemy import create_engine
 from sqlalchemy import Column, UniqueConstraint
 from sqlalchemy import String, Integer
 from sqlalchemy.orm import sessionmaker, scoped_session
-from sqlalchemy.ext.declarative import declarative_base
+try:
+    # Compability with sqlalchemy 2.0
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.exc import OperationalError, InvalidRequestError
 
 try:
@@ -198,8 +202,8 @@ def getDrive(drive=None, gauth=None):
                 gauth.Refresh()
             except RefreshError as e:
                 log.error("Google Drive error: %s", e)
-            except Exception as e:
+            except Exception as ex:
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
         else:
             # Initialize the saved creds
             gauth.Authorize()
@@ -493,8 +497,8 @@ def getChangeById (drive, change_id):
     except (errors.HttpError) as error:
         log.error(error)
         return None
-    except Exception as e:
+    except Exception as ex:
-        log.error(e)
+        log.error(ex)
         return None
 
 

cps/helper.py

@@ -35,7 +35,7 @@ from babel.units import format_unit
 from flask import send_from_directory, make_response, redirect, abort, url_for
 from flask_babel import gettext as _
 from flask_login import current_user
-from sqlalchemy.sql.expression import true, false, and_, text
+from sqlalchemy.sql.expression import true, false, and_, text, func
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash
 
@@ -480,8 +480,8 @@ def reset_password(user_id):
         password = generate_random_password()
         existing_user.password = generate_password_hash(password)
         ub.session.commit()
-        send_registration_mail(existing_user.email, existing_user.nickname, password, True)
+        send_registration_mail(existing_user.email, existing_user.name, password, True)
-        return 1, existing_user.nickname
+        return 1, existing_user.name
     except Exception:
         ub.session.rollback()
         return 0, None
@@ -498,11 +498,37 @@ def generate_random_password():
 
 def uniq(inpt):
     output = []
+    inpt = [ " ".join(inp.split()) for inp in inpt]
     for x in inpt:
         if x not in output:
             output.append(x)
     return output
 
+def check_email(email):
+    email = valid_email(email)
+    if ub.session.query(ub.User).filter(func.lower(ub.User.email) == email.lower()).first():
+        log.error(u"Found an existing account for this e-mail address")
+        raise Exception(_(u"Found an existing account for this e-mail address"))
+    return email
+
+
+def check_username(username):
+    username = username.strip()
+    if ub.session.query(ub.User).filter(func.lower(ub.User.name) == username.lower()).scalar():
+        log.error(u"This username is already taken")
+        raise Exception (_(u"This username is already taken"))
+    return username
+
+
+def valid_email(email):
+    email = email.strip()
+    # Regex according to https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/email#validation
+    if not re.search(r"^[\w.!#$%&'*+\\/=?^_`{|}~-]+@[\w](?:[\w-]{0,61}[\w])?(?:\.[\w](?:[\w-]{0,61}[\w])?)*$",
+                     email):
+        log.error(u"Invalid e-mail address format")
+        raise Exception(_(u"Invalid e-mail address format"))
+    return email
+
 # ################################# External interface #################################
 
 
@@ -550,8 +576,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure):
                 else:
                     log.error('%s/cover.jpg not found on Google Drive', book.path)
                     return get_cover_on_failure(use_generic_cover_on_failure)
-            except Exception as e:
+            except Exception as ex:
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
                 return get_cover_on_failure(use_generic_cover_on_failure)
         else:
             cover_file_path = os.path.join(config.config_calibre_dir, book.path)
@@ -731,7 +757,7 @@ def format_runtime(runtime):
 def render_task_status(tasklist):
     renderedtasklist = list()
     for __, user, __, task in tasklist:
-        if user == current_user.nickname or current_user.role_admin():
+        if user == current_user.name or current_user.role_admin():
             ret = {}
             if task.start_time:
                 ret['starttime'] = format_datetime(task.start_time, format='short', locale=get_locale())

cps/isoLanguages.py

@@ -63,11 +63,12 @@ def get_language_codes(locale, language_names, remainder=None):
         if v in language_names:
             lang.append(k)
             language_names.remove(v)
-    if remainder is not None:
+    if remainder is not None and language_names:
         remainder.extend(language_names)
     return lang
 
 
+
 def get_valid_language_codes(locale, language_names, remainder=None):
     lang = list()
     if "" in language_names:

cps/jinjia.py

@@ -82,7 +82,7 @@ def formatdate_filter(val):
     except AttributeError as e:
         log.error('Babel error: %s, Current user locale: %s, Current User: %s', e,
                   current_user.locale,
-                  current_user.nickname
+                  current_user.name
                   )
         return val
 

cps/kobo.py

@@ -177,7 +177,7 @@ def HandleSyncRequest():
     for book in changed_entries:
         formats = [data.format for data in book.Books.data]
         if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
-            helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.nickname)
+            helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
 
         kobo_reading_state = get_or_create_reading_state(book.Books.id)
         entitlement = {
@@ -262,8 +262,8 @@ def generate_sync_response(sync_token, sync_results, set_cont=False):
             extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode")
             extra_headers["x-kobo-recent-reads"] = store_response.headers.get("x-kobo-recent-reads")
 
-        except Exception as e:
+        except Exception as ex:
-            log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e))
+            log.error("Failed to receive or parse response from Kobo's sync endpoint: {}".format(ex))
     if set_cont:
         extra_headers["x-kobo-sync"] = "continue"
     sync_token.to_headers(extra_headers)

cps/kobo_auth.py

@@ -155,7 +155,7 @@ def generate_auth_token(user_id):
         for book in books:
             formats = [data.format for data in book.data]
             if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
-                helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.nickname)
+                helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
 
         return render_title_template(
             "generate_kobo_auth_url.html",

cps/oauth_bb.py

@@ -42,6 +42,7 @@ except NameError:
 
 
 oauth_check = {}
+oauthblueprints = []
 oauth = Blueprint('oauth', __name__)
 log = logger.create()
 
@@ -87,7 +88,7 @@ def register_user_with_oauth(user=None):
             except NoResultFound:
                 # no found, return error
                 return
-            ub.session_commit("User {} with OAuth for provider {} registered".format(user.nickname, oauth_key))
+            ub.session_commit("User {} with OAuth for provider {} registered".format(user.name, oauth_key))
 
 
 def logout_oauth_user():
@@ -133,8 +134,8 @@ def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider
         # already bind with user, just login
         if oauth_entry.user:
             login_user(oauth_entry.user)
-            log.debug(u"You are now logged in as: '%s'", oauth_entry.user.nickname)
+            log.debug(u"You are now logged in as: '%s'", oauth_entry.user.name)
-            flash(_(u"you are now logged in as: '%(nickname)s'", nickname= oauth_entry.user.nickname),
+            flash(_(u"you are now logged in as: '%(nickname)s'", nickname= oauth_entry.user.name),
                   category="success")
             return redirect(url_for('web.index'))
         else:
@@ -146,8 +147,8 @@ def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider
                     ub.session.commit()
                     flash(_(u"Link to %(oauth)s Succeeded", oauth=provider_name), category="success")
                     return redirect(url_for('web.profile'))
-                except Exception as e:
+                except Exception as ex:
-                    log.debug_or_exception(e)
+                    log.debug_or_exception(ex)
                     ub.session.rollback()
             else:
                 flash(_(u"Login failed, No User Linked With OAuth Account"), category="error")
@@ -193,8 +194,8 @@ def unlink_oauth(provider):
                 ub.session.commit()
                 logout_oauth_user()
                 flash(_(u"Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success")
-            except Exception as e:
+            except Exception as ex:
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
                 ub.session.rollback()
                 flash(_(u"Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error")
     except NoResultFound:
@@ -203,7 +204,6 @@ def unlink_oauth(provider):
     return redirect(url_for('web.profile'))
 
 def generate_oauth_blueprints():
-    oauthblueprints = []
     if not ub.session.query(ub.OAuthProvider).count():
         for provider in ("github", "google"):
             oauthProvider = ub.OAuthProvider()
@@ -299,39 +299,6 @@ if ub.oauth_support:
         )  # ToDo: Translate
         flash(msg, category="error")
 
-
-    @oauth.route('/link/github')
-    @oauth_required
-    def github_login():
-        if not github.authorized:
-            return redirect(url_for('github.login'))
-        account_info = github.get('/user')
-        if account_info.ok:
-            account_info_json = account_info.json()
-            return bind_oauth_or_register(oauthblueprints[0]['id'], account_info_json['id'], 'github.login', 'github')
-        flash(_(u"GitHub Oauth error, please retry later."), category="error")
-        return redirect(url_for('web.login'))
-
-
-    @oauth.route('/unlink/github', methods=["GET"])
-    @login_required
-    def github_login_unlink():
-        return unlink_oauth(oauthblueprints[0]['id'])
-
-
-    @oauth.route('/link/google')
-    @oauth_required
-    def google_login():
-        if not google.authorized:
-            return redirect(url_for("google.login"))
-        resp = google.get("/oauth2/v2/userinfo")
-        if resp.ok:
-            account_info_json = resp.json()
-            return bind_oauth_or_register(oauthblueprints[1]['id'], account_info_json['id'], 'google.login', 'google')
-        flash(_(u"Google Oauth error, please retry later."), category="error")
-        return redirect(url_for('web.login'))
-
-
     @oauth_error.connect_via(oauthblueprints[1]['blueprint'])
     def google_error(blueprint, error, error_description=None, error_uri=None):
         msg = (
@@ -346,7 +313,39 @@ if ub.oauth_support:
         flash(msg, category="error")
 
 
-    @oauth.route('/unlink/google', methods=["GET"])
+@oauth.route('/link/github')
-    @login_required
+@oauth_required
-    def google_login_unlink():
+def github_login():
-        return unlink_oauth(oauthblueprints[1]['id'])
+    if not github.authorized:
+        return redirect(url_for('github.login'))
+    account_info = github.get('/user')
+    if account_info.ok:
+        account_info_json = account_info.json()
+        return bind_oauth_or_register(oauthblueprints[0]['id'], account_info_json['id'], 'github.login', 'github')
+    flash(_(u"GitHub Oauth error, please retry later."), category="error")
+    return redirect(url_for('web.login'))
+
+
+@oauth.route('/unlink/github', methods=["GET"])
+@login_required
+def github_login_unlink():
+    return unlink_oauth(oauthblueprints[0]['id'])
+
+
+@oauth.route('/link/google')
+@oauth_required
+def google_login():
+    if not google.authorized:
+        return redirect(url_for("google.login"))
+    resp = google.get("/oauth2/v2/userinfo")
+    if resp.ok:
+        account_info_json = resp.json()
+        return bind_oauth_or_register(oauthblueprints[1]['id'], account_info_json['id'], 'google.login', 'google')
+    flash(_(u"Google Oauth error, please retry later."), category="error")
+    return redirect(url_for('web.login'))
+
+
+@oauth.route('/unlink/google', methods=["GET"])
+@login_required
+def google_login_unlink():
+    return unlink_oauth(oauthblueprints[1]['id'])

cps/opds.py

@@ -27,7 +27,7 @@ from functools import wraps
 
 from flask import Blueprint, request, render_template, Response, g, make_response, abort
 from flask_login import current_user
-from sqlalchemy.sql.expression import func, text, or_, and_
+from sqlalchemy.sql.expression import func, text, or_, and_, true
 from werkzeug.security import check_password_hash
 
 from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages
@@ -97,6 +97,44 @@ def feed_normal_search():
     return feed_search(request.args.get("query", "").strip())
 
 
+@opds.route("/opds/books")
+@requires_basic_auth_if_no_ano
+def feed_booksindex():
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Books.sort, 1, 1)).label('id'))\
+        .filter(calibre_db.common_filters()).group_by(func.upper(func.substr(db.Books.sort, 1, 1))).all()
+
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_books',
+                               pagination=pagination)
+
+
+@opds.route("/opds/books/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_books(book_id):
+    off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Books.sort).startswith(book_id)
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
+                                                        db.Books,
+                                                        letter,
+                                                        [db.Books.sort])
+
+    return render_xml_template('feed.xml', entries=entries, pagination=pagination)
+
+
 @opds.route("/opds/new")
 @requires_basic_auth_if_no_ano
 def feed_new():
@@ -150,14 +188,41 @@ def feed_hot():
 @opds.route("/opds/author")
 @requires_basic_auth_if_no_ano
 def feed_authorindex():
-    off = request.args.get("offset") or 0
+    shift = 0
-    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
+    off = int(request.args.get("offset") or 0)
-        .filter(calibre_db.common_filters())\
+    entries = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('id'))\
-        .group_by(text('books_authors_link.author'))\
+        .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters())\
-        .order_by(db.Authors.sort).limit(config.config_books_per_page)\
+        .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all()
-        .offset(off)
+
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(calibre_db.session.query(db.Authors).all()))
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_author',
+                               pagination=pagination)
+
+
+@opds.route("/opds/author/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_author(book_id):
+    off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Authors.sort).startswith(book_id)
+    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
+        .filter(calibre_db.common_filters()).filter(letter)\
+        .group_by(text('books_authors_link.author'))\
+        .order_by(db.Authors.sort)
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            entries.count())
+    entries = entries.limit(config.config_books_per_page).offset(off).all()
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_author', pagination=pagination)
 
 
@@ -201,17 +266,41 @@ def feed_publisher(book_id):
 @opds.route("/opds/category")
 @requires_basic_auth_if_no_ano
 def feed_categoryindex():
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('id'))\
+        .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all()
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_category',
+                               pagination=pagination)
+
+@opds.route("/opds/category/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_category(book_id):
     off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Tags.name).startswith(book_id)
     entries = calibre_db.session.query(db.Tags)\
         .join(db.books_tags_link)\
         .join(db.Books)\
-        .filter(calibre_db.common_filters())\
+        .filter(calibre_db.common_filters()).filter(letter)\
         .group_by(text('books_tags_link.tag'))\
-        .order_by(db.Tags.name)\
+        .order_by(db.Tags.name)
-        .offset(off)\
-        .limit(config.config_books_per_page)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(calibre_db.session.query(db.Tags).all()))
+                            entries.count())
+    entries = entries.offset(off).limit(config.config_books_per_page).all()
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_category', pagination=pagination)
 
 
@@ -229,16 +318,40 @@ def feed_category(book_id):
 @opds.route("/opds/series")
 @requires_basic_auth_if_no_ano
 def feed_seriesindex():
+    shift = 0
+    off = int(request.args.get("offset") or 0)
+    entries = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('id'))\
+        .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
+    elements = []
+    if off == 0:
+        elements.append({'id': "00", 'name':_("All")})
+        shift = 1
+    for entry in entries[
+                 off + shift - 1:
+                 int(off + int(config.config_books_per_page) - shift)]:
+        elements.append({'id': entry.id, 'name': entry.id})
+    pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
+                            len(entries) + 1)
+    return render_xml_template('feed.xml',
+                               letterelements=elements,
+                               folder='opds.feed_letter_series',
+                               pagination=pagination)
+
+@opds.route("/opds/series/letter/<book_id>")
+@requires_basic_auth_if_no_ano
+def feed_letter_series(book_id):
     off = request.args.get("offset") or 0
+    letter = true() if book_id == "00" else func.upper(db.Series.sort).startswith(book_id)
     entries = calibre_db.session.query(db.Series)\
         .join(db.books_series_link)\
         .join(db.Books)\
-        .filter(calibre_db.common_filters())\
+        .filter(calibre_db.common_filters()).filter(letter)\
         .group_by(text('books_series_link.series'))\
-        .order_by(db.Series.sort)\
+        .order_by(db.Series.sort)
-        .offset(off).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(calibre_db.session.query(db.Series).all()))
+                            entries.count())
+    entries = entries.offset(off).limit(config.config_books_per_page).all()
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_series', pagination=pagination)
 
 
@@ -269,7 +382,7 @@ def feed_ratingindex():
                             len(entries))
     element = list()
     for entry in entries:
-        element.append(FeedObject(entry[0].id, "{} Stars".format(entry.name)))
+        element.append(FeedObject(entry[0].id, _("{} Stars").format(entry.name)))
     return render_xml_template('feed.xml', listelements=element, folder='opds.feed_ratings', pagination=pagination)
 
 
@@ -428,7 +541,7 @@ def check_auth(username, password):
             username = username.encode('windows-1252')
         except UnicodeEncodeError:
             username = username.encode('utf-8')
-    user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) ==
+    user = ub.session.query(ub.User).filter(func.lower(ub.User.name) ==
                                             username.decode('utf-8').lower()).first()
     if bool(user and check_password_hash(str(user.password), password)):
         return True

cps/remotelogin.py

@@ -126,11 +126,11 @@ def token_verified():
         login_user(user)
 
         ub.session.delete(auth_token)
-        ub.session_commit("User {} logged in via remotelogin, token deleted".format(user.nickname))
+        ub.session_commit("User {} logged in via remotelogin, token deleted".format(user.name))
 
         data['status'] = 'success'
         log.debug(u"Remote Login for userid %s succeded", user.id)
-        flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.nickname), category="success")
+        flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.name), category="success")
 
     response = make_response(json.dumps(data, ensure_ascii=False))
     response.headers["Content-Type"] = "application/json; charset=utf-8"

cps/render_template.py

@@ -42,10 +42,16 @@ def get_sidebar_config(kwargs=None):
     sidebar.append({"glyph": "glyphicon-fire", "text": _('Hot Books'), "link": 'web.books_list', "id": "hot",
                     "visibility": constants.SIDEBAR_HOT, 'public': True, "page": "hot",
                     "show_text": _('Show Hot Books'), "config_show": True})
-    sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.books_list',
+    if current_user.role_admin():
-                    "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+        sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.download_list',
-                    "page": "download", "show_text": _('Show Downloaded Books'),
+                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
-                    "config_show": content})
+                        "page": "download", "show_text": _('Show Downloaded Books'),
+                        "config_show": content})
+    else:
+        sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.books_list',
+                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+                        "page": "download", "show_text": _('Show Downloaded Books'),
+                        "config_show": content})
     sidebar.append(
         {"glyph": "glyphicon-star", "text": _('Top Rated Books'), "link": 'web.books_list', "id": "rated",
          "visibility": constants.SIDEBAR_BEST_RATED, 'public': True, "page": "rated",

cps/services/__init__.py

@@ -45,3 +45,9 @@ except ImportError as err:
     log.debug("Cannot import SyncToken, syncing books with Kobo Devices will not work: %s", err)
     kobo = None
     SyncToken = None
+
+try:
+    from . import gmail
+except ImportError as err:
+    log.debug("Cannot import Gmail, sending books via G-Mail Accounts will not work: %s", err)
+    gmail = None

cps/services/gmail.py

@@ -0,0 +1,80 @@
+from __future__ import print_function
+import os.path
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
+from google.oauth2.credentials import Credentials
+
+from datetime import datetime
+import base64
+from flask_babel import gettext as _
+from ..constants import BASE_DIR
+from .. import logger
+
+
+log = logger.create()
+
+SCOPES = ['openid', 'https://www.googleapis.com/auth/gmail.send', 'https://www.googleapis.com/auth/userinfo.email']
+
+def setup_gmail(token):
+    # If there are no (valid) credentials available, let the user log in.
+    creds = None
+    if "token" in token:
+        creds = Credentials(
+            token=token['token'],
+            refresh_token=token['refresh_token'],
+            token_uri=token['token_uri'],
+            client_id=token['client_id'],
+            client_secret=token['client_secret'],
+            scopes=token['scopes'],
+        )
+        creds.expiry = datetime.fromisoformat(token['expiry'])
+
+    if not creds or not creds.valid:
+        # don't forget to dump one more time after the refresh
+        # also, some file-locking routines wouldn't be needless
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            cred_file = os.path.join(BASE_DIR, 'gmail.json')
+            if not os.path.exists(cred_file):
+                raise Exception(_("Found no valid gmail.json file with OAuth information"))
+            flow = InstalledAppFlow.from_client_secrets_file(
+                os.path.join(BASE_DIR, 'gmail.json'), SCOPES)
+            creds = flow.run_local_server(port=0)
+            user_info = get_user_info(creds)
+        return {
+            'token': creds.token,
+            'refresh_token': creds.refresh_token,
+            'token_uri': creds.token_uri,
+            'client_id': creds.client_id,
+            'client_secret': creds.client_secret,
+            'scopes': creds.scopes,
+            'expiry': creds.expiry.isoformat(),
+            'email': user_info
+        }
+
+def get_user_info(credentials):
+    user_info_service = build(serviceName='oauth2', version='v2',credentials=credentials)
+    user_info = user_info_service.userinfo().get().execute()
+    return user_info.get('email', "")
+
+def send_messsage(token, msg):
+    creds = Credentials(
+        token=token['token'],
+        refresh_token=token['refresh_token'],
+        token_uri=token['token_uri'],
+        client_id=token['client_id'],
+        client_secret=token['client_secret'],
+        scopes=token['scopes'],
+    )
+    creds.expiry = datetime.fromisoformat(token['expiry'])
+    if creds and creds.expired and creds.refresh_token:
+        creds.refresh(Request())
+    service = build('gmail', 'v1', credentials=creds)
+    message_as_bytes = msg.as_bytes()  # the message should converted from string to bytes.
+    message_as_base64 = base64.urlsafe_b64encode(message_as_bytes)  # encode in base64 (printable letters coding)
+    raw = message_as_base64.decode()  # convert to something  JSON serializable
+    body = {'raw': raw}
+
+    (service.users().messages().send(userId='me', body=body).execute())

cps/services/worker.py

@@ -159,9 +159,9 @@ class CalibreTask:
         # catch any unhandled exceptions in a task and automatically fail it
         try:
             self.run(*args)
-        except Exception as e:
+        except Exception as ex:
-            self._handleError(str(e))
+            self._handleError(str(ex))
-            log.debug_or_exception(e)
+            log.debug_or_exception(ex)
 
         self.end_time = datetime.now()
 

cps/shelf.py

@@ -255,13 +255,13 @@ def create_edit_shelf(shelf, title, page, shelf_id=False):
                 log.info(u"Shelf {} {}".format(to_save["title"], shelf_action))
                 flash(flash_text, category="success")
                 return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id))
-            except (OperationalError, InvalidRequestError) as e:
+            except (OperationalError, InvalidRequestError) as ex:
                 ub.session.rollback()
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
                 flash(_(u"Settings DB is not Writeable"), category="error")
-            except Exception as e:
+            except Exception as ex:
                 ub.session.rollback()
-                log.debug_or_exception(e)
+                log.debug_or_exception(ex)
                 flash(_(u"There was an error"), category="error")
     return render_title_template('shelf_edit.html', shelf=shelf, title=title, page=page)
 

cps/static/css/style.css

@@ -56,6 +56,7 @@ a,
 .book-remove,
 .editable-empty,
 .editable-empty:hover { color: #45b29d; }
+.book-remove:hover { color: #23527c; }
 .user-remove:hover { color: #23527c; }
 .btn-default a { color: #444; }
 .panel-title > a { text-decoration: none; }

cps/static/js/filter_grid.js

@@ -15,6 +15,8 @@
  *  along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+var direction = $("#asc").data('order');  // 0=Descending order; 1= ascending order
+
 var $list = $("#list").isotope({
     itemSelector: ".book",
     layoutMode: "fitRows",
@@ -24,6 +26,9 @@ var $list = $("#list").isotope({
 });
 
 $("#desc").click(function() {
+    if (direction === 0) {
+        return;
+    }
     var page = $(this).data("id");
     $.ajax({
         method:"post",
@@ -39,6 +44,9 @@ $("#desc").click(function() {
 });
 
 $("#asc").click(function() {
+    if (direction === 1) {
+        return;
+    }
     var page = $(this).data("id");
     $.ajax({
         method:"post",

cps/static/js/filter_list.js

@@ -15,7 +15,7 @@
  *  along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-var direction = 0;  // Descending order
+var direction = $("#asc").data('order');  // 0=Descending order; 1= ascending order
 var sort = 0;       // Show sorted entries
 
 $("#sort_name").click(function() {

cps/static/js/main.js

@@ -114,26 +114,23 @@ $(document).ready(function() {
   }
 });
 
-function confirmDialog(id, dataValue, yesFn, noFn) {
+function confirmDialog(id, dialogid, dataValue, yesFn, noFn) {
-    var $confirm = $("#GeneralDeleteModal");
+    var $confirm = $("#" + dialogid);
-    // var dataValue= e.data('value'); // target.data('value');
     $confirm.modal('show');
     $.ajax({
         method:"get",
         dataType: "json",
         url: getPath() + "/ajax/loaddialogtexts/" + id,
         success: function success(data) {
-            $("#header").html(data.header);
+            $("#header-"+ dialogid).html(data.header);
-            $("#text").html(data.main);
+            $("#text-"+ dialogid).html(data.main);
         }
     });
-
+    $("#btnConfirmYes-"+ dialogid).off('click').click(function () {
-
-    $("#btnConfirmYes").off('click').click(function () {
         yesFn(dataValue);
         $confirm.modal("hide");
     });
-    $("#btnConfirmNo").off('click').click(function () {
+    $("#btnConfirmNo-"+ dialogid).off('click').click(function () {
         if (typeof noFn !== 'undefined') {
             noFn(dataValue);
         }
@@ -485,6 +482,7 @@ $(function() {
     $("#config_delete_kobo_token").click(function() {
         confirmDialog(
             $(this).attr('id'),
+            "GeneralDeleteModal",
             $(this).data('value'),
             function (value) {
                 $.ajax({
@@ -513,6 +511,7 @@ $(function() {
     $("#btndeluser").click(function() {
         confirmDialog(
             $(this).attr('id'),
+            "GeneralDeleteModal",
             $(this).data('value'),
             function(value){
                 var subform = $('#user_submit').closest("form");
@@ -531,6 +530,7 @@ $(function() {
     $("#delete_shelf").click(function() {
         confirmDialog(
             $(this).attr('id'),
+            "GeneralDeleteModal",
             $(this).data('value'),
             function(value){
                 window.location.href = window.location.pathname + "/../../shelf/delete/" + value

cps/static/js/table.js

@@ -94,14 +94,23 @@ $(function() {
                 editable: {
                     mode: "inline",
                     emptytext: "<span class='glyphicon glyphicon-plus'></span>",
+                    success: function (response, __) {
+                        if (!response.success) return response.msg;
+                        return {newValue: response.newValue};
+                    },
+                    params: function (params) {
+                        params.checkA = $('#autoupdate_authorsort').prop('checked');
+                        params.checkT = $('#autoupdate_titlesort').prop('checked');
+                        return params
+                    }
                 }
             };
-        }
+            var validateText = $(this).attr("data-edit-validate");
-        var validateText = $(this).attr("data-edit-validate");
+            if (validateText) {
-        if (validateText) {
+                element.editable.validate = function (value) {
-            element.editable.validate = function (value) {
+                    if ($.trim(value) === "") return validateText;
-                if ($.trim(value) === "") return validateText;
+                };
-            };
+            }
         }
         column.push(element);
     });
@@ -117,7 +126,7 @@ $(function() {
         search: true,
         showColumns: true,
         searchAlign: "left",
-        showSearchButton : false,
+        showSearchButton : true,
         searchOnEnterKey: true,
         checkboxHeader: false,
         maintainMetaData: true,
@@ -128,7 +137,8 @@ $(function() {
         },
         // eslint-disable-next-line no-unused-vars
         onEditableSave: function (field, row, oldvalue, $el) {
-            if (field === "title" || field === "authors") {
+            if ($.inArray(field, [ "title", "sort" ]) !== -1 && $('#autoupdate_titlesort').prop('checked')
+                || $.inArray(field, [ "authors", "author_sort" ]) !== -1 && $('#autoupdate_authorsort').prop('checked')) {
                 $.ajax({
                     method:"get",
                     dataType: "json",
@@ -236,16 +246,16 @@ $(function() {
     }
     $("#domain-allow-table").on("click-cell.bs.table", function (field, value, row, $element) {
         if (value === 2) {
-            confirmDialog("btndeletedomain", $element.id, domainHandle);
+            confirmDialog("btndeletedomain", "GeneralDeleteModal", $element.id, domainHandle);
         }
     });
     $("#domain-deny-table").on("click-cell.bs.table", function (field, value, row, $element) {
         if (value === 2) {
-            confirmDialog("btndeletedomain", $element.id, domainHandle);
+            confirmDialog("btndeletedomain", "GeneralDeleteModal", $element.id, domainHandle);
         }
     });
 
-    $("#restrictModal").on("hidden.bs.modal", function () {
+    $("#restrictModal").on("hidden.bs.modal", function (e) {
         // Destroy table and remove hooks for buttons
         $("#restrict-elements-table").unbind();
         $("#restrict-elements-table").bootstrapTable("destroy");
@@ -254,8 +264,54 @@ $(function() {
         $("#h2").addClass("hidden");
         $("#h3").addClass("hidden");
         $("#h4").addClass("hidden");
+        $("#add_element").val("");
     });
-    function startTable(type, userId) {
+
+    function startTable(target, userId) {
+        var type = 0;
+        switch(target) {
+            case "get_column_values":
+                type = 1;
+                $("#h2").removeClass("hidden");
+                break;
+            case "get_tags":
+                type = 0;
+                $("#h1").removeClass("hidden");
+                break;
+            case "get_user_column_values":
+                type = 3;
+                $("#h4").removeClass("hidden");
+                break;
+            case "get_user_tags":
+                type = 2;
+                $("#h3").removeClass("hidden");
+                break;
+            case "denied_tags":
+                type = 2;
+                $("#h2").removeClass("hidden");
+                $("#submit_allow").addClass("hidden");
+                $("#submit_restrict").removeClass("hidden");
+                break;
+            case "allowed_tags":
+                type = 2;
+                $("#h2").removeClass("hidden");
+                $("#submit_restrict").addClass("hidden");
+                $("#submit_allow").removeClass("hidden");
+                break;
+            case "allowed_column_value":
+                type = 3;
+                $("#h2").removeClass("hidden");
+                $("#submit_restrict").addClass("hidden");
+                $("#submit_allow").removeClass("hidden");
+                break;
+            case "denied_column_value":
+                type = 3;
+                $("#h2").removeClass("hidden");
+                $("#submit_allow").addClass("hidden");
+                $("#submit_restrict").removeClass("hidden");
+                break;
+        }
+
         $("#restrict-elements-table").bootstrapTable({
             formatNoMatches: function () {
                 return "";
@@ -269,6 +325,10 @@ $(function() {
                     return {classes: "bg-dark-danger"};
                 }
             },
+            onLoadSuccess: function () {
+                $(".no-records-found").addClass("hidden");
+                $(".fixed-table-loading").addClass("hidden");
+            },
             onClickCell: function (field, value, row) {
                 if (field === 3) {
                     $.ajax ({
@@ -322,28 +382,192 @@ $(function() {
             return;
         });
     }
-    $("#get_column_values").on("click", function() {
+
-        startTable(1, 0);
+    $("#restrictModal").on("show.bs.modal", function(e) {
-        $("#h2").removeClass("hidden");
+         var target = $(e.relatedTarget).attr('id');
+         var dataId;
+         $(e.relatedTarget).one('focus', function(e){$(this).blur();});
+         //$(e.relatedTarget).blur();
+         if ($(e.relatedTarget).hasClass("button_head")) {
+             dataId = $('#user-table').bootstrapTable('getSelections').map(a => a.id);
+         } else {
+             dataId = $(e.relatedTarget).data("id");
+         }
+         startTable(target, dataId);
     });
 
-    $("#get_tags").on("click", function() {
+    // User table handling
-        startTable(0, 0);
+    var user_column = [];
-        $("#h1").removeClass("hidden");
+    $("#user-table > thead > tr > th").each(function() {
-    });
+        var element = {};
-    $("#get_user_column_values").on("click", function() {
+        if ($(this).attr("data-edit")) {
-        startTable(3, $(this).data("id"));
+            element = {
-        $("#h4").removeClass("hidden");
+                editable: {
+                    mode: "inline",
+                    emptytext: "<span class='glyphicon glyphicon-plus'></span>",
+                    error: function(response) {
+                        return response.responseText;
+                    }
+                }
+            };
+        }
+        var validateText = $(this).attr("data-edit-validate");
+        if (validateText) {
+            element.editable.validate = function (value) {
+                if ($.trim(value) === "") return validateText;
+            };
+        }
+        user_column.push(element);
     });
 
-    $("#get_user_tags").on("click", function() {
+    $("#user-table").bootstrapTable({
-        startTable(2,  $(this).data("id"));
+        sidePagination: "server",
-        $(this)[0].blur();
+        pagination: true,
-        $("#h3").removeClass("hidden");
+        paginationLoop: false,
+        paginationDetailHAlign: " hidden",
+        paginationHAlign: "left",
+        idField: "id",
+        uniqueId: "id",
+        search: true,
+        showColumns: true,
+        searchAlign: "left",
+        showSearchButton : true,
+        searchOnEnterKey: true,
+        checkboxHeader: true,
+        maintainMetaData: true,
+        responseHandler: responseHandler,
+        columns: user_column,
+        formatNoMatches: function () {
+            return "";
+        },
+        onPostBody () {
+            // Remove all checkboxes from Headers for showing the texts in the column selector
+            $('.columns [data-field]').each(function(){
+                var elText = $(this).next().text();
+                $(this).next().empty();
+                var index = elText.lastIndexOf('\n', elText.length - 2);
+                if ( index > -1) {
+                    elText = elText.substr(index);
+                }
+                $(this).next().text(elText);
+            });
+        },
+        onLoadSuccess: function () {
+            var guest = $(".editable[data-name='name'][data-value='Guest']");
+            guest.editable("disable");
+            $(".editable[data-name='locale'][data-pk='"+guest.data("pk")+"']").editable("disable");
+            $("input[data-name='admin_role'][data-pk='"+guest.data("pk")+"']").prop("disabled", true);
+            $("input[data-name='passwd_role'][data-pk='"+guest.data("pk")+"']").prop("disabled", true);
+            $("input[data-name='edit_shelf_role'][data-pk='"+guest.data("pk")+"']").prop("disabled", true);
+            // ToDo: Disable delete
+
+        },
+
+        // eslint-disable-next-line no-unused-vars
+        /*onEditableSave: function (field, row, oldvalue, $el) {
+            if (field === "title" || field === "authors") {
+                $.ajax({
+                    method:"get",
+                    dataType: "json",
+                    url: window.location.pathname + "/../../ajax/sort_value/" + field + "/" + row.id,
+                    success: function success(data) {
+                        var key = Object.keys(data)[0];
+                        $("#books-table").bootstrapTable("updateCellByUniqueId", {
+                            id: row.id,
+                            field: key,
+                            value: data[key]
+                        });
+                        // console.log(data);
+                    }
+                });
+            }
+        },*/
+        // eslint-disable-next-line no-unused-vars
+        onColumnSwitch: function (field, checked) {
+            var visible = $("#user-table").bootstrapTable("getVisibleColumns");
+            var hidden  = $("#user-table").bootstrapTable("getHiddenColumns");
+            var st = "";
+            visible.forEach(function(item) {
+                st += "\"" + item.name + "\":\"" + "true" + "\",";
+            });
+            hidden.forEach(function(item) {
+                st += "\"" + item.name + "\":\"" + "false" + "\",";
+            });
+            st = st.slice(0, -1);
+            $.ajax({
+                method:"post",
+                contentType: "application/json; charset=utf-8",
+                dataType: "json",
+                url: window.location.pathname + "/../../ajax/user_table_settings",
+                data: "{" + st + "}",
+            });
+        },
     });
 
+    $("#user_delete_selection").click(function() {
+        $("#user-table").bootstrapTable("uncheckAll");
+    });
+
+    function user_handle (userId) {
+        $.ajax({
+            method:"post",
+            url: window.location.pathname + "/../../ajax/deleteuser",
+            data: {"userid":userId}
+        });
+        $.ajax({
+            method:"get",
+            url: window.location.pathname + "/../../ajax/listusers",
+            async: true,
+            timeout: 900,
+            success:function(data) {
+                $("#user-table").bootstrapTable("load", data);
+            }
+        });
+    }
+
+
+    $("#user-table").on("click-cell.bs.table", function (field, value, row, $element) {
+        if (value === "denied_column_value") {
+            ConfirmDialog("btndeluser", "GeneralDeleteModal", $element.id, user_handle);
+        }
+    });
+
+    $("#user-table").on("check.bs.table check-all.bs.table uncheck.bs.table uncheck-all.bs.table",
+    function (e, rowsAfter, rowsBefore) {
+        var rows = rowsAfter;
+
+        if (e.type === "uncheck-all") {
+            rows = rowsBefore;
+        }
+
+        var ids = $.map(!$.isArray(rows) ? [rows] : rows, function (row) {
+            return row.id;
+        });
+        var func = $.inArray(e.type, ["check", "check-all"]) > -1 ? "union" : "difference";
+        selections = window._[func](selections, ids);
+        if (selections.length < 1) {
+            $("#user_delete_selection").addClass("disabled");
+            $("#user_delete_selection").attr("aria-disabled", true);
+            $(".check_head").attr("aria-disabled", true);
+            $(".check_head").attr("disabled", true);
+            $(".check_head").prop('checked', false);
+            $(".button_head").attr("aria-disabled", true);
+            $(".button_head").addClass("disabled");
+            $(".header_select").attr("disabled", true);
+        } else {
+            $("#user_delete_selection").removeClass("disabled");
+            $("#user_delete_selection").attr("aria-disabled", false);
+            $(".check_head").attr("aria-disabled", false);
+            $(".check_head").removeAttr("disabled");
+            $(".button_head").attr("aria-disabled", false);
+            $(".button_head").removeClass("disabled");
+            $(".header_select").removeAttr("disabled");
+        }
+
+    });
 });
 
+
 /* Function for deleting domain restrictions */
 function TableActions (value, row) {
     return [
@@ -354,7 +578,10 @@ function TableActions (value, row) {
     ].join("");
 }
 
-
+function editEntry(param)
+{
+    console.log(param);
+}
 /* Function for deleting domain restrictions */
 function RestrictionActions (value, row) {
     return [
@@ -373,6 +600,15 @@ function EbookActions (value, row) {
     ].join("");
 }
 
+/* Function for deleting books */
+function UserActions (value, row) {
+    return [
+        "<div class=\"user-remove\" data-target=\"#GeneralDeleteModal\" title=\"Remove\">",
+        "<i class=\"glyphicon glyphicon-trash\"></i>",
+        "</div>"
+    ].join("");
+}
+
 /* Function for keeping checked rows */
 function responseHandler(res) {
     $.each(res.rows, function (i, row) {
@@ -380,3 +616,122 @@ function responseHandler(res) {
     });
     return res;
 }
+
+function singleUserFormatter(value, row) {
+    return '<a class="btn btn-default" href="/admin/user/' + row.id + '">' + this.buttontext + '</a>'
+}
+
+function checkboxFormatter(value, row, index){
+    if(value & this.column)
+        return '<input type="checkbox" class="chk" data-pk="' + row.id + '" data-name="' + this.name + '" checked onchange="checkboxChange(this, ' + row.id + ', \'' + this.field + '\', ' + this.column + ')">';
+    else
+        return '<input type="checkbox" class="chk" data-pk="' + row.id + '" data-name="' + this.name + '" onchange="checkboxChange(this, ' + row.id + ', \'' + this.field + '\', ' + this.column + ')">';
+}
+
+function checkboxChange(checkbox, userId, field, field_index) {
+    $.ajax({
+        method:"post",
+        url: window.location.pathname + "/../../ajax/editlistusers/" + field,
+        data: {"pk":userId, "field_index":field_index, "value": checkbox.checked}
+        /*<div className="editable-buttons">
+            <button type="button" className="btn btn-default btn-sm editable-cancel"><i
+                className="glyphicon glyphicon-remove"></i></button>
+        </div>*/
+        /*<div className="editable-error-block help-block" style="">Text to show</div>*/
+    });
+    $.ajax({
+        method:"get",
+        url: window.location.pathname + "/../../ajax/listusers",
+        async: true,
+        timeout: 900,
+        success:function(data) {
+            $("#user-table").bootstrapTable("load", data);
+        }
+    });
+}
+function deactivateHeaderButtons(e) {
+    $("#user_delete_selection").addClass("disabled");
+    $("#user_delete_selection").attr("aria-disabled", true);
+    $(".check_head").attr("aria-disabled", true);
+    $(".check_head").attr("disabled", true);
+    $(".check_head").prop('checked', false);
+    $(".button_head").attr("aria-disabled", true);
+    $(".button_head").addClass("disabled");
+    $(".header_select").attr("disabled", true);
+}
+
+function selectHeader(element, field) {
+    if (element.value !== "None") {
+        confirmDialog(element.id, "GeneralChangeModal", 0, function () {
+            var result = $('#user-table').bootstrapTable('getSelections').map(a => a.id);
+            $.ajax({
+                method: "post",
+                url: window.location.pathname + "/../../ajax/editlistusers/" + field,
+                data: {"pk": result, "value": element.value},
+                success: function () {
+                    $.ajax({
+                        method: "get",
+                        url: window.location.pathname + "/../../ajax/listusers",
+                        async: true,
+                        timeout: 900,
+                        success: function (data) {
+                            $("#user-table").bootstrapTable("load", data);
+                            deactivateHeaderButtons();
+                        }
+                    });
+                }
+            });
+        });
+    }
+}
+
+function checkboxHeader(CheckboxState, field, field_index) {
+    confirmDialog(field, "GeneralChangeModal", 0, function() {
+        var result = $('#user-table').bootstrapTable('getSelections').map(a => a.id);
+        $.ajax({
+            method: "post",
+            url: window.location.pathname + "/../../ajax/editlistusers/" + field,
+            data: {"pk": result, "field_index": field_index, "value": CheckboxState},
+            success: function () {
+                $.ajax({
+                    method: "get",
+                    url: window.location.pathname + "/../../ajax/listusers",
+                    async: true,
+                    timeout: 900,
+                    success: function (data) {
+                        $("#user-table").bootstrapTable("load", data);
+                        $("#user_delete_selection").addClass("disabled");
+                        $("#user_delete_selection").attr("aria-disabled", true);
+                        $(".check_head").attr("aria-disabled", true);
+                        $(".check_head").attr("disabled", true);
+                        $(".check_head").prop('checked', false);
+                        $(".button_head").attr("aria-disabled", true);
+                        $(".button_head").addClass("disabled");
+                        $(".header_select").attr("disabled", true);
+                    }
+                });
+            }
+        });
+    });
+}
+
+function user_handle (userId) {
+    $.ajax({
+        method:"post",
+        url: window.location.pathname + "/../../ajax/deleteuser",
+        data: {"userid":userId}
+    });
+    $.ajax({
+        method:"get",
+        url: window.location.pathname + "/../../ajax/listusers",
+        async: true,
+        timeout: 900,
+        success:function(data) {
+            $("#user-table").bootstrapTable("load", data);
+        }
+    });
+}
+
+function test(){
+    console.log("hello");
+}

cps/tasks/convert.py

@@ -75,8 +75,8 @@ class TaskConvert(CalibreTask):
                                                            self.settings['body'],
                                                            internal=True)
                                       )
-                except Exception as e:
+                except Exception as ex:
-                    return self._handleError(str(e))
+                    return self._handleError(str(ex))
 
     def _convert_ebook_format(self):
         error_message = None

cps/tasks/mail.py

@@ -4,6 +4,8 @@ import os
 import smtplib
 import threading
 import socket
+import mimetypes
+import base64
 
 try:
     from StringIO import StringIO
@@ -16,11 +18,14 @@ except ImportError:
     from email.mime.multipart import MIMEMultipart
     from email.mime.text import MIMEText
 
+
+
 from email import encoders
 from email.utils import formatdate, make_msgid
 from email.generator import Generator
 
 from cps.services.worker import CalibreTask
+from cps.services import gmail
 from cps import logger, config
 
 from cps import gdriveutils
@@ -107,68 +112,38 @@ class TaskEmail(CalibreTask):
         self.recipent = recipient
         self.text = text
         self.asyncSMTP = None
-
         self.results = dict()
 
-    def run(self, worker_thread):
+    def prepare_message(self):
-        # create MIME message
+        message = MIMEMultipart()
-        msg = MIMEMultipart()
+        message['to'] = self.recipent
-        msg['Subject'] = self.subject
+        message['from'] = self.settings["mail_from"]
-        msg['Message-Id'] = make_msgid('calibre-web')
+        message['subject'] = self.subject
-        msg['Date'] = formatdate(localtime=True)
+        message['Message-Id'] = make_msgid('calibre-web')
+        message['Date'] = formatdate(localtime=True)
         text = self.text
-        msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
+        msg = MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8')
+        message.attach(msg)
         if self.attachment:
             result = self._get_attachment(self.filepath, self.attachment)
             if result:
-                msg.attach(result)
+                message.attach(result)
             else:
                 self._handleError(u"Attachment not found")
                 return
+        return message
 
-        msg['From'] = self.settings["mail_from"]
+    def run(self, worker_thread):
-        msg['To'] = self.recipent
+        # create MIME message
-
+        msg = self.prepare_message()
-        use_ssl = int(self.settings.get('mail_use_ssl', 0))
         try:
-            # convert MIME message to string
+            if self.settings['mail_server_type'] == 0:
-            fp = StringIO()
+                self.send_standard_email(msg)
-            gen = Generator(fp, mangle_from_=False)
-            gen.flatten(msg)
-            msg = fp.getvalue()
-
-            # send email
-            timeout = 600  # set timeout to 5mins
-
-            # redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
-            # _print_debug function
-            if sys.version_info < (3, 0):
-                org_smtpstderr = smtplib.stderr
-                smtplib.stderr = logger.StderrLogger('worker.smtp')
-
-            if use_ssl == 2:
-                self.asyncSMTP = EmailSSL(self.settings["mail_server"], self.settings["mail_port"],
-                                           timeout=timeout)
             else:
-                self.asyncSMTP = Email(self.settings["mail_server"], self.settings["mail_port"], timeout=timeout)
+                self.send_gmail_email(msg)
-
+        except MemoryError as e:
-            # link to logginglevel
-            if logger.is_debug_enabled():
-                self.asyncSMTP.set_debuglevel(1)
-            if use_ssl == 1:
-                self.asyncSMTP.starttls()
-            if self.settings["mail_password"]:
-                self.asyncSMTP.login(str(self.settings["mail_login"]), str(self.settings["mail_password"]))
-            self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipent, msg)
-            self.asyncSMTP.quit()
-            self._handleSuccess()
-
-            if sys.version_info < (3, 0):
-                smtplib.stderr = org_smtpstderr
-
-        except (MemoryError) as e:
             log.debug_or_exception(e)
-            self._handleError(u'MemoryError sending email: ' + str(e))
+            self._handleError(u'MemoryError sending email: {}'.format(str(e)))
         except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
             log.debug_or_exception(e)
             if hasattr(e, "smtp_error"):
@@ -179,12 +154,55 @@ class TaskEmail(CalibreTask):
                 text = '\n'.join(e.args)
             else:
                 text = ''
-            self._handleError(u'Smtplib Error sending email: ' + text)
+            self._handleError(u'Smtplib Error sending email: {}'.format(text))
-        except (socket.error) as e:
+        except socket.error as e:
             log.debug_or_exception(e)
-            self._handleError(u'Socket Error sending email: ' + e.strerror)
+            self._handleError(u'Socket Error sending email: {}'.format(e.strerror))
+        except Exception as ex:
+            log.debug_or_exception(ex)
+            self._handleError(u'Error sending email: {}'.format(ex))
 
 
+    def send_standard_email(self, msg):
+        use_ssl = int(self.settings.get('mail_use_ssl', 0))
+        timeout = 600  # set timeout to 5mins
+
+        # redirect output to logfile on python2 on python3 debugoutput is caught with overwritten
+        # _print_debug function
+        if sys.version_info < (3, 0):
+            org_smtpstderr = smtplib.stderr
+            smtplib.stderr = logger.StderrLogger('worker.smtp')
+
+        if use_ssl == 2:
+            self.asyncSMTP = EmailSSL(self.settings["mail_server"], self.settings["mail_port"],
+                                       timeout=timeout)
+        else:
+            self.asyncSMTP = Email(self.settings["mail_server"], self.settings["mail_port"], timeout=timeout)
+
+        # link to logginglevel
+        if logger.is_debug_enabled():
+            self.asyncSMTP.set_debuglevel(1)
+        if use_ssl == 1:
+            self.asyncSMTP.starttls()
+        if self.settings["mail_password"]:
+            self.asyncSMTP.login(str(self.settings["mail_login"]), str(self.settings["mail_password"]))
+
+        # Convert message to something to send
+        fp = StringIO()
+        gen = Generator(fp, mangle_from_=False)
+        gen.flatten(msg)
+
+        self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipent, fp.getvalue())
+        self.asyncSMTP.quit()
+        self._handleSuccess()
+
+        if sys.version_info < (3, 0):
+            smtplib.stderr = org_smtpstderr
+
+
+    def send_gmail_email(self, message):
+        return gmail.send_messsage(self.settings.get('mail_gmail_token', None), message)
+
     @property
     def progress(self):
         if self.asyncSMTP is not None:
@@ -203,13 +221,13 @@ class TaskEmail(CalibreTask):
     @classmethod
     def _get_attachment(cls, bookpath, filename):
         """Get file as MIMEBase message"""
-        calibrepath = config.config_calibre_dir
+        calibre_path = config.config_calibre_dir
         if config.config_use_google_drive:
             df = gdriveutils.getFileFromEbooksFolder(bookpath, filename)
             if df:
-                datafile = os.path.join(calibrepath, bookpath, filename)
+                datafile = os.path.join(calibre_path, bookpath, filename)
-                if not os.path.exists(os.path.join(calibrepath, bookpath)):
+                if not os.path.exists(os.path.join(calibre_path, bookpath)):
-                    os.makedirs(os.path.join(calibrepath, bookpath))
+                    os.makedirs(os.path.join(calibre_path, bookpath))
                 df.GetContentFile(datafile)
             else:
                 return None
@@ -219,19 +237,22 @@ class TaskEmail(CalibreTask):
             os.remove(datafile)
         else:
             try:
-                file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
+                file_ = open(os.path.join(calibre_path, bookpath, filename), 'rb')
                 data = file_.read()
                 file_.close()
             except IOError as e:
                 log.debug_or_exception(e)
                 log.error(u'The requested file could not be read. Maybe wrong permissions?')
                 return None
-
+        # Set mimetype
-        attachment = MIMEBase('application', 'octet-stream')
+        content_type, encoding = mimetypes.guess_type(filename)
+        if content_type is None or encoding is not None:
+            content_type = 'application/octet-stream'
+        main_type, sub_type = content_type.split('/', 1)
+        attachment = MIMEBase(main_type, sub_type)
         attachment.set_payload(data)
         encoders.encode_base64(attachment)
-        attachment.add_header('Content-Disposition', 'attachment',
+        attachment.add_header('Content-Disposition', 'attachment', filename=filename)
-                              filename=filename)
         return attachment
 
     @property

cps/templates/admin.html

@@ -7,6 +7,7 @@
   <div class="row">
     <div class="col">
       <h2>{{_('Users')}}</h2>
+    {% if allUser.__len__() < 10 %}
       <table class="table table-striped" id="table_user">
         <tr>
             <th>{{_('Username')}}</th>
@@ -25,7 +26,7 @@
         {% for user in allUser %}
           {% if not user.role_anonymous() or config.config_anonbrowse %}
           <tr>
-            <td><a href="{{url_for('admin.edit_user', user_id=user.id)}}">{{user.nickname}}</a></td>
+            <td><a href="{{url_for('admin.edit_user', user_id=user.id)}}">{{user.name}}</a></td>
             <td>{{user.email}}</td>
             <td>{{user.kindle_mail}}</td>
             <td>{{user.downloads.count()}}</td>
@@ -41,6 +42,8 @@
           {% endif %}
         {% endfor %}
       </table>
+    {% endif %}
+        <a class="btn btn-default" id="admin_user_table" href="{{url_for('admin.edit_user_table')}}">{{_('Edit Users')}}</a>
         <a class="btn btn-default" id="admin_new_user" href="{{url_for('admin.new_user')}}">{{_('Add New User')}}</a>
       {% if (config.config_login_type == 1) %}
         <div class="btn btn-default" id="import_ldap_users" data-toggle="modal" data-target="#StatusDialog">{{_('Import LDAP Users')}}</div>
@@ -52,29 +55,42 @@
     <div class="col">
       <h2>{{_('E-mail Server Settings')}}</h2>
       {% if config.get_mail_server_configured() %}
-      <div class="col-xs-12 col-sm-12">
+        {% if email.mail_server_type == 0 %}
-        <div class="row">
+        <div class="col-xs-12 col-sm-12">
+          <div class="row">
             <div class="col-xs-6 col-sm-3">{{_('SMTP Hostname')}}</div>
             <div class="col-xs-6 col-sm-3">{{email.mail_server}}</div>
-        </div>
+          </div>
-        <div class="row">
+          <div class="row">
             <div class="col-xs-6 col-sm-3">{{_('SMTP Port')}}</div>
             <div class="col-xs-6 col-sm-3">{{email.mail_port}}</div>
-        </div>
+          </div>
-        <div class="row">
+          <div class="row">
             <div class="col-xs-6 col-sm-3">{{_('Encryption')}}</div>
             <div class="col-xs-6 col-sm-3">{{ display_bool_setting(email.mail_use_ssl) }}</div>
+          </div>
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('SMTP Login')}}</div>
+            <div class="col-xs-6 col-sm-3">{{email.mail_login}}</div>
+          </div>
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('From E-mail')}}</div>
+            <div class="col-xs-6 col-sm-3">{{email.mail_from}}</div>
+          </div>
         </div>
-        <div class="row">
+      {% else %}
-          <div class="col-xs-6 col-sm-3">{{_('SMTP Login')}}</div>
+        <div class="col-xs-12 col-sm-12">
-          <div class="col-xs-6 col-sm-3">{{email.mail_login}}</div>
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('E-Mail Service')}}</div>
+            <div class="col-xs-6 col-sm-3">{{_('Gmail via Oauth2')}}</div>
+          </div>
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('From E-mail')}}</div>
+            <div class="col-xs-6 col-sm-3">{{email.mail_gmail_token['email']}}</div>
+          </div>
         </div>
-        <div class="row">
-          <div class="col-xs-6 col-sm-3">{{_('From E-mail')}}</div>
-          <div class="col-xs-6 col-sm-3">{{email.mail_from}}</div>
-        </div>
-      </div>
       {% endif %}
+    {% endif %}
       <a class="btn btn-default emailconfig" id="admin_edit_email" href="{{url_for('admin.edit_mailsettings')}}">{{_('Edit E-mail Server Settings')}}</a>
     </div>
   </div>

cps/templates/book_table.html

@@ -30,8 +30,8 @@
         <label for="autoupdate_titlesort">{{_('Update Title Sort automatically')}}</label>
       </div>
       <div class="row">
-        <input type="checkbox" id="autoupdate_autorsort" name="autoupdate_autorsort" checked>
+        <input type="checkbox" id="autoupdate_authorsort" name="autoupdate_authorsort" checked>
-        <label for="autoupdate_autorsort">{{_('Update Author Sort automatically')}}</label>
+        <label for="autoupdate_authorsort">{{_('Update Author Sort automatically')}}</label>
       </div>
     </div>
 
@@ -53,7 +53,7 @@
             {{ text_table_row('languages', _('Enter Languages'),_('Languages'), false) }}
             <!--th data-field="pubdate" data-type="date" data-visible="{{visiblility.get('pubdate')}}" data-viewformat="dd.mm.yyyy" id="pubdate" data-sortable="true">{{_('Publishing Date')}}</th-->
             {{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false) }}
-          {% if g.user.role_edit() %}
+          {% if g.user.role_delete_books() and g.user.role_edit()%}
             <th data-align="right" data-formatter="EbookActions" data-switchable="false">{{_('Delete')}}</th>
           {% endif %}
         </tr>
@@ -94,6 +94,4 @@
 <script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-table-editable.min.js') }}"></script>
 <script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-editable.min.js') }}"></script>
 <script src="{{ url_for('static', filename='js/table.js') }}"></script>
-<script>
-</script>
 {% endblock %}

cps/templates/config_view_edit.html

@@ -141,8 +141,8 @@
             <input type="checkbox" name="Show_detail_random" id="Show_detail_random" {% if conf.show_detail_random() %}checked{% endif %}>
             <label for="Show_detail_random">{{_('Show Random Books in Detail View')}}</label>
         </div>
-        <a href="#" id="get_tags" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied Tags')}}</a>
+        <a href="#" id="get_tags" data-id="0" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied Tags')}}</a>
-        <a href="#" id="get_column_values" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied custom column values')}}</a>
+        <a href="#" id="get_column_values" data-id="0" class="btn btn-default" data-toggle="modal" data-target="#restrictModal">{{_('Add Allowed/Denied custom column values')}}</a>
       </div>
     </div>
   </div>

cps/templates/email_edit.html

@@ -7,44 +7,66 @@
 <div class="discover">
   <h1>{{title}}</h1>
   <form role="form" class="col-md-10 col-lg-6" method="POST">
+    {% if feature_support['gmail'] %}
     <div class="form-group">
-      <label for="mail_server">{{_('SMTP Hostname')}}</label>
+      <label for="mail_server_type">{{_('Choose Server Type')}}</label>
-      <input type="text" class="form-control" name="mail_server" id="mail_server" value="{{content.mail_server}}" required>
+      <select name="mail_server_type" id="config_email_type" class="form-control" data-control="email-settings">
+       <option value="0" {% if content.mail_server_type == 0 %}selected{% endif %}>{{_('Use Standard E-Mail Account')}}</option>
+       <option value="1" {% if content.mail_server_type == 1 %}selected{% endif %}>{{_('G-Mail Account with OAuth2 Verfification')}}</option>
+      </select>
     </div>
-    <div class="form-group">
+    <div data-related="email-settings-1">
-      <label for="mail_port">{{_('SMTP Port')}}</label>
+      <div class="form-group">
-      <input type="number" min="1" max="65535" step="1" class="form-control" name="mail_port" id="mail_port" value="{% if content.mail_port != None %}{{ content.mail_port }}{% endif %}" autocomplete="off" required>
+        {% if content.mail_gmail_token == {} %}
+        <button type="submit" id="gmail_server" name="gmail" value="submit" class="btn btn-default">{{_('Setup Gmail Account as E-Mail Server')}}</button>
+        {% else %}
+        <button type="submit" id="invalidate_server" name="invalidate" value="submit" class="btn btn-danger">{{_('Revoke G-Mail Access')}}</button>
+        {% endif %}
+      </div>
     </div>
-    <div class="form-group">
+    <div data-related="email-settings-0">
-      <label for="mail_use_ssl">{{_('Encryption')}}</label>
+   {% endif %}
-        <select name="mail_use_ssl" id="mail_use_ssl" class="form-control">
+      <div class="form-group">
-            <option value="0" {% if content.mail_use_ssl == 0 %}selected{% endif %}>{{ _('None') }}</option>
+        <label for="mail_server">{{_('SMTP Hostname')}}</label>
-            <option value="1" {% if content.mail_use_ssl == 1 %}selected{% endif %}>{{ _('STARTTLS') }}</option>
+        <input type="text" class="form-control" name="mail_server" id="mail_server" value="{{content.mail_server}}" required>
-            <option value="2" {% if content.mail_use_ssl == 2 %}selected{% endif %}>{{ _('SSL/TLS') }}</option>
+      </div>
-        </select>
+      <div class="form-group">
+        <label for="mail_port">{{_('SMTP Port')}}</label>
+        <input type="number" min="1" max="65535" step="1" class="form-control" name="mail_port" id="mail_port" value="{% if content.mail_port != None %}{{ content.mail_port }}{% endif %}" autocomplete="off" required>
+      </div>
+      <div class="form-group">
+        <label for="mail_use_ssl">{{_('Encryption')}}</label>
+          <select name="mail_use_ssl" id="mail_use_ssl" class="form-control">
+              <option value="0" {% if content.mail_use_ssl == 0 %}selected{% endif %}>{{ _('None') }}</option>
+              <option value="1" {% if content.mail_use_ssl == 1 %}selected{% endif %}>{{ _('STARTTLS') }}</option>
+              <option value="2" {% if content.mail_use_ssl == 2 %}selected{% endif %}>{{ _('SSL/TLS') }}</option>
+          </select>
+      </div>
+      <div class="form-group">
+        <label for="mail_login">{{_('SMTP Login')}}</label>
+        <input type="text" class="form-control" name="mail_login" id="mail_login" value="{{content.mail_login}}">
+      </div>
+      <div class="form-group">
+        <label for="mail_password">{{_('SMTP Password')}}</label>
+        <input type="password" class="form-control" name="mail_password" id="mail_password" value="{{content.mail_password}}">
+      </div>
+      <div class="form-group">
+        <label for="mail_from">{{_('From E-mail')}}</label>
+        <input type="text" class="form-control" name="mail_from" id="mail_from" value="{{content.mail_from}}" required>
+      </div>
+      <label for="mail_size">{{_('Attachment Size Limit')}}</label>
+       <div class="form-group input-group">
+        <input type="number" min="1" max="600" step="1" class="form-control" name="mail_size" id="mail_size" value="{% if content.mail_size != None %}{{ (content.mail_size / 1024 / 1024)|int }}{% endif %}" required>
+        <span class="input-group-btn">
+          <button type="button" id="attachement_size" class="btn btn-default" disabled>MB</button>
+        </span>
+      </div>
+      <button type="submit" name="submit" value="submit" class="btn btn-default">{{_('Save')}}</button>
+      <button type="submit" name="test" value="test" class="btn btn-default">{{_('Save and Send Test E-mail')}}</button>
+   {% if feature_support['gmail'] %}
     </div>
-    <div class="form-group">
+   {% endif %}
-      <label for="mail_login">{{_('SMTP Login')}}</label>
+      <a href="{{ url_for('admin.admin') }}" id="back" class="btn btn-default">{{_('Back')}}</a>
-      <input type="text" class="form-control" name="mail_login" id="mail_login" value="{{content.mail_login}}">
-    </div>
-    <div class="form-group">
-      <label for="mail_password">{{_('SMTP Password')}}</label>
-      <input type="password" class="form-control" name="mail_password" id="mail_password" value="{{content.mail_password}}">
-    </div>
-    <div class="form-group">
-      <label for="mail_from">{{_('From E-mail')}}</label>
-      <input type="text" class="form-control" name="mail_from" id="mail_from" value="{{content.mail_from}}" required>
-    </div>
-    <label for="mail_size">{{_('Attachment Size Limit')}}</label>
-     <div class="form-group input-group">
-      <input type="number" min="1" max="600" step="1" class="form-control" name="mail_size" id="mail_size" value="{% if content.mail_size != None %}{{ (content.mail_size / 1024 / 1024)|int }}{% endif %}" required>
-      <span class="input-group-btn">
-        <button type="button" id="attachement_size" class="btn btn-default" disabled>MB</button>
-      </span>
-    </div>
-    <button type="submit" name="submit" value="submit" class="btn btn-default">{{_('Save')}}</button>
-    <button type="submit" name="test" value="test" class="btn btn-default">{{_('Save and Send Test E-mail')}}</button>
-    <a href="{{ url_for('admin.admin') }}" id="back" class="btn btn-default">{{_('Cancel')}}</a>
   </form>
     {% if g.allow_registration %}
   <div class="col-md-10 col-lg-6">

cps/templates/feed.xml

@@ -84,4 +84,11 @@
     <link rel="subsection" type="application/atom+xml;profile=opds-catalog" href="{{url_for(folder, book_id=entry.id)}}"/>
   </entry>
   {% endfor %}
+  {% for entry in letterelements %}
+  <entry>
+    <title>{{entry['name']}}</title>
+    <id>{{ url_for(folder, book_id=entry['id']) }}</id>
+    <link rel="subsection" type="application/atom+xml;profile=opds-catalog" href="{{url_for(folder, book_id=entry['id'])}}"/>
+  </entry>
+  {% endfor %}
 </feed>

cps/templates/grid.html

@@ -8,8 +8,8 @@
         <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
        {% endif %}
       {% endif %}
-      <button id="desc" data-id="series" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" data-id="series" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
-      <button id="asc" data-id="series" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      <button id="desc" data-id="series" data-order="{{ order }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
       {% if charlist|length %}
       <button id="all" class="btn btn-primary">{{_('All')}}</button>
       {% endif %}

cps/templates/index.xml

@@ -14,6 +14,13 @@
     <name>{{instance}}</name>
     <uri>https://github.com/janeczku/calibre-web</uri>
   </author>
+  <entry>
+    <title>{{_('Alphabetical Books')}}</title>
+    <link href="{{url_for('opds.feed_booksindex')}}" type="application/atom+xml;profile=opds-catalog"/>
+    <id>{{url_for('opds.feed_booksindex')}}</id>
+    <updated>{{ current_time }}</updated>
+    <content type="text">{{_('Books sorted alphabetically')}}</content>
+  </entry>
   <entry>
     <title>{{_('Hot Books')}}</title>
     <link href="{{url_for('opds.feed_hot')}}" type="application/atom+xml;profile=opds-catalog"/>

cps/templates/layout.html

@@ -1,4 +1,4 @@
-{% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal %}
+{% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal %}
 <!DOCTYPE html>
 <html lang="{{ g.user.locale }}">
   <head>
@@ -76,7 +76,7 @@
               {% if g.user.role_admin() %}
                 <li><a id="top_admin" data-text="{{_('Settings')}}" href="{{url_for('admin.admin')}}"><span class="glyphicon glyphicon-dashboard"></span> <span class="hidden-sm">{{_('Admin')}}</span></a></li>
               {% endif %}
-              <li><a id="top_user" data-text="{{_('Account')}}" href="{{url_for('web.profile')}}"><span class="glyphicon glyphicon-user"></span> <span class="hidden-sm">{{g.user.nickname}}</span></a></li>
+              <li><a id="top_user" data-text="{{_('Account')}}" href="{{url_for('web.profile')}}"><span class="glyphicon glyphicon-user"></span> <span class="hidden-sm">{{g.user.name}}</span></a></li>
               {% if not g.user.is_anonymous %}
               <li><a id="logout" href="{{url_for('web.logout')}}"><span class="glyphicon glyphicon-log-out"></span> <span class="hidden-sm">{{_('Logout')}}</span></a></li>
               {% endif %}

cps/templates/list.html

@@ -8,8 +8,8 @@
         <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
        {% endif %}
       {% endif %}
-      <button id="desc" data-id="{{ data }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" data-order="{{ order }}" data-id="{{ data }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
-      <button id="asc" data-id="{{ data }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      <button id="desc" data-id="{{ data }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
       {% if charlist|length %}
       <button id="all" class="btn btn-primary">{{_('All')}}</button>
       {% endif %}

cps/templates/modal_dialogs.html

@@ -22,7 +22,7 @@
           <form id="add_restriction" action="" method="POST">
           <div class="form-group required">
             <label for="add_element">{{_('Add View Restriction')}}</label>
-            <input type="text" class="form-control" name="add_element" id="add_element" >
+            <input type="text" class="form-control" name="add_element" id="add_element">
           </div>
           <div class="form-group required">
             <input type="button" class="btn btn-default" value="{{_('Allow')}}" name="submit_allow" id="submit_allow" data-dismiss="static">
@@ -108,13 +108,31 @@
   <div class="modal-dialog modal-sm">
     <div class="modal-content">
       <div class="modal-header bg-danger text-center">
-      <span id="header"></span>
+      <span id="header-GeneralDeleteModal"></span>
       </div>
       <div class="modal-body text-center">
-        <span id="text"></span>
+        <span id="text-GeneralDeleteModal"></span>
           <p></p>
-        <button id="btnConfirmYes" type="button" class="btn btn btn-danger">{{_('Delete')}}</button>
+        <button id="btnConfirmYes-GeneralDeleteModal" type="button" class="btn btn btn-danger">{{_('Delete')}}</button>
-        <button id="btnConfirmNo" type="button" class="btn btn-default">{{_('Cancel')}}</button>
+        <button id="btnConfirmNo-GeneralDeleteModal" type="button" class="btn btn-default">{{_('Cancel')}}</button>
+      </div>
+    </div>
+  </div>
+</div>
+{% endmacro %}
+
+{% macro change_confirm_modal() %}
+<div id="GeneralChangeModal" class="modal fade" role="Dialog">
+  <div class="modal-dialog modal-sm">
+    <div class="modal-content">
+      <div class="modal-header bg-info text-center">
+      <span id="header-GeneralChangeModal"></span>
+      </div>
+      <div class="modal-body text-center">
+        <span id="text-GeneralChangeModal"></span>
+          <p></p>
+        <button id="btnConfirmYes-GeneralChangeModal" type="button" class="btn btn btn-default">{{_('Ok')}}</button>
+        <button id="btnConfirmNo-GeneralChangeModal" type="button" class="btn btn-default">{{_('Cancel')}}</button>
       </div>
     </div>
   </div>

cps/templates/register.html

@@ -5,8 +5,8 @@
   <form method="POST" role="form">
     {% if not config.config_register_email %}
     <div class="form-group required">
-      <label for="nickname">{{_('Username')}}</label>
+      <label for="name">{{_('Username')}}</label>
-      <input type="text" class="form-control" id="nickname" name="nickname" placeholder="{{_('Choose a username')}}" required>
+      <input type="text" class="form-control" id="name" name="name" placeholder="{{_('Choose a username')}}" required>
     </div>
     {% endif %}
     <div class="form-group required">

cps/templates/user_edit.html

@@ -4,10 +4,10 @@
   <h1>{{title}}</h1>
   <form role="form" method="POST" autocomplete="off">
     <div class="col-md-10 col-lg-8">
-    {% if new_user or ( g.user and content.nickname != "Guest" and g.user.role_admin() ) %}
+    {% if new_user or ( g.user and content.name != "Guest" and g.user.role_admin() ) %}
     <div class="form-group required">
-      <label for="nickname">{{_('Username')}}</label>
+      <label for="name">{{_('Username')}}</label>
-      <input type="text" class="form-control" name="nickname" id="nickname" value="{{ content.nickname if content.nickname != None }}" autocomplete="off">
+      <input type="text" class="form-control" name="name" id="name" value="{{ content.name if content.name != None }}" autocomplete="off">
     </div>
     {% endif %}
     <div class="form-group">