From 2e007a160e652b2e7bbdeb5a8319560188324502 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Tue, 26 Apr 2022 14:44:55 +0200 Subject: [PATCH] reenable startup logging Bugfixes from refactoring and merge --- cps/__init__.py | 9 +- cps/admin.py | 957 +++++++++++----------- cps/babel.py | 18 +- cps/editbooks.py | 1627 +++++++++++++++++++------------------- cps/helper.py | 80 +- cps/kobo_auth.py | 76 +- cps/main.py | 8 +- cps/opds.py | 28 +- cps/search.py | 1 - cps/services/__init__.py | 5 +- cps/shelf.py | 219 ++--- cps/tasks_status.py | 4 +- cps/templates/tasks.html | 2 +- cps/web.py | 8 +- 14 files changed, 1516 insertions(+), 1526 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index 0a452b17..f597aab5 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -27,16 +27,15 @@ from flask import Flask from .MyLoginManager import MyLoginManager from flask_principal import Principal +from . import logger from .cli import CliParameter from .constants import CONFIG_DIR from .reverseproxy import ReverseProxied from .server import WebServer from .dep_check import dependency_check -from . import services from .updater import Updater -from .babel import babel, BABEL_TRANSLATIONS +from .babel import babel from . import config_sql -from . import logger from . import cache_buster from . import ub, db @@ -157,8 +156,8 @@ def create_app(): web_server.init_app(app, config) babel.init_app(app) - BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) - BABEL_TRANSLATIONS.add('en') + + from . import services if services.ldap: services.ldap.init_app(app, config) diff --git a/cps/admin.py b/cps/admin.py index 29f2319e..f5ae1037 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -28,6 +28,7 @@ import operator from datetime import datetime, timedelta, time from functools import wraps + from flask import Blueprint, flash, redirect, url_for, abort, request, make_response, send_from_directory, g, Response from flask_login import login_required, current_user, logout_user, confirm_login from flask_babel import gettext as _ @@ -38,15 +39,16 @@ from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text -from . import constants, logger, helper, services, cli -from . import db, calibre_db, ub, web_server, config, updater_thread, babel, gdriveutils, \ +from . import constants, logger, helper, services, cli_param +from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username, update_thumbnail_cache + valid_email, check_username from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread -from . import debug_info, BABEL_TRANSLATIONS +from .babel import get_available_translations, get_available_locale, get_user_locale_language +from . import debug_info log = logger.create() @@ -57,7 +59,8 @@ feature_support = { 'kobo': bool(services.kobo), 'updater': constants.UPDATER_AVAILABLE, 'gmail': bool(services.gmail), - 'scheduler': schedule.use_APScheduler + 'scheduler': schedule.use_APScheduler, + 'gdrive': gdrive_support } try: @@ -76,7 +79,6 @@ except ImportError as err: oauth_check = {} -feature_support['gdrive'] = gdrive_support admi = Blueprint('admin', __name__) @@ -160,7 +162,7 @@ def shutdown(): # needed for docker applications, as changes on metadata.db from host are not visible to application @admi.route("/reconnect", methods=['GET']) def reconnect(): - if cli.reconnect_enable: + if cli_param.reconnect_enable: calibre_db.reconnect_db(config, ub.app_DB_path) return json.dumps({}) else: @@ -175,7 +177,7 @@ def update_thumbnails(): content = config.get_scheduled_task_settings() if content['schedule_generate_book_covers']: log.info("Update of Cover cache requested") - update_thumbnail_cache() + helper.update_thumbnail_cache() return "" @@ -264,7 +266,7 @@ def view_configuration(): restrict_columns = calibre_db.session.query(db.CustomColumns)\ .filter(and_(db.CustomColumns.datatype == 'text', db.CustomColumns.mark_for_delete == 0)).all() languages = calibre_db.speaking_language() - translations = [Locale('en')] + babel.list_translations() + translations = get_available_locale() return render_title_template("config_view_edit.html", conf=config, readColumns=read_column, restrictColumns=restrict_columns, languages=languages, @@ -278,7 +280,7 @@ def view_configuration(): def edit_user_table(): visibility = current_user.view_settings.get('useredit', {}) languages = calibre_db.speaking_language() - translations = [LC('en')] + babel.list_translations() + translations = get_available_locale() all_user = ub.session.query(ub.User) tags = calibre_db.session.query(db.Tags)\ .join(db.books_tags_link)\ @@ -349,7 +351,7 @@ def list_users(): if user.default_language == "all": user.default = _("All") else: - user.default = Locale.parse(user.default_language).get_language_name(get_locale()) + user.default = get_user_locale_language(user.default_language) table_entries = {'totalNotFiltered': total_count, 'total': filtered_count, "rows": users} js_list = json.dumps(table_entries, cls=db.AlchemyEncoder) @@ -397,7 +399,7 @@ def delete_user(): @login_required @admin_required def table_get_locale(): - locale = [LC('en')] + babel.list_translations() + locale = get_available_locale() ret = list() current_locale = get_locale() for loc in locale: @@ -498,7 +500,7 @@ def edit_list_user(param): elif param == 'locale': if user.name == "Guest": raise Exception(_("Guest's Locale is determined automatically and can't be set")) - if vals['value'] in BABEL_TRANSLATIONS: + if vals['value'] in get_available_translations(): user.locale = vals['value'] else: raise Exception(_("No Valid Locale Given")) @@ -539,22 +541,6 @@ def update_table_settings(): return "" -def check_valid_read_column(column): - if column != "0": - if not calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.id == column) \ - .filter(and_(db.CustomColumns.datatype == 'bool', db.CustomColumns.mark_for_delete == 0)).all(): - return False - return True - - -def check_valid_restricted_column(column): - if column != "0": - if not calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.id == column) \ - .filter(and_(db.CustomColumns.datatype == 'text', db.CustomColumns.mark_for_delete == 0)).all(): - return False - return True - - @admi.route("/admin/viewconfig", methods=["POST"]) @login_required @admin_required @@ -759,43 +745,6 @@ def edit_restriction(res_type, user_id): ub.session_commit("Changed denied columns of user {} to {}".format(usr.name, usr.denied_column_value)) return "" - -def restriction_addition(element, list_func): - elementlist = list_func() - if elementlist == ['']: - elementlist = [] - if not element['add_element'] in elementlist: - elementlist += [element['add_element']] - return ','.join(elementlist) - - -def restriction_deletion(element, list_func): - elementlist = list_func() - if element['Element'] in elementlist: - elementlist.remove(element['Element']) - return ','.join(elementlist) - - -def prepare_tags(user, action, tags_name, id_list): - if "tags" in tags_name: - tags = calibre_db.session.query(db.Tags).filter(db.Tags.id.in_(id_list)).all() - if not tags: - raise Exception(_("Tag not found")) - new_tags_list = [x.name for x in tags] - else: - tags = calibre_db.session.query(db.cc_classes[config.config_restricted_column])\ - .filter(db.cc_classes[config.config_restricted_column].id.in_(id_list)).all() - new_tags_list = [x.value for x in tags] - saved_tags_list = user.__dict__[tags_name].split(",") if len(user.__dict__[tags_name]) else [] - if action == "remove": - saved_tags_list = [x for x in saved_tags_list if x not in new_tags_list] - elif action == "add": - saved_tags_list.extend(x for x in new_tags_list if x not in saved_tags_list) - else: - raise Exception(_("Invalid Action")) - return ",".join(saved_tags_list) - - @admi.route("/ajax/addrestriction/", methods=['POST']) @login_required @admin_required @@ -964,6 +913,58 @@ def ajax_pathchooser(): return pathchooser() +def check_valid_read_column(column): + if column != "0": + if not calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.id == column) \ + .filter(and_(db.CustomColumns.datatype == 'bool', db.CustomColumns.mark_for_delete == 0)).all(): + return False + return True + + +def check_valid_restricted_column(column): + if column != "0": + if not calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.id == column) \ + .filter(and_(db.CustomColumns.datatype == 'text', db.CustomColumns.mark_for_delete == 0)).all(): + return False + return True + + +def restriction_addition(element, list_func): + elementlist = list_func() + if elementlist == ['']: + elementlist = [] + if not element['add_element'] in elementlist: + elementlist += [element['add_element']] + return ','.join(elementlist) + + +def restriction_deletion(element, list_func): + elementlist = list_func() + if element['Element'] in elementlist: + elementlist.remove(element['Element']) + return ','.join(elementlist) + + +def prepare_tags(user, action, tags_name, id_list): + if "tags" in tags_name: + tags = calibre_db.session.query(db.Tags).filter(db.Tags.id.in_(id_list)).all() + if not tags: + raise Exception(_("Tag not found")) + new_tags_list = [x.name for x in tags] + else: + tags = calibre_db.session.query(db.cc_classes[config.config_restricted_column])\ + .filter(db.cc_classes[config.config_restricted_column].id.in_(id_list)).all() + new_tags_list = [x.value for x in tags] + saved_tags_list = user.__dict__[tags_name].split(",") if len(user.__dict__[tags_name]) else [] + if action == "remove": + saved_tags_list = [x for x in saved_tags_list if x not in new_tags_list] + elif action == "add": + saved_tags_list.extend(x for x in new_tags_list if x not in saved_tags_list) + else: + raise Exception(_("Invalid Action")) + return ",".join(saved_tags_list) + + def pathchooser(): browse_for = "folder" folder_only = request.args.get('folder', False) == "true" @@ -1207,6 +1208,420 @@ def simulatedbchange(): return Response(json.dumps({"change": db_change, "valid": db_valid}), mimetype='application/json') +@admi.route("/admin/user/new", methods=["GET", "POST"]) +@login_required +@admin_required +def new_user(): + content = ub.User() + languages = calibre_db.speaking_language() + translations = get_available_locale() + kobo_support = feature_support['kobo'] and config.config_kobo_sync + if request.method == "POST": + to_save = request.form.to_dict() + _handle_new_user(to_save, content, languages, translations, kobo_support) + else: + content.role = config.config_default_role + content.sidebar_view = config.config_default_show + content.locale = config.config_default_locale + content.default_language = config.config_default_language + return render_title_template("user_edit.html", new_user=1, content=content, + config=config, translations=translations, + languages=languages, title=_(u"Add new user"), page="newuser", + kobo_support=kobo_support, registered_oauth=oauth_check) + + +@admi.route("/admin/mailsettings") +@login_required +@admin_required +def edit_mailsettings(): + content = config.get_mail_settings() + return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"), + page="mailset", feature_support=feature_support) + + +@admi.route("/admin/mailsettings", methods=["POST"]) +@login_required +@admin_required +def update_mailsettings(): + to_save = request.form.to_dict() + _config_int(to_save, "mail_server_type") + if to_save.get("invalidate"): + config.mail_gmail_token = {} + try: + flag_modified(config, "mail_gmail_token") + except AttributeError: + pass + elif to_save.get("gmail"): + try: + config.mail_gmail_token = services.gmail.setup_gmail(config.mail_gmail_token) + flash(_(u"Gmail Account Verification Successful"), category="success") + except Exception as ex: + flash(str(ex), category="error") + log.error(ex) + return edit_mailsettings() + + else: + _config_string(to_save, "mail_server") + _config_int(to_save, "mail_port") + _config_int(to_save, "mail_use_ssl") + _config_string(to_save, "mail_login") + _config_string(to_save, "mail_password") + _config_string(to_save, "mail_from") + _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024) + try: + config.save() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") + return edit_mailsettings() + + if to_save.get("test"): + if current_user.email: + result = send_test_mail(current_user.email, current_user.name) + if result is None: + flash(_(u"Test e-mail queued for sending to %(email)s, please check Tasks for result", + email=current_user.email), category="info") + else: + flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error") + else: + flash(_(u"Please configure your e-mail address first..."), category="error") + else: + flash(_(u"E-mail server settings updated"), category="success") + + return edit_mailsettings() + + +@admi.route("/admin/scheduledtasks") +@login_required +@admin_required +def edit_scheduledtasks(): + content = config.get_scheduled_task_settings() + time_field = list() + duration_field = list() + + for n in range(24): + time_field.append((n , format_time(time(hour=n), format="short",))) + for n in range(5, 65, 5): + t = timedelta(hours=n // 60, minutes=n % 60) + duration_field.append((n, format_timedelta(t, threshold=.9))) + + return render_title_template("schedule_edit.html", + config=content, + starttime=time_field, + duration=duration_field, + title=_(u"Edit Scheduled Tasks Settings")) + + +@admi.route("/admin/scheduledtasks", methods=["POST"]) +@login_required +@admin_required +def update_scheduledtasks(): + error = False + to_save = request.form.to_dict() + if 0 <= int(to_save.get("schedule_start_time")) <= 23: + _config_int(to_save, "schedule_start_time") + else: + flash(_(u"Invalid start time for task specified"), category="error") + error = True + if 0 < int(to_save.get("schedule_duration")) <= 60: + _config_int(to_save, "schedule_duration") + else: + flash(_(u"Invalid duration for task specified"), category="error") + error = True + _config_checkbox(to_save, "schedule_generate_book_covers") + _config_checkbox(to_save, "schedule_generate_series_covers") + _config_checkbox(to_save, "schedule_reconnect") + + if not error: + try: + config.save() + flash(_(u"Scheduled tasks settings updated"), category="success") + + # Cancel any running tasks + schedule.end_scheduled_tasks() + + # Re-register tasks with new settings + schedule.register_scheduled_tasks(config.schedule_reconnect) + except IntegrityError: + ub.session.rollback() + log.error("An unknown error occurred while saving scheduled tasks settings") + flash(_(u"An unknown error occurred. Please try again later."), category="error") + except OperationalError: + ub.session.rollback() + log.error("Settings DB is not Writeable") + flash(_("Settings DB is not Writeable"), category="error") + + return edit_scheduledtasks() + + +@admi.route("/admin/user/", methods=["GET", "POST"]) +@login_required +@admin_required +def edit_user(user_id): + content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first() # type: ub.User + if not content or (not config.config_anonbrowse and content.name == "Guest"): + flash(_(u"User not found"), category="error") + return redirect(url_for('admin.admin')) + languages = calibre_db.speaking_language(return_all_languages=True) + translations = get_available_locale() + kobo_support = feature_support['kobo'] and config.config_kobo_sync + if request.method == "POST": + to_save = request.form.to_dict() + resp = _handle_edit_user(to_save, content, languages, translations, kobo_support) + if resp: + return resp + return render_title_template("user_edit.html", + translations=translations, + languages=languages, + new_user=0, + content=content, + config=config, + registered_oauth=oauth_check, + mail_configured=config.get_mail_server_configured(), + kobo_support=kobo_support, + title=_(u"Edit User %(nick)s", nick=content.name), + page="edituser") + + +@admi.route("/admin/resetpassword/", methods=["POST"]) +@login_required +@admin_required +def reset_user_password(user_id): + if current_user is not None and current_user.is_authenticated: + ret, message = reset_password(user_id) + if ret == 1: + log.debug(u"Password for user %s reset", message) + flash(_(u"Password for user %(user)s reset", user=message), category="success") + elif ret == 0: + log.error(u"An unknown error occurred. Please try again later.") + flash(_(u"An unknown error occurred. Please try again later."), category="error") + else: + log.error(u"Please configure the SMTP mail settings first...") + flash(_(u"Please configure the SMTP mail settings first..."), category="error") + return redirect(url_for('admin.admin')) + + +@admi.route("/admin/logfile") +@login_required +@admin_required +def view_logfile(): + logfiles = {0: logger.get_logfile(config.config_logfile), + 1: logger.get_accesslogfile(config.config_access_logfile)} + return render_title_template("logviewer.html", + title=_(u"Logfile viewer"), + accesslog_enable=config.config_access_log, + log_enable=bool(config.config_logfile != logger.LOG_TO_STDOUT), + logfiles=logfiles, + page="logfile") + + +@admi.route("/ajax/log/") +@login_required +@admin_required +def send_logfile(logtype): + if logtype == 1: + logfile = logger.get_accesslogfile(config.config_access_logfile) + return send_from_directory(os.path.dirname(logfile), + os.path.basename(logfile)) + if logtype == 0: + logfile = logger.get_logfile(config.config_logfile) + return send_from_directory(os.path.dirname(logfile), + os.path.basename(logfile)) + else: + return "" + + +@admi.route("/admin/logdownload/") +@login_required +@admin_required +def download_log(logtype): + if logtype == 0: + file_name = logger.get_logfile(config.config_logfile) + elif logtype == 1: + file_name = logger.get_accesslogfile(config.config_access_logfile) + else: + abort(404) + if logger.is_valid_logfile(file_name): + return debug_info.assemble_logfiles(file_name) + abort(404) + + +@admi.route("/admin/debug") +@login_required +@admin_required +def download_debug(): + return debug_info.send_debug() + + +@admi.route("/get_update_status", methods=['GET']) +@login_required +@admin_required +def get_update_status(): + if feature_support['updater']: + log.info(u"Update status requested") + return updater_thread.get_available_updates(request.method, locale=get_locale()) + else: + return '' + + +@admi.route("/get_updater_status", methods=['GET', 'POST']) +@login_required +@admin_required +def get_updater_status(): + status = {} + if feature_support['updater']: + if request.method == "POST": + commit = request.form.to_dict() + if "start" in commit and commit['start'] == 'True': + txt = { + "1": _(u'Requesting update package'), + "2": _(u'Downloading update package'), + "3": _(u'Unzipping update package'), + "4": _(u'Replacing files'), + "5": _(u'Database connections are closed'), + "6": _(u'Stopping server'), + "7": _(u'Update finished, please press okay and reload page'), + "8": _(u'Update failed:') + u' ' + _(u'HTTP Error'), + "9": _(u'Update failed:') + u' ' + _(u'Connection error'), + "10": _(u'Update failed:') + u' ' + _(u'Timeout while establishing connection'), + "11": _(u'Update failed:') + u' ' + _(u'General error'), + "12": _(u'Update failed:') + u' ' + _(u'Update file could not be saved in temp dir'), + "13": _(u'Update failed:') + u' ' + _(u'Files could not be replaced during update') + } + status['text'] = txt + updater_thread.status = 0 + updater_thread.resume() + status['status'] = updater_thread.get_update_status() + elif request.method == "GET": + try: + status['status'] = updater_thread.get_update_status() + if status['status'] == -1: + status['status'] = 7 + except Exception: + status['status'] = 11 + return json.dumps(status) + return '' + + +def ldap_import_create_user(user, user_data): + user_login_field = extract_dynamic_field_from_filter(user, config.config_ldap_user_object) + + try: + username = user_data[user_login_field][0].decode('utf-8') + except KeyError as ex: + log.error("Failed to extract LDAP user: %s - %s", user, ex) + message = _(u'Failed to extract at least One LDAP User') + return 0, message + + # check for duplicate username + if ub.session.query(ub.User).filter(func.lower(ub.User.name) == username.lower()).first(): + # if ub.session.query(ub.User).filter(ub.User.name == username).first(): + log.warning("LDAP User %s Already in Database", user_data) + return 0, None + + kindlemail = '' + if 'mail' in user_data: + useremail = user_data['mail'][0].decode('utf-8') + if len(user_data['mail']) > 1: + kindlemail = user_data['mail'][1].decode('utf-8') + + else: + log.debug('No Mail Field Found in LDAP Response') + useremail = username + '@email.com' + + try: + # check for duplicate email + useremail = check_email(useremail) + except Exception as ex: + log.warning("LDAP Email Error: {}, {}".format(user_data, ex)) + return 0, None + content = ub.User() + content.name = username + content.password = '' # dummy password which will be replaced by ldap one + content.email = useremail + content.kindle_mail = kindlemail + content.default_language = config.config_default_language + content.locale = config.config_default_locale + content.role = config.config_default_role + content.sidebar_view = config.config_default_show + content.allowed_tags = config.config_allowed_tags + content.denied_tags = config.config_denied_tags + content.allowed_column_value = config.config_allowed_column_value + content.denied_column_value = config.config_denied_column_value + ub.session.add(content) + try: + ub.session.commit() + return 1, None # increase no of users + except Exception as ex: + log.warning("Failed to create LDAP user: %s - %s", user, ex) + ub.session.rollback() + message = _(u'Failed to Create at Least One LDAP User') + return 0, message + + +@admi.route('/import_ldap_users', methods=["POST"]) +@login_required +@admin_required +def import_ldap_users(): + showtext = {} + try: + new_users = services.ldap.get_group_members(config.config_ldap_group_name) + except (services.ldap.LDAPException, TypeError, AttributeError, KeyError) as e: + log.error_or_exception(e) + showtext['text'] = _(u'Error: %(ldaperror)s', ldaperror=e) + return json.dumps(showtext) + if not new_users: + log.debug('LDAP empty response') + showtext['text'] = _(u'Error: No user returned in response of LDAP server') + return json.dumps(showtext) + + imported = 0 + for username in new_users: + user = username.decode('utf-8') + if '=' in user: + # if member object field is empty take user object as filter + if config.config_ldap_member_user_object: + query_filter = config.config_ldap_member_user_object + else: + query_filter = config.config_ldap_user_object + try: + user_identifier = extract_user_identifier(user, query_filter) + except Exception as ex: + log.warning(ex) + continue + else: + user_identifier = user + query_filter = None + try: + user_data = services.ldap.get_object_details(user=user_identifier, query_filter=query_filter) + except AttributeError as ex: + log.error_or_exception(ex) + continue + if user_data: + user_count, message = ldap_import_create_user(user, user_data) + if message: + showtext['text'] = message + else: + imported += user_count + else: + log.warning("LDAP User: %s Not Found", user) + showtext['text'] = _(u'At Least One LDAP User Not Found in Database') + if not showtext: + showtext['text'] = _(u'{} User Successfully Imported'.format(imported)) + return json.dumps(showtext) + + +@admi.route("/ajax/canceltask", methods=['POST']) +@login_required +@admin_required +def cancel_task(): + task_id = request.get_json().get('task_id', None) + worker = WorkerThread.get_instance() + worker.end_task(task_id) + return "" + + def _db_simulate_change(): param = request.form.to_dict() to_save = dict() @@ -1575,406 +1990,6 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support): return "" -@admi.route("/admin/user/new", methods=["GET", "POST"]) -@login_required -@admin_required -def new_user(): - content = ub.User() - languages = calibre_db.speaking_language() - translations = [Locale('en')] + babel.list_translations() - kobo_support = feature_support['kobo'] and config.config_kobo_sync - if request.method == "POST": - to_save = request.form.to_dict() - _handle_new_user(to_save, content, languages, translations, kobo_support) - else: - content.role = config.config_default_role - content.sidebar_view = config.config_default_show - content.locale = config.config_default_locale - content.default_language = config.config_default_language - return render_title_template("user_edit.html", new_user=1, content=content, - config=config, translations=translations, - languages=languages, title=_(u"Add new user"), page="newuser", - kobo_support=kobo_support, registered_oauth=oauth_check) - - -@admi.route("/admin/mailsettings") -@login_required -@admin_required -def edit_mailsettings(): - content = config.get_mail_settings() - return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"), - page="mailset", feature_support=feature_support) - - -@admi.route("/admin/mailsettings", methods=["POST"]) -@login_required -@admin_required -def update_mailsettings(): - to_save = request.form.to_dict() - _config_int(to_save, "mail_server_type") - if to_save.get("invalidate"): - config.mail_gmail_token = {} - try: - flag_modified(config, "mail_gmail_token") - except AttributeError: - pass - elif to_save.get("gmail"): - try: - config.mail_gmail_token = services.gmail.setup_gmail(config.mail_gmail_token) - flash(_(u"Gmail Account Verification Successful"), category="success") - except Exception as ex: - flash(str(ex), category="error") - log.error(ex) - return edit_mailsettings() - - else: - _config_string(to_save, "mail_server") - _config_int(to_save, "mail_port") - _config_int(to_save, "mail_use_ssl") - _config_string(to_save, "mail_login") - _config_string(to_save, "mail_password") - _config_string(to_save, "mail_from") - _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024) - try: - config.save() - except (OperationalError, InvalidRequestError) as e: - ub.session.rollback() - log.error_or_exception("Settings Database error: {}".format(e)) - flash(_(u"Database error: %(error)s.", error=e.orig), category="error") - return edit_mailsettings() - - if to_save.get("test"): - if current_user.email: - result = send_test_mail(current_user.email, current_user.name) - if result is None: - flash(_(u"Test e-mail queued for sending to %(email)s, please check Tasks for result", - email=current_user.email), category="info") - else: - flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error") - else: - flash(_(u"Please configure your e-mail address first..."), category="error") - else: - flash(_(u"E-mail server settings updated"), category="success") - - return edit_mailsettings() - - -@admi.route("/admin/scheduledtasks") -@login_required -@admin_required -def edit_scheduledtasks(): - content = config.get_scheduled_task_settings() - time_field = list() - duration_field = list() - - for n in range(24): - time_field.append((n , format_time(time(hour=n), format="short",))) - for n in range(5, 65, 5): - t = timedelta(hours=n // 60, minutes=n % 60) - duration_field.append((n, format_timedelta(t, threshold=.9))) - - return render_title_template("schedule_edit.html", config=content, starttime=time_field, duration=duration_field, title=_(u"Edit Scheduled Tasks Settings")) - - -@admi.route("/admin/scheduledtasks", methods=["POST"]) -@login_required -@admin_required -def update_scheduledtasks(): - error = False - to_save = request.form.to_dict() - if 0 <= int(to_save.get("schedule_start_time")) <= 23: - _config_int(to_save, "schedule_start_time") - else: - flash(_(u"Invalid start time for task specified"), category="error") - error = True - if 0 < int(to_save.get("schedule_duration")) <= 60: - _config_int(to_save, "schedule_duration") - else: - flash(_(u"Invalid duration for task specified"), category="error") - error = True - _config_checkbox(to_save, "schedule_generate_book_covers") - _config_checkbox(to_save, "schedule_generate_series_covers") - _config_checkbox(to_save, "schedule_reconnect") - - if not error: - try: - config.save() - flash(_(u"Scheduled tasks settings updated"), category="success") - - # Cancel any running tasks - schedule.end_scheduled_tasks() - - # Re-register tasks with new settings - schedule.register_scheduled_tasks(config.schedule_reconnect) - except IntegrityError: - ub.session.rollback() - log.error("An unknown error occurred while saving scheduled tasks settings") - flash(_(u"An unknown error occurred. Please try again later."), category="error") - except OperationalError: - ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") - - return edit_scheduledtasks() - - -@admi.route("/admin/user/", methods=["GET", "POST"]) -@login_required -@admin_required -def edit_user(user_id): - content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first() # type: ub.User - if not content or (not config.config_anonbrowse and content.name == "Guest"): - flash(_(u"User not found"), category="error") - return redirect(url_for('admin.admin')) - languages = calibre_db.speaking_language(return_all_languages=True) - translations = babel.list_translations() + [Locale('en')] - kobo_support = feature_support['kobo'] and config.config_kobo_sync - if request.method == "POST": - to_save = request.form.to_dict() - resp = _handle_edit_user(to_save, content, languages, translations, kobo_support) - if resp: - return resp - return render_title_template("user_edit.html", - translations=translations, - languages=languages, - new_user=0, - content=content, - config=config, - registered_oauth=oauth_check, - mail_configured=config.get_mail_server_configured(), - kobo_support=kobo_support, - title=_(u"Edit User %(nick)s", nick=content.name), - page="edituser") - - -@admi.route("/admin/resetpassword/", methods=["POST"]) -@login_required -@admin_required -def reset_user_password(user_id): - if current_user is not None and current_user.is_authenticated: - ret, message = reset_password(user_id) - if ret == 1: - log.debug(u"Password for user %s reset", message) - flash(_(u"Password for user %(user)s reset", user=message), category="success") - elif ret == 0: - log.error(u"An unknown error occurred. Please try again later.") - flash(_(u"An unknown error occurred. Please try again later."), category="error") - else: - log.error(u"Please configure the SMTP mail settings first...") - flash(_(u"Please configure the SMTP mail settings first..."), category="error") - return redirect(url_for('admin.admin')) - - -@admi.route("/admin/logfile") -@login_required -@admin_required -def view_logfile(): - logfiles = {0: logger.get_logfile(config.config_logfile), - 1: logger.get_accesslogfile(config.config_access_logfile)} - return render_title_template("logviewer.html", - title=_(u"Logfile viewer"), - accesslog_enable=config.config_access_log, - log_enable=bool(config.config_logfile != logger.LOG_TO_STDOUT), - logfiles=logfiles, - page="logfile") - - -@admi.route("/ajax/log/") -@login_required -@admin_required -def send_logfile(logtype): - if logtype == 1: - logfile = logger.get_accesslogfile(config.config_access_logfile) - return send_from_directory(os.path.dirname(logfile), - os.path.basename(logfile)) - if logtype == 0: - logfile = logger.get_logfile(config.config_logfile) - return send_from_directory(os.path.dirname(logfile), - os.path.basename(logfile)) - else: - return "" - - -@admi.route("/admin/logdownload/") -@login_required -@admin_required -def download_log(logtype): - if logtype == 0: - file_name = logger.get_logfile(config.config_logfile) - elif logtype == 1: - file_name = logger.get_accesslogfile(config.config_access_logfile) - else: - abort(404) - if logger.is_valid_logfile(file_name): - return debug_info.assemble_logfiles(file_name) - abort(404) - - -@admi.route("/admin/debug") -@login_required -@admin_required -def download_debug(): - return debug_info.send_debug() - - -@admi.route("/get_update_status", methods=['GET']) -@login_required -@admin_required -def get_update_status(): - if feature_support['updater']: - log.info(u"Update status requested") - return updater_thread.get_available_updates(request.method, locale=get_locale()) - else: - return '' - - -@admi.route("/get_updater_status", methods=['GET', 'POST']) -@login_required -@admin_required -def get_updater_status(): - status = {} - if feature_support['updater']: - if request.method == "POST": - commit = request.form.to_dict() - if "start" in commit and commit['start'] == 'True': - txt = { - "1": _(u'Requesting update package'), - "2": _(u'Downloading update package'), - "3": _(u'Unzipping update package'), - "4": _(u'Replacing files'), - "5": _(u'Database connections are closed'), - "6": _(u'Stopping server'), - "7": _(u'Update finished, please press okay and reload page'), - "8": _(u'Update failed:') + u' ' + _(u'HTTP Error'), - "9": _(u'Update failed:') + u' ' + _(u'Connection error'), - "10": _(u'Update failed:') + u' ' + _(u'Timeout while establishing connection'), - "11": _(u'Update failed:') + u' ' + _(u'General error'), - "12": _(u'Update failed:') + u' ' + _(u'Update file could not be saved in temp dir'), - "13": _(u'Update failed:') + u' ' + _(u'Files could not be replaced during update') - } - status['text'] = txt - updater_thread.status = 0 - updater_thread.resume() - status['status'] = updater_thread.get_update_status() - elif request.method == "GET": - try: - status['status'] = updater_thread.get_update_status() - if status['status'] == -1: - status['status'] = 7 - except Exception: - status['status'] = 11 - return json.dumps(status) - return '' - - -def ldap_import_create_user(user, user_data): - user_login_field = extract_dynamic_field_from_filter(user, config.config_ldap_user_object) - - try: - username = user_data[user_login_field][0].decode('utf-8') - except KeyError as ex: - log.error("Failed to extract LDAP user: %s - %s", user, ex) - message = _(u'Failed to extract at least One LDAP User') - return 0, message - - # check for duplicate username - if ub.session.query(ub.User).filter(func.lower(ub.User.name) == username.lower()).first(): - # if ub.session.query(ub.User).filter(ub.User.name == username).first(): - log.warning("LDAP User %s Already in Database", user_data) - return 0, None - - kindlemail = '' - if 'mail' in user_data: - useremail = user_data['mail'][0].decode('utf-8') - if len(user_data['mail']) > 1: - kindlemail = user_data['mail'][1].decode('utf-8') - - else: - log.debug('No Mail Field Found in LDAP Response') - useremail = username + '@email.com' - - try: - # check for duplicate email - useremail = check_email(useremail) - except Exception as ex: - log.warning("LDAP Email Error: {}, {}".format(user_data, ex)) - return 0, None - content = ub.User() - content.name = username - content.password = '' # dummy password which will be replaced by ldap one - content.email = useremail - content.kindle_mail = kindlemail - content.default_language = config.config_default_language - content.locale = config.config_default_locale - content.role = config.config_default_role - content.sidebar_view = config.config_default_show - content.allowed_tags = config.config_allowed_tags - content.denied_tags = config.config_denied_tags - content.allowed_column_value = config.config_allowed_column_value - content.denied_column_value = config.config_denied_column_value - ub.session.add(content) - try: - ub.session.commit() - return 1, None # increase no of users - except Exception as ex: - log.warning("Failed to create LDAP user: %s - %s", user, ex) - ub.session.rollback() - message = _(u'Failed to Create at Least One LDAP User') - return 0, message - - -@admi.route('/import_ldap_users', methods=["POST"]) -@login_required -@admin_required -def import_ldap_users(): - showtext = {} - try: - new_users = services.ldap.get_group_members(config.config_ldap_group_name) - except (services.ldap.LDAPException, TypeError, AttributeError, KeyError) as e: - log.error_or_exception(e) - showtext['text'] = _(u'Error: %(ldaperror)s', ldaperror=e) - return json.dumps(showtext) - if not new_users: - log.debug('LDAP empty response') - showtext['text'] = _(u'Error: No user returned in response of LDAP server') - return json.dumps(showtext) - - imported = 0 - for username in new_users: - user = username.decode('utf-8') - if '=' in user: - # if member object field is empty take user object as filter - if config.config_ldap_member_user_object: - query_filter = config.config_ldap_member_user_object - else: - query_filter = config.config_ldap_user_object - try: - user_identifier = extract_user_identifier(user, query_filter) - except Exception as ex: - log.warning(ex) - continue - else: - user_identifier = user - query_filter = None - try: - user_data = services.ldap.get_object_details(user=user_identifier, query_filter=query_filter) - except AttributeError as ex: - log.error_or_exception(ex) - continue - if user_data: - user_count, message = ldap_import_create_user(user, user_data) - if message: - showtext['text'] = message - else: - imported += user_count - else: - log.warning("LDAP User: %s Not Found", user) - showtext['text'] = _(u'At Least One LDAP User Not Found in Database') - if not showtext: - showtext['text'] = _(u'{} User Successfully Imported'.format(imported)) - return json.dumps(showtext) - - def extract_user_data_from_field(user, field): match = re.search(field + r"=([\.\d\s\w-]+)", user, re.IGNORECASE | re.UNICODE) if match: @@ -1994,13 +2009,3 @@ def extract_dynamic_field_from_filter(user, filtr): def extract_user_identifier(user, filtr): dynamic_field = extract_dynamic_field_from_filter(user, filtr) return extract_user_data_from_field(user, dynamic_field) - - -@admi.route("/ajax/canceltask", methods=['POST']) -@login_required -@admin_required -def cancel_task(): - task_id = request.get_json().get('task_id', None) - worker = WorkerThread.get_instance() - worker.end_task(task_id) - return "" diff --git a/cps/babel.py b/cps/babel.py index b0d5c238..c1675809 100644 --- a/cps/babel.py +++ b/cps/babel.py @@ -1,4 +1,4 @@ -from babel import Locale as LC +from babel import Locale from babel import negotiate_locale from flask_babel import Babel from babel.core import UnknownLocaleError @@ -9,7 +9,7 @@ from . import logger log = logger.create() babel = Babel() -BABEL_TRANSLATIONS = set() + @babel.localeselector def get_locale(): @@ -23,8 +23,18 @@ def get_locale(): if request.accept_languages: for x in request.accept_languages.values(): try: - preferred.append(str(LC.parse(x.replace('-', '_')))) + preferred.append(str(Locale.parse(x.replace('-', '_')))) except (UnknownLocaleError, ValueError) as e: log.debug('Could not parse locale "%s": %s', x, e) - return negotiate_locale(preferred or ['en'], BABEL_TRANSLATIONS) + return negotiate_locale(preferred or ['en'], get_available_translations()) + + +def get_user_locale_language(user_language): + return Locale.parse(user_language).get_language_name(get_locale()) + +def get_available_locale(): + return [Locale('en')] + babel.list_translations() + +def get_available_translations(): + return set(str(item) for item in get_available_locale()) diff --git a/cps/editbooks.py b/cps/editbooks.py index 3ac3dfb8..d3615050 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -39,10 +39,9 @@ from flask_babel import lazy_gettext as N_ from flask_babel import get_locale from flask_login import current_user, login_required from sqlalchemy.exc import OperationalError, IntegrityError -# from sqlite3 import OperationalError as sqliteOperationalError + from . import constants, logger, isoLanguages, gdriveutils, uploader, helper, kobo_sync_status -from . import config, ub, db -from . import calibre_db +from . import config, ub, db, calibre_db from .services.worker import WorkerThread from .tasks.upload import TaskUpload from .render_template import render_title_template @@ -74,161 +73,6 @@ def edit_required(f): return inner -def search_objects_remove(db_book_object, db_type, input_elements): - del_elements = [] - for c_elements in db_book_object: - found = False - if db_type == 'languages': - type_elements = c_elements.lang_code - elif db_type == 'custom': - type_elements = c_elements.value - else: - type_elements = c_elements.name - for inp_element in input_elements: - if inp_element.lower() == type_elements.lower(): - found = True - break - # if the element was not found in the new list, add it to remove list - if not found: - del_elements.append(c_elements) - return del_elements - - -def search_objects_add(db_book_object, db_type, input_elements): - add_elements = [] - for inp_element in input_elements: - found = False - for c_elements in db_book_object: - if db_type == 'languages': - type_elements = c_elements.lang_code - elif db_type == 'custom': - type_elements = c_elements.value - else: - type_elements = c_elements.name - if inp_element == type_elements: - found = True - break - if not found: - add_elements.append(inp_element) - return add_elements - - -def remove_objects(db_book_object, db_session, del_elements): - changed = False - if len(del_elements) > 0: - for del_element in del_elements: - db_book_object.remove(del_element) - changed = True - if len(del_element.books) == 0: - db_session.delete(del_element) - return changed - - -def add_objects(db_book_object, db_object, db_session, db_type, add_elements): - changed = False - if db_type == 'languages': - db_filter = db_object.lang_code - elif db_type == 'custom': - db_filter = db_object.value - else: - db_filter = db_object.name - for add_element in add_elements: - # check if an element with that name exists - db_element = db_session.query(db_object).filter(db_filter == add_element).first() - # if no element is found add it - if db_type == 'author': - new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')), "") - elif db_type == 'series': - new_element = db_object(add_element, add_element) - elif db_type == 'custom': - new_element = db_object(value=add_element) - elif db_type == 'publisher': - new_element = db_object(add_element, None) - else: # db_type should be tag or language - new_element = db_object(add_element) - if db_element is None: - changed = True - db_session.add(new_element) - db_book_object.append(new_element) - else: - db_element = create_objects_for_addition(db_element, add_element, db_type) - # add element to book - changed = True - db_book_object.append(db_element) - return changed - - -def create_objects_for_addition(db_element, add_element, db_type): - if db_type == 'custom': - if db_element.value != add_element: - db_element.value = add_element - elif db_type == 'languages': - if db_element.lang_code != add_element: - db_element.lang_code = add_element - elif db_type == 'series': - if db_element.name != add_element: - db_element.name = add_element - db_element.sort = add_element - elif db_type == 'author': - if db_element.name != add_element: - db_element.name = add_element - db_element.sort = helper.get_sorted_author(add_element.replace('|', ',')) - elif db_type == 'publisher': - if db_element.name != add_element: - db_element.name = add_element - db_element.sort = None - elif db_element.name != add_element: - db_element.name = add_element - return db_element - - -# Modifies different Database objects, first check if elements have to be deleted, -# because they are no longer used, than check if elements have to be added to database -def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type): - # passing input_elements not as a list may lead to undesired results - if not isinstance(input_elements, list): - raise TypeError(str(input_elements) + " should be passed as a list") - input_elements = [x for x in input_elements if x != ''] - # we have all input element (authors, series, tags) names now - # 1. search for elements to remove - del_elements = search_objects_remove(db_book_object, db_type, input_elements) - # 2. search for elements that need to be added - add_elements = search_objects_add(db_book_object, db_type, input_elements) - # if there are elements to remove, we remove them now - changed = remove_objects(db_book_object, db_session, del_elements) - # if there are elements to add, we add them now! - if len(add_elements) > 0: - changed |= add_objects(db_book_object, db_object, db_session, db_type, add_elements) - return changed - - -def modify_identifiers(input_identifiers, db_identifiers, db_session): - """Modify Identifiers to match input information. - input_identifiers is a list of read-to-persist Identifiers objects. - db_identifiers is a list of already persisted list of Identifiers objects.""" - changed = False - error = False - input_dict = dict([(identifier.type.lower(), identifier) for identifier in input_identifiers]) - if len(input_identifiers) != len(input_dict): - error = True - db_dict = dict([(identifier.type.lower(), identifier) for identifier in db_identifiers]) - # delete db identifiers not present in input or modify them with input val - for identifier_type, identifier in db_dict.items(): - if identifier_type not in input_dict.keys(): - db_session.delete(identifier) - changed = True - else: - input_identifier = input_dict[identifier_type] - identifier.type = input_identifier.type - identifier.val = input_identifier.val - # add input identifiers not present in db - for identifier_type, identifier in input_dict.items(): - if identifier_type not in db_dict.keys(): - db_session.add(identifier) - changed = True - return changed, error - - @editbook.route("/ajax/delete/", methods=["POST"]) @login_required def delete_book_from_details(book_id): @@ -242,6 +86,677 @@ def delete_book_ajax(book_id, book_format): return delete_book_from_table(book_id, book_format, False) +@editbook.route("/admin/book/", methods=['GET']) +@login_required_if_no_ano +@edit_required +def show_edit_book(book_id): + return render_edit_book(book_id) + + +@editbook.route("/admin/book/", methods=['POST']) +@login_required_if_no_ano +@edit_required +def edit_book(book_id): + modify_date = False + edit_error = False + + # create the function for sorting... + calibre_db.update_title_sort(config) + + book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) + # Book not found + if not book: + flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), + category="error") + return redirect(url_for("web.index")) + + to_save = request.form.to_dict() + + try: + # Update folder of book on local disk + edited_books_id = None + title_author_error = None + # handle book title change + title_change = handle_title_on_edit(book, to_save["book_title"]) + # handle book author change + input_authors, author_change, renamed = handle_author_on_edit(book, to_save["author_name"]) + if author_change or title_change: + edited_books_id = book.id + modify_date = True + title_author_error = helper.update_dir_structure(edited_books_id, + config.config_calibre_dir, + input_authors[0], + renamed_author=renamed) + if title_author_error: + flash(title_author_error, category="error") + calibre_db.session.rollback() + book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) + + # handle upload other formats from local disk + meta = upload_single_file(request, book, book_id) + # only merge metadata if file was uploaded and no error occurred (meta equals not false or none) + if meta: + merge_metadata(to_save, meta) + # handle upload covers from local disk + cover_upload_success = upload_cover(request, book) + if cover_upload_success: + book.has_cover = 1 + modify_date = True + + # upload new covers or new file formats to google drive + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + + if to_save.get("cover_url", None): + if not current_user.role_upload(): + edit_error = True + flash(_(u"User has no rights to upload cover"), category="error") + if to_save["cover_url"].endswith('/static/generic_cover.jpg'): + book.has_cover = 0 + else: + result, error = helper.save_cover_from_url(to_save["cover_url"].strip(), book.path) + if result is True: + book.has_cover = 1 + modify_date = True + helper.replace_cover_thumbnail_cache(book.id) + else: + flash(error, category="error") + + # Add default series_index to book + modify_date |= edit_book_series_index(to_save["series_index"], book) + # Handle book comments/description + modify_date |= edit_book_comments(Markup(to_save['description']).unescape(), book) + # Handle identifiers + input_identifiers = identifier_list(to_save, book) + modification, warning = modify_identifiers(input_identifiers, book.identifiers, calibre_db.session) + if warning: + flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning") + modify_date |= modification + # Handle book tags + modify_date |= edit_book_tags(to_save['tags'], book) + # Handle book series + modify_date |= edit_book_series(to_save["series"], book) + # handle book publisher + modify_date |= edit_book_publisher(to_save['publisher'], book) + # handle book languages + try: + modify_date |= edit_book_languages(to_save['languages'], book) + except ValueError as e: + flash(str(e), category="error") + edit_error = True + # handle book ratings + modify_date |= edit_book_ratings(to_save, book) + # handle cc data + modify_date |= edit_all_cc_data(book_id, book, to_save) + + if to_save.get("pubdate", None): + try: + book.pubdate = datetime.strptime(to_save["pubdate"], "%Y-%m-%d") + except ValueError as e: + book.pubdate = db.Books.DEFAULT_PUBDATE + flash(str(e), category="error") + edit_error = True + else: + book.pubdate = db.Books.DEFAULT_PUBDATE + + if modify_date: + book.last_modified = datetime.utcnow() + kobo_sync_status.remove_synced_book(edited_books_id, all=True) + + calibre_db.session.merge(book) + calibre_db.session.commit() + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + if meta is not False \ + and edit_error is not True \ + and title_author_error is not True \ + and cover_upload_success is not False: + flash(_("Metadata successfully updated"), category="success") + if "detail_view" in to_save: + return redirect(url_for('web.show_book', book_id=book.id)) + else: + return render_edit_book(book_id) + except ValueError as e: + log.error_or_exception("Error: {}".format(e)) + calibre_db.session.rollback() + flash(str(e), category="error") + return redirect(url_for('web.show_book', book_id=book.id)) + except (OperationalError, IntegrityError) as e: + log.error_or_exception("Database error: {}".format(e)) + calibre_db.session.rollback() + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") + return redirect(url_for('web.show_book', book_id=book.id)) + except Exception as ex: + log.error_or_exception(ex) + calibre_db.session.rollback() + flash(_("Error editing book: {}".format(ex)), category="error") + return redirect(url_for('web.show_book', book_id=book.id)) + + +@editbook.route("/upload", methods=["POST"]) +@login_required_if_no_ano +@upload_required +def upload(): + if not config.config_uploading: + abort(404) + if request.method == 'POST' and 'btn-upload' in request.files: + for requested_file in request.files.getlist("btn-upload"): + try: + modify_date = False + # create the function for sorting... + calibre_db.update_title_sort(config) + calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) + + meta, error = file_handling_on_upload(requested_file) + if error: + return error + + db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta) + + # Comments need book id therefore only possible after flush + modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book) + + book_id = db_book.id + title = db_book.title + if config.config_use_google_drive: + helper.upload_new_file_gdrive(book_id, + input_authors[0], + renamed_authors, + title, + title_dir, + meta.file_path, + meta.extension.lower()) + else: + error = helper.update_dir_structure(book_id, + config.config_calibre_dir, + input_authors[0], + meta.file_path, + title_dir + meta.extension.lower(), + renamed_author=renamed_authors) + + move_coverfile(meta, db_book) + + # save data to database, reread data + calibre_db.session.commit() + + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + if error: + flash(error, category="error") + link = '{}'.format(url_for('web.show_book', book_id=book_id), escape(title)) + upload_text = N_(u"File %(file)s uploaded", file=link) + WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(title))) + helper.add_book_to_thumbnail_cache(book_id) + + if len(request.files.getlist("btn-upload")) < 2: + if current_user.role_edit() or current_user.role_admin(): + resp = {"location": url_for('edit-book.show_edit_book', book_id=book_id)} + return Response(json.dumps(resp), mimetype='application/json') + else: + resp = {"location": url_for('web.show_book', book_id=book_id)} + return Response(json.dumps(resp), mimetype='application/json') + except (OperationalError, IntegrityError) as e: + calibre_db.session.rollback() + log.error_or_exception("Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") + return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + + +@editbook.route("/admin/book/convert/", methods=['POST']) +@login_required_if_no_ano +@edit_required +def convert_bookformat(book_id): + # check to see if we have form fields to work with - if not send user back + book_format_from = request.form.get('book_format_from', None) + book_format_to = request.form.get('book_format_to', None) + + if (book_format_from is None) or (book_format_to is None): + flash(_(u"Source or destination format for conversion missing"), category="error") + return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) + + log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to) + rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(), + book_format_to.upper(), current_user.name) + + if rtn is None: + flash(_(u"Book successfully queued for converting to %(book_format)s", + book_format=book_format_to), + category="success") + else: + flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error") + return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) + + +@editbook.route("/ajax/getcustomenum/") +@login_required +def table_get_custom_enum(c_id): + ret = list() + cc = (calibre_db.session.query(db.CustomColumns) + .filter(db.CustomColumns.id == c_id) + .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)).one_or_none()) + ret.append({'value': "", 'text': ""}) + for idx, en in enumerate(cc.get_display_dict()['enum_values']): + ret.append({'value': en, 'text': en}) + return json.dumps(ret) + + +@editbook.route("/ajax/editbooks/", methods=['POST']) +@login_required_if_no_ano +@edit_required +def edit_list_book(param): + vals = request.form.to_dict() + book = calibre_db.get_book(vals['pk']) + sort_param = "" + ret = "" + try: + if param == 'series_index': + edit_book_series_index(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': book.series_index}), mimetype='application/json') + elif param == 'tags': + edit_book_tags(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': ', '.join([tag.name for tag in book.tags])}), + mimetype='application/json') + elif param == 'series': + edit_book_series(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': ', '.join([serie.name for serie in book.series])}), + mimetype='application/json') + elif param == 'publishers': + edit_book_publisher(vals['value'], book) + ret = Response(json.dumps({'success': True, + 'newValue': ', '.join([publisher.name for publisher in book.publishers])}), + mimetype='application/json') + elif param == 'languages': + invalid = list() + edit_book_languages(vals['value'], book, invalid=invalid) + if invalid: + ret = Response(json.dumps({'success': False, + 'msg': 'Invalid languages in request: {}'.format(','.join(invalid))}), + mimetype='application/json') + else: + lang_names = list() + for lang in book.languages: + lang_names.append(isoLanguages.get_language_name(get_locale(), lang.lang_code)) + ret = Response(json.dumps({'success': True, 'newValue': ', '.join(lang_names)}), + mimetype='application/json') + elif param == 'author_sort': + book.author_sort = vals['value'] + ret = Response(json.dumps({'success': True, 'newValue': book.author_sort}), + mimetype='application/json') + elif param == 'title': + sort_param = book.sort + if handle_title_on_edit(book, vals.get('value', "")): + rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir) + if not rename_error: + ret = Response(json.dumps({'success': True, 'newValue': book.title}), + mimetype='application/json') + else: + ret = Response(json.dumps({'success': False, + 'msg': rename_error}), + mimetype='application/json') + elif param == 'sort': + book.sort = vals['value'] + ret = Response(json.dumps({'success': True, 'newValue': book.sort}), + mimetype='application/json') + elif param == 'comments': + edit_book_comments(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': book.comments[0].text}), + mimetype='application/json') + elif param == 'authors': + input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") + rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir, input_authors[0], + renamed_author=renamed) + if not rename_error: + ret = Response(json.dumps({ + 'success': True, + 'newValue': ' & '.join([author.replace('|', ',') for author in input_authors])}), + mimetype='application/json') + else: + ret = Response(json.dumps({'success': False, + 'msg': rename_error}), + mimetype='application/json') + elif param == 'is_archived': + is_archived = change_archived_books(book.id, vals['value'] == "True", + message="Book {} archive bit set to: {}".format(book.id, vals['value'])) + if is_archived: + kobo_sync_status.remove_synced_book(book.id) + return "" + elif param == 'read_status': + ret = helper.edit_book_read_status(book.id, vals['value'] == "True") + if ret: + return ret, 400 + elif param.startswith("custom_column_"): + new_val = dict() + new_val[param] = vals['value'] + edit_single_cc_data(book.id, book, param[14:], new_val) + # ToDo: Very hacky find better solution + if vals['value'] in ["True", "False"]: + ret = "" + else: + ret = Response(json.dumps({'success': True, 'newValue': vals['value']}), + mimetype='application/json') + else: + return _("Parameter not found"), 400 + book.last_modified = datetime.utcnow() + + calibre_db.session.commit() + # revert change for sort if automatic fields link is deactivated + if param == 'title' and vals.get('checkT') == "false": + book.sort = sort_param + calibre_db.session.commit() + except (OperationalError, IntegrityError) as e: + calibre_db.session.rollback() + log.error_or_exception("Database error: {}".format(e)) + ret = Response(json.dumps({'success': False, + 'msg': 'Database error: {}'.format(e.orig)}), + mimetype='application/json') + return ret + + +@editbook.route("/ajax/sort_value//") +@login_required +def get_sorted_entry(field, bookid): + if field in ['title', 'authors', 'sort', 'author_sort']: + book = calibre_db.get_filtered_book(bookid) + if book: + if field == 'title': + return json.dumps({'sort': book.sort}) + elif field == 'authors': + return json.dumps({'author_sort': book.author_sort}) + if field == 'sort': + return json.dumps({'sort': book.title}) + if field == 'author_sort': + return json.dumps({'author_sort': book.author}) + return "" + + +@editbook.route("/ajax/simulatemerge", methods=['POST']) +@login_required +@edit_required +def simulate_merge_list_book(): + vals = request.get_json().get('Merge_books') + if vals: + to_book = calibre_db.get_book(vals[0]).title + vals.pop(0) + if to_book: + from_book = [] + for book_id in vals: + from_book.append(calibre_db.get_book(book_id).title) + return json.dumps({'to': to_book, 'from': from_book}) + return "" + + +@editbook.route("/ajax/mergebooks", methods=['POST']) +@login_required +@edit_required +def merge_list_book(): + vals = request.get_json().get('Merge_books') + to_file = list() + if vals: + # load all formats from target book + to_book = calibre_db.get_book(vals[0]) + vals.pop(0) + if to_book: + for file in to_book.data: + to_file.append(file.format) + to_name = helper.get_valid_filename(to_book.title, + chars=96) + ' - ' + helper.get_valid_filename(to_book.authors[0].name, + chars=96) + for book_id in vals: + from_book = calibre_db.get_book(book_id) + if from_book: + for element in from_book.data: + if element.format not in to_file: + # create new data entry with: book_id, book_format, uncompressed_size, name + filepath_new = os.path.normpath(os.path.join(config.config_calibre_dir, + to_book.path, + to_name + "." + element.format.lower())) + filepath_old = os.path.normpath(os.path.join(config.config_calibre_dir, + from_book.path, + element.name + "." + element.format.lower())) + copyfile(filepath_old, filepath_new) + to_book.data.append(db.Data(to_book.id, + element.format, + element.uncompressed_size, + to_name)) + delete_book_from_table(from_book.id, "", True) + return json.dumps({'success': True}) + return "" + + +@editbook.route("/ajax/xchange", methods=['POST']) +@login_required +@edit_required +def table_xchange_author_title(): + vals = request.get_json().get('xchange') + edited_books_id = False + if vals: + for val in vals: + modify_date = False + book = calibre_db.get_book(val) + authors = book.title + book.authors = calibre_db.order_authors([book]) + author_names = [] + for authr in book.authors: + author_names.append(authr.name.replace('|', ',')) + + title_change = handle_title_on_edit(book, " ".join(author_names)) + input_authors, author_change, renamed = handle_author_on_edit(book, authors) + if author_change or title_change: + edited_books_id = book.id + modify_date = True + + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + + if edited_books_id: + # toDo: Handle error + edit_error = helper.update_dir_structure(edited_books_id, config.config_calibre_dir, input_authors[0], + renamed_author=renamed) + if modify_date: + book.last_modified = datetime.utcnow() + try: + calibre_db.session.commit() + except (OperationalError, IntegrityError) as e: + calibre_db.session.rollback() + log.error_or_exception("Database error: %s", e) + return json.dumps({'success': False}) + + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + return json.dumps({'success': True}) + return "" + + +def merge_metadata(to_save, meta): + if to_save.get('author_name', "") == _(u'Unknown'): + to_save['author_name'] = '' + if to_save.get('book_title', "") == _(u'Unknown'): + to_save['book_title'] = '' + for s_field, m_field in [ + ('tags', 'tags'), ('author_name', 'author'), ('series', 'series'), + ('series_index', 'series_id'), ('languages', 'languages'), + ('book_title', 'title')]: + to_save[s_field] = to_save[s_field] or getattr(meta, m_field, '') + to_save["description"] = to_save["description"] or Markup( + getattr(meta, 'description', '')).unescape() + + +def identifier_list(to_save, book): + """Generate a list of Identifiers from form information""" + id_type_prefix = 'identifier-type-' + id_val_prefix = 'identifier-val-' + result = [] + for type_key, type_value in to_save.items(): + if not type_key.startswith(id_type_prefix): + continue + val_key = id_val_prefix + type_key[len(id_type_prefix):] + if val_key not in to_save.keys(): + continue + result.append(db.Identifiers(to_save[val_key], type_value, book.id)) + return result + + +def prepare_authors(authr): + # handle authors + input_authors = authr.split('&') + # handle_authors(input_authors) + input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors)) + # Remove duplicates in authors list + input_authors = helper.uniq(input_authors) + + # we have all author names now + if input_authors == ['']: + input_authors = [_(u'Unknown')] # prevent empty Author + + renamed = list() + for in_aut in input_authors: + renamed_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == in_aut).first() + if renamed_author and in_aut != renamed_author.name: + renamed.append(renamed_author.name) + all_books = calibre_db.session.query(db.Books) \ + .filter(db.Books.authors.any(db.Authors.name == renamed_author.name)).all() + sorted_renamed_author = helper.get_sorted_author(renamed_author.name) + sorted_old_author = helper.get_sorted_author(in_aut) + for one_book in all_books: + one_book.author_sort = one_book.author_sort.replace(sorted_renamed_author, sorted_old_author) + return input_authors, renamed + + +def prepare_authors_on_upload(title, authr): + if title != _(u'Unknown') and authr != _(u'Unknown'): + entry = calibre_db.check_exists_book(authr, title) + if entry: + log.info("Uploaded book probably exists in library") + flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ") + + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning") + + input_authors, renamed = prepare_authors(authr) + + sort_authors_list = list() + db_author = None + for inp in input_authors: + stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first() + if not stored_author: + if not db_author: + db_author = db.Authors(inp, helper.get_sorted_author(inp), "") + calibre_db.session.add(db_author) + calibre_db.session.commit() + sort_author = helper.get_sorted_author(inp) + else: + if not db_author: + db_author = stored_author + sort_author = stored_author.sort + sort_authors_list.append(sort_author) + sort_authors = ' & '.join(sort_authors_list) + return sort_authors, input_authors, db_author, renamed + + +def create_book_on_upload(modify_date, meta): + title = meta.title + authr = meta.author + sort_authors, input_authors, db_author, renamed_authors = prepare_authors_on_upload(title, authr) + + title_dir = helper.get_valid_filename(title, chars=96) + author_dir = helper.get_valid_filename(db_author.name, chars=96) + + # combine path and normalize path from Windows systems + path = os.path.join(author_dir, title_dir).replace('\\', '/') + + try: + pubdate = datetime.strptime(meta.pubdate[:10], "%Y-%m-%d") + except ValueError: + pubdate = datetime(101, 1, 1) + + # Calibre adds books with utc as timezone + db_book = db.Books(title, "", sort_authors, datetime.utcnow(), pubdate, + '1', datetime.utcnow(), path, meta.cover, db_author, [], "") + + modify_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session, + 'author') + + # Add series_index to book + modify_date |= edit_book_series_index(meta.series_id, db_book) + + # add languages + invalid = [] + modify_date |= edit_book_languages(meta.languages, db_book, upload_mode=True, invalid=invalid) + if invalid: + for lang in invalid: + flash(_(u"'%(langname)s' is not a valid language", langname=lang), category="warning") + + # handle tags + modify_date |= edit_book_tags(meta.tags, db_book) + + # handle publisher + modify_date |= edit_book_publisher(meta.publisher, db_book) + + # handle series + modify_date |= edit_book_series(meta.series, db_book) + + # Add file to book + file_size = os.path.getsize(meta.file_path) + db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir) + db_book.data.append(db_data) + calibre_db.session.add(db_book) + + # flush content, get db_book.id available + calibre_db.session.flush() + + # Handle identifiers now that db_book.id is available + identifier_list = [] + for type_key, type_value in meta.identifiers: + identifier_list.append(db.Identifiers(type_value, type_key, db_book.id)) + modification, warning = modify_identifiers(identifier_list, db_book.identifiers, calibre_db.session) + if warning: + flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning") + modify_date |= modification + + return db_book, input_authors, title_dir, renamed_authors + + +def file_handling_on_upload(requested_file): + # check if file extension is correct + if '.' in requested_file.filename: + file_ext = requested_file.filename.rsplit('.', 1)[-1].lower() + if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD: + flash( + _("File extension '%(ext)s' is not allowed to be uploaded to this server", + ext=file_ext), category="error") + return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + else: + flash(_('File to be uploaded must have an extension'), category="error") + return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + + # extract metadata from file + try: + meta = uploader.upload(requested_file, config.config_rarfile_location) + except (IOError, OSError): + log.error("File %s could not saved to temp dir", requested_file.filename) + flash(_(u"File %(filename)s could not saved to temp dir", + filename=requested_file.filename), category="error") + return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + return meta, None + + +def move_coverfile(meta, db_book): + # move cover to final directory, including book id + if meta.cover: + cover_file = meta.cover + else: + cover_file = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg') + new_cover_path = os.path.join(config.config_calibre_dir, db_book.path) + try: + os.makedirs(new_cover_path, exist_ok=True) + copyfile(cover_file, os.path.join(new_cover_path, "cover.jpg")) + if meta.cover: + os.unlink(meta.cover) + except OSError as e: + log.error("Failed to move cover file %s: %s", new_cover_path, e) + flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_cover_path, + error=e), + category="error") + + def delete_whole_book(book_id, book): # delete book from shelves, Downloads, Read list ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete() @@ -744,676 +1259,156 @@ def handle_author_on_edit(book, author_name, update_stored=True): return input_authors, change, renamed -@editbook.route("/admin/book/", methods=['GET']) -@login_required_if_no_ano -@edit_required -def show_edit_book(book_id): - return render_edit_book(book_id) +def search_objects_remove(db_book_object, db_type, input_elements): + del_elements = [] + for c_elements in db_book_object: + found = False + if db_type == 'languages': + type_elements = c_elements.lang_code + elif db_type == 'custom': + type_elements = c_elements.value + else: + type_elements = c_elements.name + for inp_element in input_elements: + if inp_element.lower() == type_elements.lower(): + found = True + break + # if the element was not found in the new list, add it to remove list + if not found: + del_elements.append(c_elements) + return del_elements -@editbook.route("/admin/book/", methods=['POST']) -@login_required_if_no_ano -@edit_required -def edit_book(book_id): - modify_date = False - edit_error = False - - # create the function for sorting... - #try: - calibre_db.update_title_sort(config) - #except sqliteOperationalError as e: - # log.error_or_exception(e) - # calibre_db.session.rollback() - - book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - # Book not found - if not book: - flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), - category="error") - return redirect(url_for("web.index")) - - to_save = request.form.to_dict() - - try: - # Update folder of book on local disk - edited_books_id = None - title_author_error = None - # handle book title change - title_change = handle_title_on_edit(book, to_save["book_title"]) - # handle book author change - input_authors, author_change, renamed = handle_author_on_edit(book, to_save["author_name"]) - if author_change or title_change: - edited_books_id = book.id - modify_date = True - title_author_error = helper.update_dir_structure(edited_books_id, - config.config_calibre_dir, - input_authors[0], - renamed_author=renamed) - if title_author_error: - flash(title_author_error, category="error") - calibre_db.session.rollback() - book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - - # handle upload other formats from local disk - meta = upload_single_file(request, book, book_id) - # only merge metadata if file was uploaded and no error occurred (meta equals not false or none) - if meta: - merge_metadata(to_save, meta) - # handle upload covers from local disk - cover_upload_success = upload_cover(request, book) - if cover_upload_success: - book.has_cover = 1 - modify_date = True - - # upload new covers or new file formats to google drive - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - - if to_save.get("cover_url", None): - if not current_user.role_upload(): - edit_error = True - flash(_(u"User has no rights to upload cover"), category="error") - if to_save["cover_url"].endswith('/static/generic_cover.jpg'): - book.has_cover = 0 +def search_objects_add(db_book_object, db_type, input_elements): + add_elements = [] + for inp_element in input_elements: + found = False + for c_elements in db_book_object: + if db_type == 'languages': + type_elements = c_elements.lang_code + elif db_type == 'custom': + type_elements = c_elements.value else: - result, error = helper.save_cover_from_url(to_save["cover_url"].strip(), book.path) - if result is True: - book.has_cover = 1 - modify_date = True - helper.replace_cover_thumbnail_cache(book.id) - else: - flash(error, category="error") - - # Add default series_index to book - modify_date |= edit_book_series_index(to_save["series_index"], book) - # Handle book comments/description - modify_date |= edit_book_comments(Markup(to_save['description']).unescape(), book) - # Handle identifiers - input_identifiers = identifier_list(to_save, book) - modification, warning = modify_identifiers(input_identifiers, book.identifiers, calibre_db.session) - if warning: - flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning") - modify_date |= modification - # Handle book tags - modify_date |= edit_book_tags(to_save['tags'], book) - # Handle book series - modify_date |= edit_book_series(to_save["series"], book) - # handle book publisher - modify_date |= edit_book_publisher(to_save['publisher'], book) - # handle book languages - try: - modify_date |= edit_book_languages(to_save['languages'], book) - except ValueError as e: - flash(str(e), category="error") - edit_error = True - # handle book ratings - modify_date |= edit_book_ratings(to_save, book) - # handle cc data - modify_date |= edit_all_cc_data(book_id, book, to_save) - - if to_save.get("pubdate", None): - try: - book.pubdate = datetime.strptime(to_save["pubdate"], "%Y-%m-%d") - except ValueError as e: - book.pubdate = db.Books.DEFAULT_PUBDATE - flash(str(e), category="error") - edit_error = True - else: - book.pubdate = db.Books.DEFAULT_PUBDATE - - if modify_date: - book.last_modified = datetime.utcnow() - kobo_sync_status.remove_synced_book(edited_books_id, all=True) - - calibre_db.session.merge(book) - calibre_db.session.commit() - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - if meta is not False \ - and edit_error is not True \ - and title_author_error is not True \ - and cover_upload_success is not False: - flash(_("Metadata successfully updated"), category="success") - if "detail_view" in to_save: - return redirect(url_for('web.show_book', book_id=book.id)) - else: - return render_edit_book(book_id) - except ValueError as e: - log.error_or_exception("Error: {}".format(e)) - calibre_db.session.rollback() - flash(str(e), category="error") - return redirect(url_for('web.show_book', book_id=book.id)) - except (OperationalError, IntegrityError) as e: - log.error_or_exception("Database error: {}".format(e)) - calibre_db.session.rollback() - flash(_(u"Database error: %(error)s.", error=e.orig), category="error") - return redirect(url_for('web.show_book', book_id=book.id)) - except Exception as ex: - log.error_or_exception(ex) - calibre_db.session.rollback() - flash(_("Error editing book: {}".format(ex)), category="error") - return redirect(url_for('web.show_book', book_id=book.id)) + type_elements = c_elements.name + if inp_element == type_elements: + found = True + break + if not found: + add_elements.append(inp_element) + return add_elements -def merge_metadata(to_save, meta): - if to_save.get('author_name', "") == _(u'Unknown'): - to_save['author_name'] = '' - if to_save.get('book_title', "") == _(u'Unknown'): - to_save['book_title'] = '' - for s_field, m_field in [ - ('tags', 'tags'), ('author_name', 'author'), ('series', 'series'), - ('series_index', 'series_id'), ('languages', 'languages'), - ('book_title', 'title')]: - to_save[s_field] = to_save[s_field] or getattr(meta, m_field, '') - to_save["description"] = to_save["description"] or Markup( - getattr(meta, 'description', '')).unescape() +def remove_objects(db_book_object, db_session, del_elements): + changed = False + if len(del_elements) > 0: + for del_element in del_elements: + db_book_object.remove(del_element) + changed = True + if len(del_element.books) == 0: + db_session.delete(del_element) + return changed -def identifier_list(to_save, book): - """Generate a list of Identifiers from form information""" - id_type_prefix = 'identifier-type-' - id_val_prefix = 'identifier-val-' - result = [] - for type_key, type_value in to_save.items(): - if not type_key.startswith(id_type_prefix): - continue - val_key = id_val_prefix + type_key[len(id_type_prefix):] - if val_key not in to_save.keys(): - continue - result.append(db.Identifiers(to_save[val_key], type_value, book.id)) - return result - - -def prepare_authors(authr): - # handle authors - input_authors = authr.split('&') - # handle_authors(input_authors) - input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors)) - # Remove duplicates in authors list - input_authors = helper.uniq(input_authors) - - # we have all author names now - if input_authors == ['']: - input_authors = [_(u'Unknown')] # prevent empty Author - - renamed = list() - for in_aut in input_authors: - renamed_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == in_aut).first() - if renamed_author and in_aut != renamed_author.name: - renamed.append(renamed_author.name) - all_books = calibre_db.session.query(db.Books) \ - .filter(db.Books.authors.any(db.Authors.name == renamed_author.name)).all() - sorted_renamed_author = helper.get_sorted_author(renamed_author.name) - sorted_old_author = helper.get_sorted_author(in_aut) - for one_book in all_books: - one_book.author_sort = one_book.author_sort.replace(sorted_renamed_author, sorted_old_author) - return input_authors, renamed - - -def prepare_authors_on_upload(title, authr): - if title != _(u'Unknown') and authr != _(u'Unknown'): - entry = calibre_db.check_exists_book(authr, title) - if entry: - log.info("Uploaded book probably exists in library") - flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ") - + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning") - - input_authors, renamed = prepare_authors(authr) - - sort_authors_list = list() - db_author = None - for inp in input_authors: - stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first() - if not stored_author: - if not db_author: - db_author = db.Authors(inp, helper.get_sorted_author(inp), "") - calibre_db.session.add(db_author) - calibre_db.session.commit() - sort_author = helper.get_sorted_author(inp) - else: - if not db_author: - db_author = stored_author - sort_author = stored_author.sort - sort_authors_list.append(sort_author) - sort_authors = ' & '.join(sort_authors_list) - return sort_authors, input_authors, db_author, renamed - - -def create_book_on_upload(modify_date, meta): - title = meta.title - authr = meta.author - sort_authors, input_authors, db_author, renamed_authors = prepare_authors_on_upload(title, authr) - - title_dir = helper.get_valid_filename(title, chars=96) - author_dir = helper.get_valid_filename(db_author.name, chars=96) - - # combine path and normalize path from Windows systems - path = os.path.join(author_dir, title_dir).replace('\\', '/') - - try: - pubdate = datetime.strptime(meta.pubdate[:10], "%Y-%m-%d") - except ValueError: - pubdate = datetime(101, 1, 1) - - # Calibre adds books with utc as timezone - db_book = db.Books(title, "", sort_authors, datetime.utcnow(), pubdate, - '1', datetime.utcnow(), path, meta.cover, db_author, [], "") - - modify_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session, - 'author') - - # Add series_index to book - modify_date |= edit_book_series_index(meta.series_id, db_book) - - # add languages - invalid = [] - modify_date |= edit_book_languages(meta.languages, db_book, upload_mode=True, invalid=invalid) - if invalid: - for lang in invalid: - flash(_(u"'%(langname)s' is not a valid language", langname=lang), category="warning") - - # handle tags - modify_date |= edit_book_tags(meta.tags, db_book) - - # handle publisher - modify_date |= edit_book_publisher(meta.publisher, db_book) - - # handle series - modify_date |= edit_book_series(meta.series, db_book) - - # Add file to book - file_size = os.path.getsize(meta.file_path) - db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir) - db_book.data.append(db_data) - calibre_db.session.add(db_book) - - # flush content, get db_book.id available - calibre_db.session.flush() - - # Handle identifiers now that db_book.id is available - identifier_list = [] - for type_key, type_value in meta.identifiers: - identifier_list.append(db.Identifiers(type_value, type_key, db_book.id)) - modification, warning = modify_identifiers(identifier_list, db_book.identifiers, calibre_db.session) - if warning: - flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning") - modify_date |= modification - - return db_book, input_authors, title_dir, renamed_authors - - -def file_handling_on_upload(requested_file): - # check if file extension is correct - if '.' in requested_file.filename: - file_ext = requested_file.filename.rsplit('.', 1)[-1].lower() - if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD: - flash( - _("File extension '%(ext)s' is not allowed to be uploaded to this server", - ext=file_ext), category="error") - return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') +def add_objects(db_book_object, db_object, db_session, db_type, add_elements): + changed = False + if db_type == 'languages': + db_filter = db_object.lang_code + elif db_type == 'custom': + db_filter = db_object.value else: - flash(_('File to be uploaded must have an extension'), category="error") - return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - - # extract metadata from file - try: - meta = uploader.upload(requested_file, config.config_rarfile_location) - except (IOError, OSError): - log.error("File %s could not saved to temp dir", requested_file.filename) - flash(_(u"File %(filename)s could not saved to temp dir", - filename=requested_file.filename), category="error") - return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - return meta, None - - -def move_coverfile(meta, db_book): - # move cover to final directory, including book id - if meta.cover: - cover_file = meta.cover - else: - cover_file = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg') - new_cover_path = os.path.join(config.config_calibre_dir, db_book.path) - try: - os.makedirs(new_cover_path, exist_ok=True) - copyfile(cover_file, os.path.join(new_cover_path, "cover.jpg")) - if meta.cover: - os.unlink(meta.cover) - except OSError as e: - log.error("Failed to move cover file %s: %s", new_cover_path, e) - flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_cover_path, - error=e), - category="error") - - -@editbook.route("/upload", methods=["POST"]) -@login_required_if_no_ano -@upload_required -def upload(): - if not config.config_uploading: - abort(404) - if request.method == 'POST' and 'btn-upload' in request.files: - for requested_file in request.files.getlist("btn-upload"): - try: - modify_date = False - # create the function for sorting... - calibre_db.update_title_sort(config) - calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) - - meta, error = file_handling_on_upload(requested_file) - if error: - return error - - db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta) - - # Comments need book id therefore only possible after flush - modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book) - - book_id = db_book.id - title = db_book.title - if config.config_use_google_drive: - helper.upload_new_file_gdrive(book_id, - input_authors[0], - renamed_authors, - title, - title_dir, - meta.file_path, - meta.extension.lower()) - else: - error = helper.update_dir_structure(book_id, - config.config_calibre_dir, - input_authors[0], - meta.file_path, - title_dir + meta.extension.lower(), - renamed_author=renamed_authors) - - move_coverfile(meta, db_book) - - # save data to database, reread data - calibre_db.session.commit() - - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - if error: - flash(error, category="error") - link = '{}'.format(url_for('web.show_book', book_id=book_id), escape(title)) - upload_text = N_(u"File %(file)s uploaded", file=link) - WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(title))) - helper.add_book_to_thumbnail_cache(book_id) - - if len(request.files.getlist("btn-upload")) < 2: - if current_user.role_edit() or current_user.role_admin(): - resp = {"location": url_for('edit-book.show_edit_book', book_id=book_id)} - return Response(json.dumps(resp), mimetype='application/json') - else: - resp = {"location": url_for('web.show_book', book_id=book_id)} - return Response(json.dumps(resp), mimetype='application/json') - except (OperationalError, IntegrityError) as e: - calibre_db.session.rollback() - log.error_or_exception("Database error: {}".format(e)) - flash(_(u"Database error: %(error)s.", error=e.orig), category="error") - return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - - -@editbook.route("/admin/book/convert/", methods=['POST']) -@login_required_if_no_ano -@edit_required -def convert_bookformat(book_id): - # check to see if we have form fields to work with - if not send user back - book_format_from = request.form.get('book_format_from', None) - book_format_to = request.form.get('book_format_to', None) - - if (book_format_from is None) or (book_format_to is None): - flash(_(u"Source or destination format for conversion missing"), category="error") - return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) - - log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to) - rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(), - book_format_to.upper(), current_user.name) - - if rtn is None: - flash(_(u"Book successfully queued for converting to %(book_format)s", - book_format=book_format_to), - category="success") - else: - flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error") - return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) - - -@editbook.route("/ajax/getcustomenum/") -@login_required -def table_get_custom_enum(c_id): - ret = list() - cc = (calibre_db.session.query(db.CustomColumns) - .filter(db.CustomColumns.id == c_id) - .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)).one_or_none()) - ret.append({'value': "", 'text': ""}) - for idx, en in enumerate(cc.get_display_dict()['enum_values']): - ret.append({'value': en, 'text': en}) - return json.dumps(ret) - - -@editbook.route("/ajax/editbooks/", methods=['POST']) -@login_required_if_no_ano -@edit_required -def edit_list_book(param): - vals = request.form.to_dict() - book = calibre_db.get_book(vals['pk']) - sort_param = "" - ret = "" - try: - if param == 'series_index': - edit_book_series_index(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': book.series_index}), mimetype='application/json') - elif param == 'tags': - edit_book_tags(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': ', '.join([tag.name for tag in book.tags])}), - mimetype='application/json') - elif param == 'series': - edit_book_series(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': ', '.join([serie.name for serie in book.series])}), - mimetype='application/json') - elif param == 'publishers': - edit_book_publisher(vals['value'], book) - ret = Response(json.dumps({'success': True, - 'newValue': ', '.join([publisher.name for publisher in book.publishers])}), - mimetype='application/json') - elif param == 'languages': - invalid = list() - edit_book_languages(vals['value'], book, invalid=invalid) - if invalid: - ret = Response(json.dumps({'success': False, - 'msg': 'Invalid languages in request: {}'.format(','.join(invalid))}), - mimetype='application/json') - else: - lang_names = list() - for lang in book.languages: - lang_names.append(isoLanguages.get_language_name(get_locale(), lang.lang_code)) - ret = Response(json.dumps({'success': True, 'newValue': ', '.join(lang_names)}), - mimetype='application/json') - elif param == 'author_sort': - book.author_sort = vals['value'] - ret = Response(json.dumps({'success': True, 'newValue': book.author_sort}), - mimetype='application/json') - elif param == 'title': - sort_param = book.sort - if handle_title_on_edit(book, vals.get('value', "")): - rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir) - if not rename_error: - ret = Response(json.dumps({'success': True, 'newValue': book.title}), - mimetype='application/json') - else: - ret = Response(json.dumps({'success': False, - 'msg': rename_error}), - mimetype='application/json') - elif param == 'sort': - book.sort = vals['value'] - ret = Response(json.dumps({'success': True, 'newValue': book.sort}), - mimetype='application/json') - elif param == 'comments': - edit_book_comments(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': book.comments[0].text}), - mimetype='application/json') - elif param == 'authors': - input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") - rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir, input_authors[0], - renamed_author=renamed) - if not rename_error: - ret = Response(json.dumps({ - 'success': True, - 'newValue': ' & '.join([author.replace('|', ',') for author in input_authors])}), - mimetype='application/json') - else: - ret = Response(json.dumps({'success': False, - 'msg': rename_error}), - mimetype='application/json') - elif param == 'is_archived': - is_archived = change_archived_books(book.id, vals['value'] == "True", - message="Book {} archive bit set to: {}".format(book.id, vals['value'])) - if is_archived: - kobo_sync_status.remove_synced_book(book.id) - return "" - elif param == 'read_status': - ret = helper.edit_book_read_status(book.id, vals['value'] == "True") - if ret: - return ret, 400 - elif param.startswith("custom_column_"): - new_val = dict() - new_val[param] = vals['value'] - edit_single_cc_data(book.id, book, param[14:], new_val) - # ToDo: Very hacky find better solution - if vals['value'] in ["True", "False"]: - ret = "" - else: - ret = Response(json.dumps({'success': True, 'newValue': vals['value']}), - mimetype='application/json') + db_filter = db_object.name + for add_element in add_elements: + # check if an element with that name exists + db_element = db_session.query(db_object).filter(db_filter == add_element).first() + # if no element is found add it + if db_type == 'author': + new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')), "") + elif db_type == 'series': + new_element = db_object(add_element, add_element) + elif db_type == 'custom': + new_element = db_object(value=add_element) + elif db_type == 'publisher': + new_element = db_object(add_element, None) + else: # db_type should be tag or language + new_element = db_object(add_element) + if db_element is None: + changed = True + db_session.add(new_element) + db_book_object.append(new_element) else: - return _("Parameter not found"), 400 - book.last_modified = datetime.utcnow() - - calibre_db.session.commit() - # revert change for sort if automatic fields link is deactivated - if param == 'title' and vals.get('checkT') == "false": - book.sort = sort_param - calibre_db.session.commit() - except (OperationalError, IntegrityError) as e: - calibre_db.session.rollback() - log.error_or_exception("Database error: {}".format(e)) - ret = Response(json.dumps({'success': False, - 'msg': 'Database error: {}'.format(e.orig)}), - mimetype='application/json') - return ret + db_element = create_objects_for_addition(db_element, add_element, db_type) + # add element to book + changed = True + db_book_object.append(db_element) + return changed -@editbook.route("/ajax/sort_value//") -@login_required -def get_sorted_entry(field, bookid): - if field in ['title', 'authors', 'sort', 'author_sort']: - book = calibre_db.get_filtered_book(bookid) - if book: - if field == 'title': - return json.dumps({'sort': book.sort}) - elif field == 'authors': - return json.dumps({'author_sort': book.author_sort}) - if field == 'sort': - return json.dumps({'sort': book.title}) - if field == 'author_sort': - return json.dumps({'author_sort': book.author}) - return "" +def create_objects_for_addition(db_element, add_element, db_type): + if db_type == 'custom': + if db_element.value != add_element: + db_element.value = add_element + elif db_type == 'languages': + if db_element.lang_code != add_element: + db_element.lang_code = add_element + elif db_type == 'series': + if db_element.name != add_element: + db_element.name = add_element + db_element.sort = add_element + elif db_type == 'author': + if db_element.name != add_element: + db_element.name = add_element + db_element.sort = helper.get_sorted_author(add_element.replace('|', ',')) + elif db_type == 'publisher': + if db_element.name != add_element: + db_element.name = add_element + db_element.sort = None + elif db_element.name != add_element: + db_element.name = add_element + return db_element -@editbook.route("/ajax/simulatemerge", methods=['POST']) -@login_required -@edit_required -def simulate_merge_list_book(): - vals = request.get_json().get('Merge_books') - if vals: - to_book = calibre_db.get_book(vals[0]).title - vals.pop(0) - if to_book: - from_book = [] - for book_id in vals: - from_book.append(calibre_db.get_book(book_id).title) - return json.dumps({'to': to_book, 'from': from_book}) - return "" +# Modifies different Database objects, first check if elements have to be deleted, +# because they are no longer used, than check if elements have to be added to database +def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type): + # passing input_elements not as a list may lead to undesired results + if not isinstance(input_elements, list): + raise TypeError(str(input_elements) + " should be passed as a list") + input_elements = [x for x in input_elements if x != ''] + # we have all input element (authors, series, tags) names now + # 1. search for elements to remove + del_elements = search_objects_remove(db_book_object, db_type, input_elements) + # 2. search for elements that need to be added + add_elements = search_objects_add(db_book_object, db_type, input_elements) + # if there are elements to remove, we remove them now + changed = remove_objects(db_book_object, db_session, del_elements) + # if there are elements to add, we add them now! + if len(add_elements) > 0: + changed |= add_objects(db_book_object, db_object, db_session, db_type, add_elements) + return changed -@editbook.route("/ajax/mergebooks", methods=['POST']) -@login_required -@edit_required -def merge_list_book(): - vals = request.get_json().get('Merge_books') - to_file = list() - if vals: - # load all formats from target book - to_book = calibre_db.get_book(vals[0]) - vals.pop(0) - if to_book: - for file in to_book.data: - to_file.append(file.format) - to_name = helper.get_valid_filename(to_book.title, - chars=96) + ' - ' + helper.get_valid_filename(to_book.authors[0].name, - chars=96) - for book_id in vals: - from_book = calibre_db.get_book(book_id) - if from_book: - for element in from_book.data: - if element.format not in to_file: - # create new data entry with: book_id, book_format, uncompressed_size, name - filepath_new = os.path.normpath(os.path.join(config.config_calibre_dir, - to_book.path, - to_name + "." + element.format.lower())) - filepath_old = os.path.normpath(os.path.join(config.config_calibre_dir, - from_book.path, - element.name + "." + element.format.lower())) - copyfile(filepath_old, filepath_new) - to_book.data.append(db.Data(to_book.id, - element.format, - element.uncompressed_size, - to_name)) - delete_book_from_table(from_book.id, "", True) - return json.dumps({'success': True}) - return "" - - -@editbook.route("/ajax/xchange", methods=['POST']) -@login_required -@edit_required -def table_xchange_author_title(): - vals = request.get_json().get('xchange') - edited_books_id = False - if vals: - for val in vals: - modify_date = False - book = calibre_db.get_book(val) - authors = book.title - book.authors = calibre_db.order_authors([book]) - author_names = [] - for authr in book.authors: - author_names.append(authr.name.replace('|', ',')) - - title_change = handle_title_on_edit(book, " ".join(author_names)) - input_authors, author_change, renamed = handle_author_on_edit(book, authors) - if author_change or title_change: - edited_books_id = book.id - modify_date = True - - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - - if edited_books_id: - # toDo: Handle error - edit_error = helper.update_dir_structure(edited_books_id, config.config_calibre_dir, input_authors[0], - renamed_author=renamed) - if modify_date: - book.last_modified = datetime.utcnow() - try: - calibre_db.session.commit() - except (OperationalError, IntegrityError) as e: - calibre_db.session.rollback() - log.error_or_exception("Database error: %s", e) - return json.dumps({'success': False}) - - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - return json.dumps({'success': True}) - return "" +def modify_identifiers(input_identifiers, db_identifiers, db_session): + """Modify Identifiers to match input information. + input_identifiers is a list of read-to-persist Identifiers objects. + db_identifiers is a list of already persisted list of Identifiers objects.""" + changed = False + error = False + input_dict = dict([(identifier.type.lower(), identifier) for identifier in input_identifiers]) + if len(input_identifiers) != len(input_dict): + error = True + db_dict = dict([(identifier.type.lower(), identifier) for identifier in db_identifiers]) + # delete db identifiers not present in input or modify them with input val + for identifier_type, identifier in db_dict.items(): + if identifier_type not in input_dict.keys(): + db_session.delete(identifier) + changed = True + else: + input_identifier = input_dict[identifier_type] + identifier.type = input_identifier.type + identifier.val = input_identifier.val + # add input identifiers not present in db + for identifier_type, identifier in input_dict.items(): + if identifier_type not in db_dict.keys(): + db_session.add(identifier) + changed = True + return changed, error diff --git a/cps/helper.py b/cps/helper.py index aec14668..7c76d180 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -29,11 +29,9 @@ from tempfile import gettempdir import requests import unidecode - from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ from flask_babel import lazy_gettext as N_ -from flask_babel import format_datetime, get_locale from flask_login import current_user from sqlalchemy.sql.expression import true, false, and_, or_, text, func from sqlalchemy.exc import InvalidRequestError, OperationalError @@ -42,7 +40,6 @@ from werkzeug.security import generate_password_hash from markupsafe import escape from urllib.parse import quote - try: import advocate from advocate.exceptions import UnacceptableAddressException @@ -52,14 +49,13 @@ except ImportError: advocate = requests UnacceptableAddressException = MissingSchema = BaseException -from . import calibre_db, cli +from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES from .subproc_wrapper import process_wait -from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS, STAT_ENDED, \ - STAT_CANCELLED +from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -76,10 +72,10 @@ except (ImportError, RuntimeError) as e: # Convert existing book entry to new format -def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None): +def convert_book_format(book_id, calibre_path, old_book_format, new_book_format, user_id, kindle_mail=None): book = calibre_db.get_book(book_id) data = calibre_db.get_book_format(book.id, old_book_format) - file_path = os.path.join(calibrepath, book.path, data.name) + file_path = os.path.join(calibre_path, book.path, data.name) if not data: error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) log.error("convert_book_format: %s", error_message) @@ -144,20 +140,20 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False): def check_send_to_kindle_with_converter(formats): - bookformats = list() + book_formats = list() if 'EPUB' in formats and 'MOBI' not in formats: - bookformats.append({'format': 'Mobi', - 'convert': 1, - 'text': _('Convert %(orig)s to %(format)s and send to Kindle', - orig='Epub', - format='Mobi')}) + book_formats.append({'format': 'Mobi', + 'convert': 1, + 'text': _('Convert %(orig)s to %(format)s and send to Kindle', + orig='Epub', + format='Mobi')}) if 'AZW3' in formats and 'MOBI' not in formats: - bookformats.append({'format': 'Mobi', - 'convert': 2, - 'text': _('Convert %(orig)s to %(format)s and send to Kindle', - orig='Azw3', - format='Mobi')}) - return bookformats + book_formats.append({'format': 'Mobi', + 'convert': 2, + 'text': _('Convert %(orig)s to %(format)s and send to Kindle', + orig='Azw3', + format='Mobi')}) + return book_formats def check_send_to_kindle(entry): @@ -165,26 +161,26 @@ def check_send_to_kindle(entry): returns all available book formats for sending to Kindle """ formats = list() - bookformats = list() + book_formats = list() if len(entry.data): for ele in iter(entry.data): if ele.uncompressed_size < config.mail_size: formats.append(ele.format) if 'MOBI' in formats: - bookformats.append({'format': 'Mobi', - 'convert': 0, - 'text': _('Send %(format)s to Kindle', format='Mobi')}) + book_formats.append({'format': 'Mobi', + 'convert': 0, + 'text': _('Send %(format)s to Kindle', format='Mobi')}) if 'PDF' in formats: - bookformats.append({'format': 'Pdf', - 'convert': 0, - 'text': _('Send %(format)s to Kindle', format='Pdf')}) + book_formats.append({'format': 'Pdf', + 'convert': 0, + 'text': _('Send %(format)s to Kindle', format='Pdf')}) if 'AZW' in formats: - bookformats.append({'format': 'Azw', - 'convert': 0, - 'text': _('Send %(format)s to Kindle', format='Azw')}) + book_formats.append({'format': 'Azw', + 'convert': 0, + 'text': _('Send %(format)s to Kindle', format='Azw')}) if config.config_converterpath: - bookformats.extend(check_send_to_kindle_with_converter(formats)) - return bookformats + book_formats.extend(check_send_to_kindle_with_converter(formats)) + return book_formats else: log.error(u'Cannot find book entry %d', entry.id) return None @@ -194,12 +190,12 @@ def check_send_to_kindle(entry): # list with supported formats def check_read_formats(entry): extensions_reader = {'TXT', 'PDF', 'EPUB', 'CBZ', 'CBT', 'CBR', 'DJVU'} - bookformats = list() + book_formats = list() if len(entry.data): for ele in iter(entry.data): if ele.format.upper() in extensions_reader: - bookformats.append(ele.format.lower()) - return bookformats + book_formats.append(ele.format.lower()) + return book_formats # Files are processed in the following order/priority: @@ -229,23 +225,11 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): return _(u"The requested file could not be read. Maybe wrong permissions?") -def shorten_component(s, by_what): - l = len(s) - if l < by_what: - return s - l = (l - by_what)//2 - if l <= 0: - return s - return s[:l] + s[-l:] - - def get_valid_filename(value, replace_whitespace=True, chars=128): """ Returns the given string converted to a string that can be used for a clean filename. Limits num characters to 128 max. """ - - if value[-1:] == u'.': value = value[:-1]+u'_' value = value.replace("/", "_").replace(":", "_").strip('\0') @@ -814,7 +798,7 @@ def get_series_thumbnail(series_id, resolution): # saves book cover from url def save_cover_from_url(url, book_path): try: - if cli.allow_localhost: + if cli_param.allow_localhost: img = requests.get(url, timeout=(10, 200), allow_redirects=False) # ToDo: Error Handling elif use_advocate: img = advocate.get(url, timeout=(10, 200), allow_redirects=False) # ToDo: Error Handling diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index 5668e6da..9865b993 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -71,47 +71,8 @@ from flask_babel import gettext as _ from . import logger, config, calibre_db, db, helper, ub, lm from .render_template import render_title_template - log = logger.create() - -def register_url_value_preprocessor(kobo): - @kobo.url_value_preprocessor - # pylint: disable=unused-variable - def pop_auth_token(__, values): - g.auth_token = values.pop("auth_token") - - -def disable_failed_auth_redirect_for_blueprint(bp): - lm.blueprint_login_views[bp.name] = None - - -def get_auth_token(): - if "auth_token" in g: - return g.get("auth_token") - else: - return None - - -def requires_kobo_auth(f): - @wraps(f) - def inner(*args, **kwargs): - auth_token = get_auth_token() - if auth_token is not None: - user = ( - ub.session.query(ub.User) - .join(ub.RemoteAuthToken) - .filter(ub.RemoteAuthToken.auth_token == auth_token).filter(ub.RemoteAuthToken.token_type==1) - .first() - ) - if user is not None: - login_user(user) - return f(*args, **kwargs) - log.debug("Received Kobo request without a recognizable auth token.") - return abort(401) - return inner - - kobo_auth = Blueprint("kobo_auth", __name__, url_prefix="/kobo_auth") @@ -165,3 +126,40 @@ def delete_auth_token(user_id): .filter(ub.RemoteAuthToken.token_type==1).delete() return ub.session_commit() + + +def disable_failed_auth_redirect_for_blueprint(bp): + lm.blueprint_login_views[bp.name] = None + + +def get_auth_token(): + if "auth_token" in g: + return g.get("auth_token") + else: + return None + + +def register_url_value_preprocessor(kobo): + @kobo.url_value_preprocessor + # pylint: disable=unused-variable + def pop_auth_token(__, values): + g.auth_token = values.pop("auth_token") + + +def requires_kobo_auth(f): + @wraps(f) + def inner(*args, **kwargs): + auth_token = get_auth_token() + if auth_token is not None: + user = ( + ub.session.query(ub.User) + .join(ub.RemoteAuthToken) + .filter(ub.RemoteAuthToken.auth_token == auth_token).filter(ub.RemoteAuthToken.token_type==1) + .first() + ) + if user is not None: + login_user(user) + return f(*args, **kwargs) + log.debug("Received Kobo request without a recognizable auth token.") + return abort(401) + return inner diff --git a/cps/main.py b/cps/main.py index 304a244a..ccf03c15 100644 --- a/cps/main.py +++ b/cps/main.py @@ -20,11 +20,7 @@ import sys from . import create_app from .jinjia import jinjia -from .shelf import shelf from .remotelogin import remotelogin -from .search_metadata import meta -from .error_handler import init_errorhandler -from .tasks_status import tasks try: from kobo import kobo, get_kobo_activated @@ -50,6 +46,10 @@ def main(): from .editbooks import editbook from .about import about from .search import search + from .search_metadata import meta + from .shelf import shelf + from .tasks_status import tasks + from .error_handler import init_errorhandler from . import web_server init_errorhandler() diff --git a/cps/opds.py b/cps/opds.py index 2b8ab6d6..60dbd551 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -56,20 +56,6 @@ def requires_basic_auth_if_no_ano(f): return decorated -class FeedObject: - def __init__(self, rating_id, rating_name): - self.rating_id = rating_id - self.rating_name = rating_name - - @property - def id(self): - return self.rating_id - - @property - def name(self): - return self.rating_name - - @opds.route("/opds/") @opds.route("/opds") @requires_basic_auth_if_no_ano @@ -468,6 +454,20 @@ def feed_unread_books(): return render_xml_template('feed.xml', entries=result, pagination=pagination) +class FeedObject: + def __init__(self, rating_id, rating_name): + self.rating_id = rating_id + self.rating_name = rating_name + + @property + def id(self): + return self.rating_id + + @property + def name(self): + return self.rating_name + + def feed_search(term): if term: entries, __, ___ = calibre_db.get_search_results(term, config=config) diff --git a/cps/search.py b/cps/search.py index 429aea17..cd172b6b 100644 --- a/cps/search.py +++ b/cps/search.py @@ -406,7 +406,6 @@ def render_search_results(term, offset=None, order=None, limit=None): offset, order, limit, - False, *join) return render_title_template('search.html', searchterm=term, diff --git a/cps/services/__init__.py b/cps/services/__init__.py index 32a9d485..f93eca34 100644 --- a/cps/services/__init__.py +++ b/cps/services/__init__.py @@ -18,11 +18,10 @@ from .. import logger - log = logger.create() - -try: from . import goodreads_support +try: + from . import goodreads_support except ImportError as err: log.debug("Cannot import goodreads, showing authors-metadata will not work: %s", err) goodreads_support = None diff --git a/cps/shelf.py b/cps/shelf.py index 35f2941d..49d9a633 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -33,27 +33,9 @@ from . import calibre_db, config, db, logger, ub from .render_template import render_title_template from .usermanagement import login_required_if_no_ano -shelf = Blueprint('shelf', __name__) log = logger.create() - -def check_shelf_edit_permissions(cur_shelf): - if not cur_shelf.is_public and not cur_shelf.user_id == int(current_user.id): - log.error("User {} not allowed to edit shelf: {}".format(current_user.id, cur_shelf.name)) - return False - if cur_shelf.is_public and not current_user.role_edit_shelfs(): - log.info("User {} not allowed to edit public shelves".format(current_user.id)) - return False - return True - - -def check_shelf_view_permissions(cur_shelf): - if cur_shelf.is_public: - return True - if current_user.is_anonymous or cur_shelf.user_id != current_user.id: - log.error("User is unauthorized to view non-public shelf: {}".format(cur_shelf.name)) - return False - return True +shelf = Blueprint('shelf', __name__) @shelf.route("/shelf/add//", methods=["POST"]) @@ -238,96 +220,6 @@ def edit_shelf(shelf_id): return create_edit_shelf(shelf, page_title=_(u"Edit a shelf"), page="shelfedit", shelf_id=shelf_id) -# if shelf ID is set, we are editing a shelf -def create_edit_shelf(shelf, page_title, page, shelf_id=False): - sync_only_selected_shelves = current_user.kobo_only_shelves_sync - # calibre_db.session.query(ub.Shelf).filter(ub.Shelf.user_id == current_user.id).filter(ub.Shelf.kobo_sync).count() - if request.method == "POST": - to_save = request.form.to_dict() - if not current_user.role_edit_shelfs() and to_save.get("is_public") == "on": - flash(_(u"Sorry you are not allowed to create a public shelf"), category="error") - return redirect(url_for('web.index')) - is_public = 1 if to_save.get("is_public") == "on" else 0 - if config.config_kobo_sync: - shelf.kobo_sync = True if to_save.get("kobo_sync") else False - if shelf.kobo_sync: - ub.session.query(ub.ShelfArchive).filter(ub.ShelfArchive.user_id == current_user.id).filter( - ub.ShelfArchive.uuid == shelf.uuid).delete() - ub.session_commit() - shelf_title = to_save.get("title", "") - if check_shelf_is_unique(shelf, shelf_title, is_public, shelf_id): - shelf.name = shelf_title - shelf.is_public = is_public - if not shelf_id: - shelf.user_id = int(current_user.id) - ub.session.add(shelf) - shelf_action = "created" - flash_text = _(u"Shelf %(title)s created", title=shelf_title) - else: - shelf_action = "changed" - flash_text = _(u"Shelf %(title)s changed", title=shelf_title) - try: - ub.session.commit() - log.info(u"Shelf {} {}".format(shelf_title, shelf_action)) - flash(flash_text, category="success") - return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id)) - except (OperationalError, InvalidRequestError) as ex: - ub.session.rollback() - log.error_or_exception(ex) - log.error_or_exception("Settings Database error: {}".format(ex)) - flash(_(u"Database error: %(error)s.", error=ex.orig), category="error") - except Exception as ex: - ub.session.rollback() - log.error_or_exception(ex) - flash(_(u"There was an error"), category="error") - return render_title_template('shelf_edit.html', - shelf=shelf, - title=page_title, - page=page, - kobo_sync_enabled=config.config_kobo_sync, - sync_only_selected_shelves=sync_only_selected_shelves) - - -def check_shelf_is_unique(shelf, title, is_public, shelf_id=False): - if shelf_id: - ident = ub.Shelf.id != shelf_id - else: - ident = true() - if is_public == 1: - is_shelf_name_unique = ub.session.query(ub.Shelf) \ - .filter((ub.Shelf.name == title) & (ub.Shelf.is_public == 1)) \ - .filter(ident) \ - .first() is None - - if not is_shelf_name_unique: - log.error("A public shelf with the name '{}' already exists.".format(title)) - flash(_(u"A public shelf with the name '%(title)s' already exists.", title=title), - category="error") - else: - is_shelf_name_unique = ub.session.query(ub.Shelf) \ - .filter((ub.Shelf.name == title) & (ub.Shelf.is_public == 0) & - (ub.Shelf.user_id == int(current_user.id))) \ - .filter(ident) \ - .first() is None - - if not is_shelf_name_unique: - log.error("A private shelf with the name '{}' already exists.".format(title)) - flash(_(u"A private shelf with the name '%(title)s' already exists.", title=title), - category="error") - return is_shelf_name_unique - - -def delete_shelf_helper(cur_shelf): - if not cur_shelf or not check_shelf_edit_permissions(cur_shelf): - return False - shelf_id = cur_shelf.id - ub.session.delete(cur_shelf) - ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete() - ub.session.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id)) - ub.session_commit("successfully deleted Shelf {}".format(cur_shelf.name)) - return True - - @shelf.route("/shelf/delete/", methods=["POST"]) @login_required def delete_shelf(shelf_id): @@ -392,6 +284,115 @@ def order_shelf(shelf_id): abort(404) +def check_shelf_edit_permissions(cur_shelf): + if not cur_shelf.is_public and not cur_shelf.user_id == int(current_user.id): + log.error("User {} not allowed to edit shelf: {}".format(current_user.id, cur_shelf.name)) + return False + if cur_shelf.is_public and not current_user.role_edit_shelfs(): + log.info("User {} not allowed to edit public shelves".format(current_user.id)) + return False + return True + + +def check_shelf_view_permissions(cur_shelf): + if cur_shelf.is_public: + return True + if current_user.is_anonymous or cur_shelf.user_id != current_user.id: + log.error("User is unauthorized to view non-public shelf: {}".format(cur_shelf.name)) + return False + return True + + +# if shelf ID is set, we are editing a shelf +def create_edit_shelf(shelf, page_title, page, shelf_id=False): + sync_only_selected_shelves = current_user.kobo_only_shelves_sync + # calibre_db.session.query(ub.Shelf).filter(ub.Shelf.user_id == current_user.id).filter(ub.Shelf.kobo_sync).count() + if request.method == "POST": + to_save = request.form.to_dict() + if not current_user.role_edit_shelfs() and to_save.get("is_public") == "on": + flash(_(u"Sorry you are not allowed to create a public shelf"), category="error") + return redirect(url_for('web.index')) + is_public = 1 if to_save.get("is_public") == "on" else 0 + if config.config_kobo_sync: + shelf.kobo_sync = True if to_save.get("kobo_sync") else False + if shelf.kobo_sync: + ub.session.query(ub.ShelfArchive).filter(ub.ShelfArchive.user_id == current_user.id).filter( + ub.ShelfArchive.uuid == shelf.uuid).delete() + ub.session_commit() + shelf_title = to_save.get("title", "") + if check_shelf_is_unique(shelf_title, is_public, shelf_id): + shelf.name = shelf_title + shelf.is_public = is_public + if not shelf_id: + shelf.user_id = int(current_user.id) + ub.session.add(shelf) + shelf_action = "created" + flash_text = _(u"Shelf %(title)s created", title=shelf_title) + else: + shelf_action = "changed" + flash_text = _(u"Shelf %(title)s changed", title=shelf_title) + try: + ub.session.commit() + log.info(u"Shelf {} {}".format(shelf_title, shelf_action)) + flash(flash_text, category="success") + return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id)) + except (OperationalError, InvalidRequestError) as ex: + ub.session.rollback() + log.error_or_exception(ex) + log.error_or_exception("Settings Database error: {}".format(ex)) + flash(_(u"Database error: %(error)s.", error=ex.orig), category="error") + except Exception as ex: + ub.session.rollback() + log.error_or_exception(ex) + flash(_(u"There was an error"), category="error") + return render_title_template('shelf_edit.html', + shelf=shelf, + title=page_title, + page=page, + kobo_sync_enabled=config.config_kobo_sync, + sync_only_selected_shelves=sync_only_selected_shelves) + + +def check_shelf_is_unique(title, is_public, shelf_id=False): + if shelf_id: + ident = ub.Shelf.id != shelf_id + else: + ident = true() + if is_public == 1: + is_shelf_name_unique = ub.session.query(ub.Shelf) \ + .filter((ub.Shelf.name == title) & (ub.Shelf.is_public == 1)) \ + .filter(ident) \ + .first() is None + + if not is_shelf_name_unique: + log.error("A public shelf with the name '{}' already exists.".format(title)) + flash(_(u"A public shelf with the name '%(title)s' already exists.", title=title), + category="error") + else: + is_shelf_name_unique = ub.session.query(ub.Shelf) \ + .filter((ub.Shelf.name == title) & (ub.Shelf.is_public == 0) & + (ub.Shelf.user_id == int(current_user.id))) \ + .filter(ident) \ + .first() is None + + if not is_shelf_name_unique: + log.error("A private shelf with the name '{}' already exists.".format(title)) + flash(_(u"A private shelf with the name '%(title)s' already exists.", title=title), + category="error") + return is_shelf_name_unique + + +def delete_shelf_helper(cur_shelf): + if not cur_shelf or not check_shelf_edit_permissions(cur_shelf): + return False + shelf_id = cur_shelf.id + ub.session.delete(cur_shelf) + ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).delete() + ub.session.add(ub.ShelfArchive(uuid=cur_shelf.uuid, user_id=cur_shelf.user_id)) + ub.session_commit("successfully deleted Shelf {}".format(cur_shelf.name)) + return True + + def change_shelf_order(shelf_id, order): result = calibre_db.session.query(db.Books).outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book)\ diff --git a/cps/tasks_status.py b/cps/tasks_status.py index ca9b5796..e355ed85 100644 --- a/cps/tasks_status.py +++ b/cps/tasks_status.py @@ -34,7 +34,7 @@ log = logger.create() @tasks.route("/ajax/emailstat") @login_required def get_email_status_json(): - tasks = WorkerThread.getInstance().tasks + tasks = WorkerThread.get_instance().tasks return jsonify(render_task_status(tasks)) @@ -42,7 +42,7 @@ def get_email_status_json(): @login_required def get_tasks_status(): # if current user admin, show all email, otherwise only own emails - tasks = WorkerThread.getInstance().tasks + tasks = WorkerThread.get_instance().tasks answer = render_task_status(tasks) return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") diff --git a/cps/templates/tasks.html b/cps/templates/tasks.html index b0b5b3bd..5cbc5f8b 100644 --- a/cps/templates/tasks.html +++ b/cps/templates/tasks.html @@ -5,7 +5,7 @@ {% block body %}

{{_('Tasks')}}

- +
{% if g.user.role_admin() %} diff --git a/cps/web.py b/cps/web.py index 178ed18c..83425191 100644 --- a/cps/web.py +++ b/cps/web.py @@ -72,10 +72,10 @@ except ImportError: from functools import wraps -#try: -# from natsort import natsorted as sort -#except ImportError: -# sort = sorted # Just use regular sort then, may cause issues with badly named pages in cbz/cbr files +try: + from natsort import natsorted as sort +except ImportError: + sort = sorted # Just use regular sort then, may cause issues with badly named pages in cbz/cbr files @app.after_request