calibre-web/cps/editbooks.py

# -*- coding: utf-8 -*-

#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
#    Copyright (C) 2018-2019 OzzieIsaacs, cervinko, jkrehm, bodybybuddha, ok11,
#                            andy29485, idalin, Kyosfonica, wuqi, Kennyl, lemmsh,
#                            falgh1, grunjol, csitko, ytils, xybydy, trasba, vrabe,
#                            ruben-herold, marblepebble, JackED42, SiphonSquirrel,
#                            apetresc, nanu-c, mutschler
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import division, print_function, unicode_literals
import os
from datetime import datetime
import json
from shutil import copyfile
from uuid import uuid4

from babel import Locale as LC
from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
from flask_babel import gettext as _
from flask_login import current_user, login_required
from sqlalchemy.exc import OperationalError

from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
from . import config, get_locale, ub, db
from . import calibre_db
from .services.worker import WorkerThread
from .tasks.upload import TaskUpload
from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required


editbook = Blueprint('editbook', __name__)
log = logger.create()


# Modifies different Database objects, first check if elements have to be added to database, than check
# if elements have to be deleted, because they are no longer used
def modify_database_object(input_elements, db_book_object, db_object, db_session, db_type):
    # passing input_elements not as a list may lead to undesired results
    if not isinstance(input_elements, list):
        raise TypeError(str(input_elements) + " should be passed as a list")
    changed = False
    input_elements = [x for x in input_elements if x != '']
    # we have all input element (authors, series, tags) names now
    # 1. search for elements to remove
    del_elements = []
    for c_elements in db_book_object:
        found = False
        if db_type == 'languages':
            type_elements = c_elements.lang_code
        elif db_type == 'custom':
            type_elements = c_elements.value
        else:
            type_elements = c_elements.name
        for inp_element in input_elements:
            if inp_element.lower() == type_elements.lower():
                # if inp_element == type_elements:
                found = True
                break
        # if the element was not found in the new list, add it to remove list
        if not found:
            del_elements.append(c_elements)
    # 2. search for elements that need to be added
    add_elements = []
    for inp_element in input_elements:
        found = False
        for c_elements in db_book_object:
            if db_type == 'languages':
                type_elements = c_elements.lang_code
            elif db_type == 'custom':
                type_elements = c_elements.value
            else:
                type_elements = c_elements.name
            if inp_element == type_elements:
                found = True
                break
        if not found:
            add_elements.append(inp_element)
    # if there are elements to remove, we remove them now
    if len(del_elements) > 0:
        for del_element in del_elements:
            db_book_object.remove(del_element)
            changed = True
            if len(del_element.books) == 0:
                db_session.delete(del_element)
    # if there are elements to add, we add them now!
    if len(add_elements) > 0:
        if db_type == 'languages':
            db_filter = db_object.lang_code
        elif db_type == 'custom':
            db_filter = db_object.value
        else:
            db_filter = db_object.name
        for add_element in add_elements:
            # check if a element with that name exists
            db_element = db_session.query(db_object).filter(db_filter == add_element).first()
            # if no element is found add it
            # if new_element is None:
            if db_type == 'author':
                new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')), "")
            elif db_type == 'series':
                new_element = db_object(add_element, add_element)
            elif db_type == 'custom':
                new_element = db_object(value=add_element)
            elif db_type == 'publisher':
                new_element = db_object(add_element, None)
            else:  # db_type should be tag or language
                new_element = db_object(add_element)
            if db_element is None:
                changed = True
                db_session.add(new_element)
                db_book_object.append(new_element)
            else:
                if db_type == 'custom':
                    if db_element.value != add_element:
                        new_element.value = add_element
                elif db_type == 'languages':
                    if db_element.lang_code != add_element:
                        db_element.lang_code = add_element
                elif db_type == 'series':
                    if db_element.name != add_element:
                        db_element.name = add_element
                        db_element.sort = add_element
                elif db_type == 'author':
                    if db_element.name != add_element:
                        db_element.name = add_element
                        db_element.sort = add_element.replace('|', ',')
                elif db_type == 'publisher':
                    if db_element.name != add_element:
                        db_element.name = add_element
                        db_element.sort = None
                elif db_element.name != add_element:
                    db_element.name = add_element
                # add element to book
                changed = True
                db_book_object.append(db_element)
    return changed


def modify_identifiers(input_identifiers, db_identifiers, db_session):
    """Modify Identifiers to match input information.
       input_identifiers is a list of read-to-persist Identifiers objects.
       db_identifiers is a list of already persisted list of Identifiers objects."""
    changed = False
    error = False
    input_dict = dict([(identifier.type.lower(), identifier) for identifier in input_identifiers])
    if len(input_identifiers) != len(input_dict):
        error = True
    db_dict = dict([(identifier.type.lower(), identifier) for identifier in db_identifiers ])
    # delete db identifiers not present in input or modify them with input val
    for identifier_type, identifier in db_dict.items():
        if identifier_type not in input_dict.keys():
            db_session.delete(identifier)
            changed = True
        else:
            input_identifier = input_dict[identifier_type]
            identifier.type = input_identifier.type
            identifier.val = input_identifier.val
    # add input identifiers not present in db
    for identifier_type, identifier in input_dict.items():
        if identifier_type not in db_dict.keys():
            db_session.add(identifier)
            changed = True
    return changed, error

@editbook.route("/ajax/delete/<int:book_id>")
@login_required
def delete_book_from_details(book_id):
    return Response(delete_book(book_id,"", True), mimetype='application/json')


@editbook.route("/delete/<int:book_id>", defaults={'book_format': ""})
@editbook.route("/delete/<int:book_id>/<string:book_format>")
@login_required
def delete_book_ajax(book_id, book_format):
    return delete_book(book_id,book_format, False)

def delete_book(book_id, book_format, jsonResponse):
    warning = {}
    if current_user.role_delete_books():
        book = calibre_db.get_book(book_id)
        if book:
            try:
                result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
                if not result:
                    if jsonResponse:
                        return json.dumps({"location": url_for("editbook.edit_book"),
                                           "type": "alert",
                                           "format": "",
                                           "error": error}),
                    else:
                        flash(error, category="error")
                        return redirect(url_for('editbook.edit_book', book_id=book_id))
                if error:
                    if jsonResponse:
                        warning = {"location": url_for("editbook.edit_book"),
                                                "type": "warning",
                                                "format": "",
                                                "error": error}
                    else:
                        flash(error, category="warning")
                if not book_format:
                    # delete book from Shelfs, Downloads, Read list
                    ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
                    ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete()
                    ub.delete_download(book_id)
                    ub.session.commit()

                    # check if only this book links to:
                    # author, language, series, tags, custom columns
                    modify_database_object([u''], book.authors, db.Authors, calibre_db.session, 'author')
                    modify_database_object([u''], book.tags, db.Tags, calibre_db.session, 'tags')
                    modify_database_object([u''], book.series, db.Series, calibre_db.session, 'series')
                    modify_database_object([u''], book.languages, db.Languages, calibre_db.session, 'languages')
                    modify_database_object([u''], book.publishers, db.Publishers, calibre_db.session, 'publishers')

                    cc = calibre_db.session.query(db.Custom_Columns).\
                        filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
                    for c in cc:
                        cc_string = "custom_column_" + str(c.id)
                        if not c.is_multiple:
                            if len(getattr(book, cc_string)) > 0:
                                if c.datatype == 'bool' or c.datatype == 'integer' or c.datatype == 'float':
                                    del_cc = getattr(book, cc_string)[0]
                                    getattr(book, cc_string).remove(del_cc)
                                    log.debug('remove ' + str(c.id))
                                    calibre_db.session.delete(del_cc)
                                    calibre_db.session.commit()
                                elif c.datatype == 'rating':
                                    del_cc = getattr(book, cc_string)[0]
                                    getattr(book, cc_string).remove(del_cc)
                                    if len(del_cc.books) == 0:
                                        log.debug('remove ' + str(c.id))
                                        calibre_db.session.delete(del_cc)
                                        calibre_db.session.commit()
                                else:
                                    del_cc = getattr(book, cc_string)[0]
                                    getattr(book, cc_string).remove(del_cc)
                                    log.debug('remove ' + str(c.id))
                                    calibre_db.session.delete(del_cc)
                                    calibre_db.session.commit()
                        else:
                            modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id],
                                                   calibre_db.session, 'custom')
                    calibre_db.session.query(db.Books).filter(db.Books.id == book_id).delete()
                else:
                    calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\
                        filter(db.Data.format == book_format).delete()
                calibre_db.session.commit()
            except Exception as e:
                log.debug(e)
                calibre_db.session.rollback()
        else:
            # book not found
            log.error('Book with id "%s" could not be deleted: not found', book_id)
    if book_format:
        if jsonResponse:
            return json.dumps([warning, {"location": url_for("editbook.edit_book", book_id=book_id),
                                         "type": "success",
                                         "format": book_format,
                                         "message": _('Book Format Successfully Deleted')}])
        else:
            flash(_('Book Format Successfully Deleted'), category="success")
            return redirect(url_for('editbook.edit_book', book_id=book_id))
    else:
        if jsonResponse:
            return json.dumps([warning, {"location": url_for('web.index'),
                                         "type": "success",
                                         "format": book_format,
                                         "message": _('Book Successfully Deleted')}])
        else:
            flash(_('Book Successfully Deleted'), category="success")
            return redirect(url_for('web.index'))


def render_edit_book(book_id):
    calibre_db.update_title_sort(config)
    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
    book = calibre_db.get_filtered_book(book_id)
    if not book:
        flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
        return redirect(url_for("web.index"))

    for lang in book.languages:
        lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)

    book = calibre_db.order_authors(book)

    author_names = []
    for authr in book.authors:
        author_names.append(authr.name.replace('|', ','))

    # Option for showing convertbook button
    valid_source_formats=list()
    allowed_conversion_formats = list()
    kepub_possible=None
    if config.config_converterpath:
        for file in book.data:
            if file.format.lower() in constants.EXTENSIONS_CONVERT:
                valid_source_formats.append(file.format.lower())
    if config.config_kepubifypath and 'epub' in [file.format.lower() for file in book.data]:
        kepub_possible = True
        if not config.config_converterpath:
            valid_source_formats.append('epub')

    # Determine what formats don't already exist
    if config.config_converterpath:
        allowed_conversion_formats = constants.EXTENSIONS_CONVERT[:]
        for file in book.data:
            if file.format.lower() in allowed_conversion_formats:
                allowed_conversion_formats.remove(file.format.lower())
    if kepub_possible:
        allowed_conversion_formats.append('kepub')
    return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
                                 title=_(u"edit metadata"), page="editbook",
                                 conversion_formats=allowed_conversion_formats,
                                 config=config,
                                 source_formats=valid_source_formats)


def edit_book_ratings(to_save, book):
    changed = False
    if to_save["rating"].strip():
        old_rating = False
        if len(book.ratings) > 0:
            old_rating = book.ratings[0].rating
        ratingx2 = int(float(to_save["rating"]) * 2)
        if ratingx2 != old_rating:
            changed = True
            is_rating = calibre_db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first()
            if is_rating:
                book.ratings.append(is_rating)
            else:
                new_rating = db.Ratings(rating=ratingx2)
                book.ratings.append(new_rating)
            if old_rating:
                book.ratings.remove(book.ratings[0])
    else:
        if len(book.ratings) > 0:
            book.ratings.remove(book.ratings[0])
            changed = True
    return changed

def edit_book_tags(tags, book):
    input_tags = tags.split(',')
    input_tags = list(map(lambda it: it.strip(), input_tags))
    # Remove duplicates
    input_tags = helper.uniq(input_tags)
    return modify_database_object(input_tags, book.tags, db.Tags, calibre_db.session, 'tags')


def edit_book_series(series, book):
    input_series = [series.strip()]
    input_series = [x for x in input_series if x != '']
    return modify_database_object(input_series, book.series, db.Series, calibre_db.session, 'series')


def edit_book_series_index(series_index, book):
    # Add default series_index to book
    modif_date = False
    series_index = series_index or '1'
    if book.series_index != series_index:
        book.series_index = series_index
        modif_date = True
    return modif_date

# Handle book comments/description
def edit_book_comments(comments, book):
    modif_date = False
    if len(book.comments):
        if book.comments[0].text != comments:
            book.comments[0].text = comments
            modif_date = True
    else:
        if comments:
            book.comments.append(db.Comments(text=comments, book=book.id))
            modif_date = True
    return modif_date


def edit_book_languages(languages, book, upload=False):
    input_languages = languages.split(',')
    unknown_languages = []
    input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages)
    for l in unknown_languages:
        log.error('%s is not a valid language', l)
        flash(_(u"%(langname)s is not a valid language", langname=l), category="warning")
    # ToDo: Not working correct
    if upload and len(input_l) == 1:
        # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
        # the book it's language is set to the filter language
        if input_l[0] != current_user.filter_language() and current_user.filter_language() != "all":
            input_l[0] = calibre_db.session.query(db.Languages). \
                filter(db.Languages.lang_code == current_user.filter_language()).first()
    # Remove duplicates
    input_l = helper.uniq(input_l)
    return modify_database_object(input_l, book.languages, db.Languages, calibre_db.session, 'languages')


def edit_book_publisher(to_save, book):
    changed = False
    if to_save["publisher"]:
        publisher = to_save["publisher"].rstrip().strip()
        if len(book.publishers) == 0 or (len(book.publishers) > 0 and publisher != book.publishers[0].name):
            changed |= modify_database_object([publisher], book.publishers, db.Publishers, calibre_db.session,
                                              'publisher')
    elif len(book.publishers):
        changed |= modify_database_object([], book.publishers, db.Publishers, calibre_db.session, 'publisher')
    return changed


def edit_cc_data(book_id, book, to_save):
    changed = False
    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
    for c in cc:
        cc_string = "custom_column_" + str(c.id)
        if not c.is_multiple:
            if len(getattr(book, cc_string)) > 0:
                cc_db_value = getattr(book, cc_string)[0].value
            else:
                cc_db_value = None
            if to_save[cc_string].strip():
                if c.datatype == 'int' or c.datatype == 'bool' or c.datatype == 'float':
                    if to_save[cc_string] == 'None':
                        to_save[cc_string] = None
                    elif c.datatype == 'bool':
                        to_save[cc_string] = 1 if to_save[cc_string] == 'True' else 0

                    if to_save[cc_string] != cc_db_value:
                        if cc_db_value is not None:
                            if to_save[cc_string] is not None:
                                setattr(getattr(book, cc_string)[0], 'value', to_save[cc_string])
                                changed = True
                            else:
                                del_cc = getattr(book, cc_string)[0]
                                getattr(book, cc_string).remove(del_cc)
                                calibre_db.session.delete(del_cc)
                                changed = True
                        else:
                            cc_class = db.cc_classes[c.id]
                            new_cc = cc_class(value=to_save[cc_string], book=book_id)
                            calibre_db.session.add(new_cc)
                            changed = True

                else:
                    if c.datatype == 'rating':
                        to_save[cc_string] = str(int(float(to_save[cc_string]) * 2))
                    if to_save[cc_string].strip() != cc_db_value:
                        if cc_db_value is not None:
                            # remove old cc_val
                            del_cc = getattr(book, cc_string)[0]
                            getattr(book, cc_string).remove(del_cc)
                            if len(del_cc.books) == 0:
                                calibre_db.session.delete(del_cc)
                                changed = True
                        cc_class = db.cc_classes[c.id]
                        new_cc = calibre_db.session.query(cc_class).filter(
                            cc_class.value == to_save[cc_string].strip()).first()
                        # if no cc val is found add it
                        if new_cc is None:
                            new_cc = cc_class(value=to_save[cc_string].strip())
                            calibre_db.session.add(new_cc)
                            changed = True
                            calibre_db.session.flush()
                            new_cc = calibre_db.session.query(cc_class).filter(
                                cc_class.value == to_save[cc_string].strip()).first()
                        # add cc value to book
                        getattr(book, cc_string).append(new_cc)
            else:
                if cc_db_value is not None:
                    # remove old cc_val
                    del_cc = getattr(book, cc_string)[0]
                    getattr(book, cc_string).remove(del_cc)
                    if not del_cc.books or len(del_cc.books) == 0:
                        calibre_db.session.delete(del_cc)
                        changed = True
        else:
            input_tags = to_save[cc_string].split(',')
            input_tags = list(map(lambda it: it.strip(), input_tags))
            changed |= modify_database_object(input_tags,
                                              getattr(book, cc_string),
                                              db.cc_classes[c.id],
                                              calibre_db.session,
                                              'custom')
    return changed

def upload_single_file(request, book, book_id):
    # Check and handle Uploaded file
    if 'btn-upload-format' in request.files:
        requested_file = request.files['btn-upload-format']
        # check for empty request
        if requested_file.filename != '':
            if '.' in requested_file.filename:
                file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
                if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
                    flash(_("File extension '%(ext)s' is not allowed to be uploaded to this server", ext=file_ext),
                          category="error")
                    return redirect(url_for('web.show_book', book_id=book.id))
            else:
                flash(_('File to be uploaded must have an extension'), category="error")
                return redirect(url_for('web.show_book', book_id=book.id))

            file_name = book.path.rsplit('/', 1)[-1]
            filepath = os.path.normpath(os.path.join(config.config_calibre_dir, book.path))
            saved_filename = os.path.join(filepath, file_name + '.' + file_ext)

            # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
            if not os.path.exists(filepath):
                try:
                    os.makedirs(filepath)
                except OSError:
                    flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
                    return redirect(url_for('web.show_book', book_id=book.id))
            try:
                requested_file.save(saved_filename)
            except OSError:
                flash(_(u"Failed to store file %(file)s.", file=saved_filename), category="error")
                return redirect(url_for('web.show_book', book_id=book.id))

            file_size = os.path.getsize(saved_filename)
            is_format = calibre_db.get_book_format(book_id, file_ext.upper())

            # Format entry already exists, no need to update the database
            if is_format:
                log.warning('Book format %s already existing', file_ext.upper())
            else:
                try:
                    db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
                    calibre_db.session.add(db_format)
                    calibre_db.session.commit()
                    calibre_db.update_title_sort(config)
                except OperationalError as e:
                    calibre_db.session.rollback()
                    log.error('Database error: %s', e)
                    flash(_(u"Database error: %(error)s.", error=e), category="error")
                    return redirect(url_for('web.show_book', book_id=book.id))

            # Queue uploader info
            uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
            WorkerThread.add(current_user.nickname, TaskUpload(
                "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>"))

            return uploader.process(
                saved_filename, *os.path.splitext(requested_file.filename),
                rarExecutable=config.config_rarfile_location)


def upload_cover(request, book):
    if 'btn-upload-cover' in request.files:
        requested_file = request.files['btn-upload-cover']
        # check for empty request
        if requested_file.filename != '':
            ret, message = helper.save_cover(requested_file, book.path)
            if ret is True:
                return True
            else:
                flash(message, category="error")
                return False
    return None


@editbook.route("/admin/book/<int:book_id>", methods=['GET', 'POST'])
@login_required_if_no_ano
@edit_required
def edit_book(book_id):
    modif_date = False
    # Show form
    if request.method != 'POST':
        return render_edit_book(book_id)

    # create the function for sorting...
    calibre_db.update_title_sort(config)
    book = calibre_db.get_filtered_book(book_id)

    # Book not found
    if not book:
        flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
        return redirect(url_for("web.index"))

    meta = upload_single_file(request, book, book_id)
    if upload_cover(request, book) is True:
        book.has_cover = 1
        modif_date = True
    try:
        to_save = request.form.to_dict()
        merge_metadata(to_save, meta)
        # Update book
        edited_books_id = None

        #handle book title
        if book.title != to_save["book_title"].rstrip().strip():
            if to_save["book_title"] == '':
                to_save["book_title"] = _(u'Unknown')
            book.title = to_save["book_title"].rstrip().strip()
            edited_books_id = book.id
            modif_date = True

        # handle author(s)
        input_authors = to_save["author_name"].split('&')
        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
        # Remove duplicates in authors list
        input_authors = helper.uniq(input_authors)
        # we have all author names now
        if input_authors == ['']:
            input_authors = [_(u'Unknown')]  # prevent empty Author

        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')

        # Search for each author if author is in database, if not, authorname and sorted authorname is generated new
        # everything then is assembled for sorted author field in database
        sort_authors_list = list()
        for inp in input_authors:
            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
            if not stored_author:
                stored_author = helper.get_sorted_author(inp)
            else:
                stored_author = stored_author.sort
            sort_authors_list.append(helper.get_sorted_author(stored_author))
        sort_authors = ' & '.join(sort_authors_list)
        if book.author_sort != sort_authors:
            edited_books_id = book.id
            book.author_sort = sort_authors
            modif_date = True

        if config.config_use_google_drive:
            gdriveutils.updateGdriveCalibreFromLocal()

        error = False
        if edited_books_id:
            error = helper.update_dir_stucture(edited_books_id, config.config_calibre_dir, input_authors[0])

        if not error:
            if to_save["cover_url"]:
                result, error = helper.save_cover_from_url(to_save["cover_url"], book.path)
                if result is True:
                    book.has_cover = 1
                    modif_date = True
                else:
                    flash(error, category="error")

            # Add default series_index to book
            modif_date |= edit_book_series_index(to_save["series_index"], book)

            # Handle book comments/description
            modif_date |= edit_book_comments(to_save["description"], book)

            # Handle identifiers
            input_identifiers = identifier_list(to_save, book)
            modification, warning = modify_identifiers(input_identifiers, book.identifiers, calibre_db.session)
            if warning:
                flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning")
            modif_date |= modification
            # Handle book tags
            modif_date |= edit_book_tags(to_save['tags'], book)

            # Handle book series
            modif_date |= edit_book_series(to_save["series"], book)

            if to_save["pubdate"]:
                try:
                    book.pubdate = datetime.strptime(to_save["pubdate"], "%Y-%m-%d")
                except ValueError:
                    book.pubdate = db.Books.DEFAULT_PUBDATE
            else:
                book.pubdate = db.Books.DEFAULT_PUBDATE

            # handle book publisher
            modif_date |= edit_book_publisher(to_save, book)

            # handle book languages
            modif_date |= edit_book_languages(to_save['languages'], book)

            # handle book ratings
            modif_date |= edit_book_ratings(to_save, book)

            # handle cc data
            modif_date |= edit_cc_data(book_id, book, to_save)

            if modif_date:
                book.last_modified = datetime.utcnow()
            calibre_db.session.merge(book)
            calibre_db.session.commit()
            if config.config_use_google_drive:
                gdriveutils.updateGdriveCalibreFromLocal()
            if "detail_view" in to_save:
                return redirect(url_for('web.show_book', book_id=book.id))
            else:
                flash(_("Metadata successfully updated"), category="success")
                return render_edit_book(book_id)
        else:
            calibre_db.session.rollback()
            flash(error, category="error")
            return render_edit_book(book_id)
    except Exception as e:
        log.exception(e)
        calibre_db.session.rollback()
        flash(_("Error editing book, please check logfile for details"), category="error")
        return redirect(url_for('web.show_book', book_id=book.id))


def merge_metadata(to_save, meta):
    if to_save['author_name'] == _(u'Unknown'):
        to_save['author_name'] = ''
    if to_save['book_title'] == _(u'Unknown'):
        to_save['book_title'] = ''
    for s_field, m_field in [
            ('tags', 'tags'), ('author_name', 'author'), ('series', 'series'),
            ('series_index', 'series_id'), ('languages', 'languages'),
            ('book_title', 'title')]:
        to_save[s_field] = to_save[s_field] or getattr(meta, m_field, '')
    to_save["description"] = to_save["description"] or Markup(
        getattr(meta, 'description', '')).unescape()

def identifier_list(to_save, book):
    """Generate a list of Identifiers from form information"""
    id_type_prefix = 'identifier-type-'
    id_val_prefix = 'identifier-val-'
    result = []
    for type_key, type_value in to_save.items():
        if not type_key.startswith(id_type_prefix):
            continue
        val_key = id_val_prefix + type_key[len(id_type_prefix):]
        if val_key not in to_save.keys():
            continue
        result.append(db.Identifiers(to_save[val_key], type_value, book.id))
    return result

@editbook.route("/upload", methods=["GET", "POST"])
@login_required_if_no_ano
@upload_required
def upload():
    if not config.config_uploading:
        abort(404)
    if request.method == 'POST' and 'btn-upload' in request.files:
        for requested_file in request.files.getlist("btn-upload"):
            try:
                modif_date = False
                # create the function for sorting...
                calibre_db.update_title_sort(config)
                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))

                # check if file extension is correct
                if '.' in requested_file.filename:
                    file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
                    if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
                        flash(
                            _("File extension '%(ext)s' is not allowed to be uploaded to this server",
                              ext=file_ext), category="error")
                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
                else:
                    flash(_('File to be uploaded must have an extension'), category="error")
                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')

                # extract metadata from file
                try:
                    meta = uploader.upload(requested_file, config.config_rarfile_location)
                except (IOError, OSError):
                    log.error("File %s could not saved to temp dir", requested_file.filename)
                    flash(_(u"File %(filename)s could not saved to temp dir",
                            filename= requested_file.filename), category="error")
                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
                title = meta.title
                authr = meta.author

                if title != _(u'Unknown') and authr != _(u'Unknown'):
                    entry = calibre_db.check_exists_book(authr, title)
                    if entry:
                        log.info("Uploaded book probably exists in library")
                        flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
                            + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")

                # handle authors
                input_authors = authr.split('&')
                # handle_authors(input_authors)
                input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
                # Remove duplicates in authors list
                input_authors = helper.uniq(input_authors)

                # we have all author names now
                if input_authors == ['']:
                    input_authors = [_(u'Unknown')]  # prevent empty Author

                sort_authors_list=list()
                db_author = None
                for inp in input_authors:
                    stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
                    if not stored_author:
                        if not db_author:
                            db_author = db.Authors(inp, helper.get_sorted_author(inp), "")
                            calibre_db.session.add(db_author)
                            calibre_db.session.commit()
                        sort_author = helper.get_sorted_author(inp)
                    else:
                        if not db_author:
                            db_author = stored_author
                        sort_author = stored_author.sort
                    sort_authors_list.append(sort_author)
                sort_authors = ' & '.join(sort_authors_list)

                title_dir = helper.get_valid_filename(title)
                author_dir = helper.get_valid_filename(db_author.name)

                # combine path and normalize path from windows systems
                path = os.path.join(author_dir, title_dir).replace('\\', '/')
                # Calibre adds books with utc as timezone
                db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
                                   '1', datetime.utcnow(), path, meta.cover, db_author, [], "")

                modif_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
                                                     'author')

                # Add series_index to book
                modif_date |= edit_book_series_index(meta.series_id, db_book)

                # add languages
                modif_date |= edit_book_languages(meta.languages, db_book, upload=True)

                # handle tags
                modif_date |= edit_book_tags(meta.tags, db_book)

                # handle series
                modif_date |= edit_book_series(meta.series, db_book)

                # Add file to book
                file_size = os.path.getsize(meta.file_path)
                db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir)
                db_book.data.append(db_data)
                calibre_db.session.add(db_book)

                # flush content, get db_book.id available
                calibre_db.session.flush()

                # Comments needs book id therfore only possible after flush
                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)

                book_id = db_book.id
                title = db_book.title

                error = helper.update_dir_stucture(book_id,
                                                   config.config_calibre_dir,
                                                   input_authors[0],
                                                   meta.file_path,
                                                   title_dir + meta.extension)

                # move cover to final directory, including book id
                if meta.cover:
                    coverfile = meta.cover
                else:
                    coverfile = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
                new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
                try:
                    copyfile(coverfile, new_coverpath)
                    if meta.cover:
                        os.unlink(meta.cover)
                except OSError as e:
                    log.error("Failed to move cover file %s: %s", new_coverpath, e)
                    flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
                            error=e),
                          category="error")

                # save data to database, reread data
                calibre_db.session.commit()
                #calibre_db.setup_db(config, ub.app_DB_path)
                # Reread book. It's important not to filter the result, as it could have language which hide it from
                # current users view (tags are not stored/extracted from metadata and could also be limited)
                #book = calibre_db.get_book(book_id)
                if config.config_use_google_drive:
                    gdriveutils.updateGdriveCalibreFromLocal()
                if error:
                    flash(error, category="error")
                uploadText=_(u"File %(file)s uploaded", file=title)
                WorkerThread.add(current_user.nickname, TaskUpload(
                    "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>"))

                if len(request.files.getlist("btn-upload")) < 2:
                    if current_user.role_edit() or current_user.role_admin():
                        resp = {"location": url_for('editbook.edit_book', book_id=book_id)}
                        return Response(json.dumps(resp), mimetype='application/json')
                    else:
                        resp = {"location": url_for('web.show_book', book_id=book_id)}
                        return Response(json.dumps(resp), mimetype='application/json')
            except OperationalError as e:
                calibre_db.session.rollback()
                log.error("Database error: %s", e)
                flash(_(u"Database error: %(error)s.", error=e), category="error")
        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')

@editbook.route("/admin/book/convert/<int:book_id>", methods=['POST'])
@login_required_if_no_ano
@edit_required
def convert_bookformat(book_id):
    # check to see if we have form fields to work with -  if not send user back
    book_format_from = request.form.get('book_format_from', None)
    book_format_to = request.form.get('book_format_to', None)

    if (book_format_from is None) or (book_format_to is None):
        flash(_(u"Source or destination format for conversion missing"), category="error")
        return redirect(url_for('editbook.edit_book', book_id=book_id))

    log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to)
    rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(),
                                     book_format_to.upper(), current_user.nickname)

    if rtn is None:
        flash(_(u"Book successfully queued for converting to %(book_format)s",
                    book_format=book_format_to),
                    category="success")
    else:
        flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
    return redirect(url_for('editbook.edit_book', book_id=book_id))

@editbook.route("/ajax/editbooks/<param>", methods=['POST'])
@login_required_if_no_ano
def edit_list_book(param):
    vals = request.form.to_dict()
    # calibre_db.update_title_sort(config)
    #calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
    book = calibre_db.get_book(vals['pk'])
    if param =='series_index':
        edit_book_series_index(vals['value'], book)
    elif param =='tags':
        edit_book_tags(vals['value'], book)
    elif param =='series':
        edit_book_series(vals['value'], book)
    elif param =='publishers':
        vals['publisher'] = vals['value']
        edit_book_publisher(vals, book)
    elif param =='languages':
        edit_book_languages(vals['value'], book)
    elif param =='author_sort':
        book.author_sort = vals['value']
    elif param =='title':
        book.title = vals['value']
        helper.update_dir_stucture(book.id, config.config_calibre_dir)
    elif param =='sort':
        book.sort = vals['value']
    # ToDo: edit books
    elif param =='authors':
        input_authors = vals['value'].split('&')
        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
        modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
        sort_authors_list = list()
        for inp in input_authors:
            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
            if not stored_author:
                stored_author = helper.get_sorted_author(inp)
            else:
                stored_author = stored_author.sort
            sort_authors_list.append(helper.get_sorted_author(stored_author))
        sort_authors = ' & '.join(sort_authors_list)
        if book.author_sort != sort_authors:
            book.author_sort = sort_authors
        helper.update_dir_stucture(book.id, config.config_calibre_dir, input_authors[0])
    book.last_modified = datetime.utcnow()
    calibre_db.session.commit()
    return ""

@editbook.route("/ajax/sort_value/<field>/<int:bookid>")
@login_required
def get_sorted_entry(field, bookid):
    if field == 'title' or field == 'authors':
        book = calibre_db.get_filtered_book(bookid)
        if book:
            if field == 'title':
                return json.dumps({'sort': book.sort})
            elif field == 'authors':
                return json.dumps({'author_sort': book.author_sort})
    return ""


@editbook.route("/ajax/simulatemerge", methods=['POST'])
@login_required
def simulate_merge_list_book():
    vals = request.get_json().get('Merge_books')
    if vals:
        to_book = calibre_db.get_book(vals[0]).title
        vals.pop(0)
        if to_book:
            for book_id in vals:
                from_book = []
                from_book.append(calibre_db.get_book(book_id).title)
            return json.dumps({'to': to_book, 'from': from_book})
    return ""


@editbook.route("/ajax/mergebooks", methods=['POST'])
@login_required
def merge_list_book():
    vals = request.get_json().get('Merge_books')
    to_file = list()
    if vals:
        # load all formats from target book
        to_book = calibre_db.get_book(vals[0])
        vals.pop(0)
        if to_book:
            for file in to_book.data:
                to_file.append(file.format)
            to_name = helper.get_valid_filename(to_book.title) + ' - ' + \
                      helper.get_valid_filename(to_book.authors[0].name)
            for book_id in vals:
                from_book = calibre_db.get_book(book_id)
                if from_book:
                    for element in from_book.data:
                        if element.format not in to_file:
                            # create new data entry with: book_id, book_format, uncompressed_size, name
                            filepath_new = os.path.normpath(os.path.join(config.config_calibre_dir,
                                                                         to_book.path,
                                                                         to_name + "." + element.format.lower()))
                            filepath_old = os.path.normpath(os.path.join(config.config_calibre_dir,
                                                                         from_book.path,
                                                                         element.name + "." + element.format.lower()))
                            copyfile(filepath_old, filepath_new)
                            to_book.data.append(db.Data(to_book.id,
                                                        element.format,
                                                        element.uncompressed_size,
                                                        to_name))
                    delete_book(from_book.id,"", True) # json_resp =
                    return json.dumps({'success': True})
    return ""