Refactored rename_authors

Bugfixes from tests

cps/__init__.py

@@ -57,9 +57,9 @@ mimetypes.init()
 mimetypes.add_type('application/xhtml+xml', '.xhtml')
 mimetypes.add_type('application/epub+zip', '.epub')
 mimetypes.add_type('application/epub+zip', '.kepub')
-mimetypes.add_type('application/fb2+zip', '.fb2')
+mimetypes.add_type('text/xml', '.fb2')
-mimetypes.add_type('application/x-mobipocket-ebook', '.mobi')
+mimetypes.add_type('application/octet-stream', '.mobi')
-mimetypes.add_type('application/x-mobipocket-ebook', '.prc')
+mimetypes.add_type('application/octet-stream', '.prc')
 mimetypes.add_type('application/vnd.amazon.ebook', '.azw')
 mimetypes.add_type('application/x-mobi8-ebook', '.azw3')
 mimetypes.add_type('application/x-rar', '.cbr')
@@ -69,10 +69,10 @@ mimetypes.add_type('application/x-7z-compressed', '.cb7')
 mimetypes.add_type('image/vnd.djv', '.djv')
 mimetypes.add_type('image/vnd.djv', '.djvu')
 mimetypes.add_type('application/mpeg', '.mpeg')
-mimetypes.add_type('application/mpeg', '.mp3')
+mimetypes.add_type('audio/mpeg', '.mp3')
 mimetypes.add_type('application/mp4', '.m4a')
 mimetypes.add_type('application/mp4', '.m4b')
-mimetypes.add_type('application/ogg', '.ogg')
+mimetypes.add_type('audio/ogg', '.ogg')
 mimetypes.add_type('application/ogg', '.oga')
 mimetypes.add_type('text/css', '.css')
 mimetypes.add_type('application/x-ms-reader', '.lit')

cps/editbooks.py

@@ -23,7 +23,7 @@
 import os
 from datetime import datetime
 import json
-from shutil import copyfile
+from shutil import copyfile, move
 from uuid import uuid4
 from markupsafe import escape, Markup  # dependency of flask
 from functools import wraps
@@ -118,14 +118,13 @@ def edit_book(book_id):
         # handle book title change
         title_change = handle_title_on_edit(book, to_save["book_title"])
         # handle book author change
-        input_authors, author_change, renamed = handle_author_on_edit(book, to_save["author_name"])
+        input_authors, author_change = handle_author_on_edit(book, to_save["author_name"])
         if author_change or title_change:
             edited_books_id = book.id
             modify_date = True
             title_author_error = helper.update_dir_structure(edited_books_id,
                                                              config.get_book_path(),
-                                                             input_authors[0],
+                                                             input_authors[0])
-                                                             renamed_author=renamed)
         if title_author_error:
             flash(title_author_error, category="error")
             calibre_db.session.rollback()
@@ -251,7 +250,7 @@ def upload():
                 if error:
                     return error
 
-                db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta)
+                db_book, input_authors, title_dir = create_book_on_upload(modify_date, meta)
 
                 # Comments need book id therefore only possible after flush
                 modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
@@ -261,7 +260,6 @@ def upload():
                 if config.config_use_google_drive:
                     helper.upload_new_file_gdrive(book_id,
                                                   input_authors[0],
-                                                  renamed_authors,
                                                   title,
                                                   title_dir,
                                                   meta.file_path,
@@ -271,8 +269,7 @@ def upload():
                                                         config.get_book_path(),
                                                         input_authors[0],
                                                         meta.file_path,
-                                                        title_dir + meta.extension.lower(),
+                                                        title_dir + meta.extension.lower())
-                                                        renamed_author=renamed_authors)
 
                 move_coverfile(meta, db_book)
 
@@ -405,9 +402,8 @@ def edit_list_book(param):
             ret = Response(json.dumps({'success': True, 'newValue':  book.comments[0].text}),
                            mimetype='application/json')
         elif param == 'authors':
-            input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
+            input_authors, __ = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
-            rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0],
+            rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0])
-                                                       renamed_author=renamed)
             if not rename_error:
                 ret = Response(json.dumps({
                     'success': True,
@@ -543,7 +539,7 @@ def table_xchange_author_title():
                 author_names.append(authr.name.replace('|', ','))
 
             title_change = handle_title_on_edit(book, " ".join(author_names))
-            input_authors, author_change, renamed = handle_author_on_edit(book, authors)
+            input_authors, author_change = handle_author_on_edit(book, authors)
             if author_change or title_change:
                 edited_books_id = book.id
                 modify_date = True
@@ -553,8 +549,7 @@ def table_xchange_author_title():
 
             if edited_books_id:
                 # toDo: Handle error
-                edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0],
+                edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0])
-                                                         renamed_author=renamed)
             if modify_date:
                 book.last_modified = datetime.utcnow()
                 calibre_db.set_metadata_dirty(book.id)
@@ -602,7 +597,9 @@ def identifier_list(to_save, book):
     return result
 
 
-def prepare_authors(authr):
+def prepare_authors(authr, calibre_path, gdrive=False):
+    if gdrive:
+        calibre_path = ""
     # handle authors
     input_authors = authr.split('&')
     # handle_authors(input_authors)
@@ -614,20 +611,44 @@ def prepare_authors(authr):
     if input_authors == ['']:
         input_authors = [_('Unknown')]  # prevent empty Author
 
-    renamed = list()
+    for in_aut in input_authors:
-    for index,in_aut in enumerate(input_authors):
-        # renamed_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == in_aut).first()
         renamed_author = calibre_db.session.query(db.Authors).filter(func.lower(db.Authors.name).ilike(in_aut)).first()
         if renamed_author and in_aut != renamed_author.name:
-            renamed.append(renamed_author.name)
+            old_author_name = renamed_author.name
+            # rename author in Database
+            create_objects_for_addition(renamed_author, in_aut,"author")
+            # rename all Books with this author as first author:
+            # rename all book author_sort strings with the new author name
             all_books = calibre_db.session.query(db.Books) \
                 .filter(db.Books.authors.any(db.Authors.name == renamed_author.name)).all()
-            sorted_renamed_author = helper.get_sorted_author(renamed_author.name)
-            sorted_old_author = helper.get_sorted_author(in_aut)
             for one_book in all_books:
-                one_book.author_sort = one_book.author_sort.replace(sorted_renamed_author, sorted_old_author)
+                # ToDo: check
-            input_authors[index] = renamed_author.name
+                sorted_old_author = helper.get_sorted_author(old_author_name)
-    return input_authors, renamed
+                sorted_renamed_author = helper.get_sorted_author(in_aut)
+                # change author sort path
+                try:
+                    author_index = one_book.author_sort.index(sorted_old_author)
+                    one_book.author_sort = one_book.author_sort.replace(sorted_old_author, sorted_renamed_author)
+                except ValueError:
+                    log.error("Sorted author {} not found in database".format(sorted_old_author))
+                    author_index = -1
+                # change book path if changed author is first author -> match on first position
+                if author_index == 0:
+                    one_titledir = one_book.path.split('/')[1]
+                    one_old_authordir = one_book.path.split('/')[0]
+                    # rename author path only once per renamed author -> search all books with author name in book.path
+                    # das muss einmal geschehen aber pro Buch geprüft werden ansonsten habe ich das Problem das vlt. 2 gleiche Ordner bis auf Groß/Kleinschreibung vorhanden sind im Umzug
+                    new_author_dir = helper.rename_author_path(in_aut, one_old_authordir, renamed_author.name, calibre_path, gdrive)
+                    one_book.path = os.path.join(new_author_dir, one_titledir).replace('\\', '/')
+                    # rename all books in book data with the new author name and move corresponding files to new locations
+                    # old_path = os.path.join(calibre_path, new_author_dir, one_titledir)
+                    new_path = os.path.join(calibre_path, new_author_dir, one_titledir)
+                    all_new_name = helper.get_valid_filename(one_book.title, chars=42) + ' - ' \
+                                   + helper.get_valid_filename(renamed_author.name, chars=42)
+                    # change location in database to new author/title path
+                    helper.rename_all_files_on_change(one_book, new_path, new_path, all_new_name, gdrive)
+
+    return input_authors
 
 
 def prepare_authors_on_upload(title, authr):
@@ -638,7 +659,7 @@ def prepare_authors_on_upload(title, authr):
             flash(_("Uploaded book probably exists in the library, consider to change before upload new: ")
                   + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")
 
-    input_authors, renamed = prepare_authors(authr)
+    input_authors = prepare_authors(authr, config.get_book_path(), config.config_use_google_drive)
 
     sort_authors_list = list()
     db_author = None
@@ -657,13 +678,13 @@ def prepare_authors_on_upload(title, authr):
             sort_author = stored_author.sort
         sort_authors_list.append(sort_author)
     sort_authors = ' & '.join(sort_authors_list)
-    return sort_authors, input_authors, db_author, renamed
+    return sort_authors, input_authors, db_author
 
 
 def create_book_on_upload(modify_date, meta):
     title = meta.title
     authr = meta.author
-    sort_authors, input_authors, db_author, renamed_authors = prepare_authors_on_upload(title, authr)
+    sort_authors, input_authors, db_author = prepare_authors_on_upload(title, authr)
 
     title_dir = helper.get_valid_filename(title, chars=96)
     author_dir = helper.get_valid_filename(db_author.name, chars=96)
@@ -720,14 +741,14 @@ def create_book_on_upload(modify_date, meta):
         flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning")
     modify_date |= modification
 
-    return db_book, input_authors, title_dir, renamed_authors
+    return db_book, input_authors, title_dir
 
 
 def file_handling_on_upload(requested_file):
     # check if file extension is correct
     allowed_extensions = config.config_upload_formats.split(',')
     if requested_file:
-        if config.config_check_extensions:
+        if config.config_check_extensions and allowed_extensions != ['']:
             if not validate_mime_type(requested_file, allowed_extensions):
                 flash(_("File type isn't allowed to be uploaded to this server"), category="error")
                 return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
@@ -1163,7 +1184,7 @@ def upload_single_file(file_request, book, book_id):
     requested_file = file_request.files.get('btn-upload-format', None)
     allowed_extensions = config.config_upload_formats.split(',')
     if requested_file:
-        if config.config_check_extensions:
+        if config.config_check_extensions and allowed_extensions != ['']:
             if not validate_mime_type(requested_file, allowed_extensions):
                 flash(_("File type isn't allowed to be uploaded to this server"), category="error")
                 return False
@@ -1262,7 +1283,7 @@ def handle_title_on_edit(book, book_title):
 def handle_author_on_edit(book, author_name, update_stored=True):
     change = False
     # handle author(s)
-    input_authors, renamed = prepare_authors(author_name)
+    input_authors = prepare_authors(author_name, config.get_book_path(), config.config_use_google_drive)
 
     # change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
     # Search for each author if author is in database, if not, author name and sorted author name is generated new
@@ -1282,7 +1303,7 @@ def handle_author_on_edit(book, author_name, update_stored=True):
 
     change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
 
-    return input_authors, change, renamed
+    return input_authors, change
 
 
 def search_objects_remove(db_book_object, db_type, input_elements):

cps/gdriveutils.py

@@ -581,7 +581,7 @@ def get_cover_via_gdrive(cover_path):
             session.add(permissionAdded)
             try:
                 session.commit()
-            except OperationalError as ex:
+            except (OperationalError, IntegrityError) as ex:
                 log.error_or_exception('Database error: {}'.format(ex))
                 session.rollback()
         return df.metadata.get('webContentLink')

cps/helper.py

@@ -388,41 +388,26 @@ def delete_book_file(book, calibrepath, book_format=None):
                    id=book.id,
                    path=book.path)
 
-
+def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdrive=False):
-def clean_author_database(renamed_author, calibre_path="", local_book=None, gdrive=None):
+    for file_format in one_book.data:
-    valid_filename_authors = [get_valid_filename(r, chars=96) for r in renamed_author]
-    for r in renamed_author:
-        if local_book:
-            all_books = [local_book]
-        else:
-            all_books = calibre_db.session.query(db.Books) \
-                .filter(db.Books.authors.any(db.Authors.name == r)).all()
-        for book in all_books:
-            book_author_path = book.path.split('/')[0]
-            if book_author_path in valid_filename_authors or local_book:
-                new_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == r).first()
-                all_new_authordir = get_valid_filename(new_author.name, chars=96)
-                all_titledir = book.path.split('/')[1]
-                all_new_path = os.path.join(calibre_path, all_new_authordir, all_titledir)
-                all_new_name = get_valid_filename(book.title, chars=42) + ' - ' \
-                    + get_valid_filename(new_author.name, chars=42)
-                # change location in database to new author/title path
-                book.path = os.path.join(all_new_authordir, all_titledir).replace('\\', '/')
-                for file_format in book.data:
         if not gdrive:
-                        shutil.move(os.path.normcase(os.path.join(all_new_path,
+            if not os.path.exists(new_path):
-                                                                  file_format.name + '.' + file_format.format.lower())),
+                os.makedirs(new_path)
-                                    os.path.normcase(os.path.join(all_new_path,
+            shutil.move(os.path.normcase(
-                                                                  all_new_name + '.' + file_format.format.lower())))
+                os.path.join(old_path, file_format.name + '.' + file_format.format.lower())),
+                os.path.normcase(
+                    os.path.join(new_path, all_new_name + '.' + file_format.format.lower())))
         else:
-                        g_file = gd.getFileFromEbooksFolder(all_new_path,
+            g_file = gd.getFileFromEbooksFolder(old_path,
                                                 file_format.name + '.' + file_format.format.lower())
             if g_file:
                 gd.moveGdriveFileRemote(g_file, all_new_name + '.' + file_format.format.lower())
                 gd.updateDatabaseOnEdit(g_file['id'], all_new_name + '.' + file_format.format.lower())
             else:
                 log.error("File {} not found on gdrive"
-                                      .format(all_new_path, file_format.name + '.' + file_format.format.lower()))
+                          .format(old_path, file_format.name + '.' + file_format.format.lower()))
+
+        # change name in Database
         file_format.name = all_new_name
 
 
@@ -455,8 +440,32 @@ def rename_all_authors(first_author, renamed_author, calibre_path="", localbook=
     return new_authordir
 
 
+def rename_author_path(first_author, old_author_dir, renamed_author, calibre_path="", gdrive=False):
+    # Create new_author_dir from parameter or from database
+    # Create new title_dir from database and add id
+    new_authordir = get_valid_filename(first_author, chars=96)
+    # new_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == renamed_author).first()
+    # old_author_dir = get_valid_filename(old_author_name, chars=96)
+    new_author_rename_dir = get_valid_filename(renamed_author, chars=96)
+    if gdrive:
+        g_file = gd.getFileFromEbooksFolder(None, old_author_dir)
+        if g_file:
+            gd.moveGdriveFolderRemote(g_file, new_author_rename_dir)
+    else:
+        if os.path.isdir(os.path.join(calibre_path, old_author_dir)):
+            old_author_path = os.path.join(calibre_path, old_author_dir)
+            new_author_path = os.path.join(calibre_path, new_author_rename_dir)
+            try:
+                shutil.move(os.path.normcase(old_author_path), os.path.normcase(new_author_path))
+            except OSError as ex:
+                log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex)
+                log.debug(ex, exc_info=True)
+                return _("Rename author from: '%(src)s' to '%(dest)s' failed with error: %(error)s",
+                         src=old_author_path, dest=new_author_path, error=str(ex))
+    return new_authordir
+
 # Moves files in file storage during author/title rename, or from temp dir to file storage
-def update_dir_structure_file(book_id, calibre_path, first_author, original_filepath, db_filename, renamed_author):
+def update_dir_structure_file(book_id, calibre_path, original_filepath, db_filename):
     # get book database entry from id, if original path overwrite source with original_filepath
     local_book = calibre_db.get_book(book_id)
     if original_filepath:
@@ -468,49 +477,47 @@ def update_dir_structure_file(book_id, calibre_path, first_author, original_file
     author_dir = local_book.path.split('/')[0]
     title_dir = local_book.path.split('/')[1]
 
-    # Create new_author_dir from parameter or from database
-    # Create new title_dir from database and add id
-    new_author_dir = rename_all_authors(first_author, renamed_author, calibre_path, local_book)
-    if first_author:
-        if first_author.lower() in [r.lower() for r in renamed_author]:
-            if os.path.isdir(os.path.join(calibre_path, new_author_dir)):
-                path = os.path.join(calibre_path, new_author_dir, title_dir)
-
     new_title_dir = get_valid_filename(local_book.title, chars=96) + " (" + str(book_id) + ")"
 
-    if title_dir != new_title_dir or author_dir != new_author_dir or original_filepath:
+    if title_dir != new_title_dir or original_filepath:
         error = move_files_on_change(calibre_path,
-                                     new_author_dir,
+                                     author_dir,
                                      new_title_dir,
                                      local_book,
                                      db_filename,
                                      original_filepath,
                                      path)
+        new_path = os.path.join(calibre_path, author_dir, new_title_dir).replace('\\', '/')
+        all_new_name = get_valid_filename(local_book.title, chars=42) + ' - ' \
+                       + get_valid_filename(author_dir, chars=42)
+        # Book folder already moved, only files need to be renamed
+        rename_all_files_on_change(local_book, new_path, new_path, all_new_name)
+
         if error:
             return error
 
     # Rename all files from old names to new names
-    return rename_files_on_change(first_author, renamed_author, local_book, original_filepath, path, calibre_path)
+    return False
 
 
-def upload_new_file_gdrive(book_id, first_author, renamed_author, title, title_dir, original_filepath, filename_ext):
+def upload_new_file_gdrive(book_id, first_author, title, title_dir, original_filepath, filename_ext):
     book = calibre_db.get_book(book_id)
     file_name = get_valid_filename(title, chars=42) + ' - ' + \
         get_valid_filename(first_author, chars=42) + filename_ext
-    rename_all_authors(first_author, renamed_author, gdrive=True)
     gdrive_path = os.path.join(get_valid_filename(first_author, chars=96),
                                title_dir + " (" + str(book_id) + ")")
     book.path = gdrive_path.replace("\\", "/")
     gd.uploadFileToEbooksFolder(os.path.join(gdrive_path, file_name).replace("\\", "/"), original_filepath)
-    return rename_files_on_change(first_author, renamed_author, local_book=book, gdrive=True)
+    return False # rename_files_on_change(first_author, renamed_author, local_book=book, gdrive=True)
 
 
-def update_dir_structure_gdrive(book_id, first_author, renamed_author):
+def update_dir_structure_gdrive(book_id):
     book = calibre_db.get_book(book_id)
 
     authordir = book.path.split('/')[0]
     titledir = book.path.split('/')[1]
-    new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True)
+    # new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True)
+    # new_authordir = get_valid_filename(book.title, chars=96)
     new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")"
 
     if titledir != new_titledir:
@@ -522,21 +529,25 @@ def update_dir_structure_gdrive(book_id, first_author, renamed_author):
         else:
             return _('File %(file)s not found on Google Drive', file=book.path)  # file not found
 
-    if authordir != new_authordir and authordir not in renamed_author:
+    '''if authordir != new_authordir:
         g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), new_titledir)
         if g_file:
             gd.moveGdriveFolderRemote(g_file, new_authordir)
             book.path = new_authordir + '/' + book.path.split('/')[1]
             gd.updateDatabaseOnEdit(g_file['id'], book.path)
         else:
-            return _('File %(file)s not found on Google Drive', file=authordir)  # file not found
+            return _('File %(file)s not found on Google Drive', file=authordir)  # file not found'''
-
+    if titledir != new_titledir:
+        all_new_name = get_valid_filename(book.title, chars=42) + ' - ' \
+                       + get_valid_filename(authordir, chars=42)
+        rename_all_files_on_change(book, book.path, book.path, all_new_name, gdrive=True)  # todo: Move filenames on gdrive
     # change location in database to new author/title path
-    book.path = os.path.join(new_authordir, new_titledir).replace('\\', '/')
+    # book.path = os.path.join(authordir, new_titledir).replace('\\', '/')
-    return rename_files_on_change(first_author, renamed_author, book, gdrive=True)
+    return False
 
 
 def move_files_on_change(calibre_path, new_authordir, new_titledir, localbook, db_filename, original_filepath, path):
+    new_authordir = get_valid_filename(new_authordir, chars=96)
     new_path = os.path.join(calibre_path, new_authordir, new_titledir)
     new_name = get_valid_filename(localbook.title, chars=96) + ' - ' + new_authordir
     try:
@@ -575,15 +586,15 @@ def rename_files_on_change(first_author,
                            calibre_path="",
                            gdrive=False):
     # Rename all files from old names to new names
-    try:
+    #try:
-        clean_author_database(renamed_author, calibre_path, gdrive=gdrive)
+        #clean_author_database(renamed_author, calibre_path, gdrive=gdrive)
-        if first_author and first_author not in renamed_author:
+        #if first_author and first_author not in renamed_author:
-            clean_author_database([first_author], calibre_path, local_book, gdrive)
+        #    clean_author_database([first_author], calibre_path, local_book, gdrive)
-        if not gdrive and not renamed_author and not original_filepath and len(os.listdir(os.path.dirname(path))) == 0:
+        #if not gdrive and not renamed_author and not original_filepath and len(os.listdir(os.path.dirname(path))) == 0:
-            shutil.rmtree(os.path.dirname(path))
+        #    shutil.rmtree(os.path.dirname(path))
-    except (OSError, FileNotFoundError) as ex:
+    #except (OSError, FileNotFoundError) as ex:
-        log.error_or_exception("Error in rename file in path {}".format(ex))
+    #    log.error_or_exception("Error in rename file in path {}".format(ex))
-        return _("Error in rename file in path: {}".format(str(ex)))
+    #    return _("Error in rename file in path: {}".format(str(ex)))
     return False
 
 
@@ -648,12 +659,6 @@ def generate_random_password(min_length):
     return ''.join(password)
 
 
-'''def generate_random_password(min_length):
-    s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?"
-    passlen = min_length
-    return "".join(s[c % len(s)] for c in os.urandom(passlen))'''
-
-
 def uniq(inpt):
     output = []
     inpt = [" ".join(inp.split()) for inp in inpt]
@@ -717,17 +722,14 @@ def update_dir_structure(book_id,
                          calibre_path,
                          first_author=None,     # change author of book to this author
                          original_filepath=None,
-                         db_filename=None,
+                         db_filename=None):
-                         renamed_author=None):
-    renamed_author = renamed_author or []
     if config.config_use_google_drive:
-        return update_dir_structure_gdrive(book_id, first_author, renamed_author)
+        return update_dir_structure_gdrive(book_id, first_author)
     else:
         return update_dir_structure_file(book_id,
                                          calibre_path,
-                                         first_author,
                                          original_filepath,
-                                         db_filename, renamed_author)
+                                         db_filename)
 
 
 def delete_book(book, calibrepath, book_format):