Refactored rename_authors

Bugfixes from tests
This commit is contained in:
Ozzie Isaacs 2024-06-28 19:38:01 +02:00
parent b28a3635f8
commit 09c0794f1a
5 changed files with 1085 additions and 1747 deletions

View File

@ -57,9 +57,9 @@ mimetypes.init()
mimetypes.add_type('application/xhtml+xml', '.xhtml')
mimetypes.add_type('application/epub+zip', '.epub')
mimetypes.add_type('application/epub+zip', '.kepub')
mimetypes.add_type('application/fb2+zip', '.fb2')
mimetypes.add_type('application/x-mobipocket-ebook', '.mobi')
mimetypes.add_type('application/x-mobipocket-ebook', '.prc')
mimetypes.add_type('text/xml', '.fb2')
mimetypes.add_type('application/octet-stream', '.mobi')
mimetypes.add_type('application/octet-stream', '.prc')
mimetypes.add_type('application/vnd.amazon.ebook', '.azw')
mimetypes.add_type('application/x-mobi8-ebook', '.azw3')
mimetypes.add_type('application/x-rar', '.cbr')
@ -69,10 +69,10 @@ mimetypes.add_type('application/x-7z-compressed', '.cb7')
mimetypes.add_type('image/vnd.djv', '.djv')
mimetypes.add_type('image/vnd.djv', '.djvu')
mimetypes.add_type('application/mpeg', '.mpeg')
mimetypes.add_type('application/mpeg', '.mp3')
mimetypes.add_type('audio/mpeg', '.mp3')
mimetypes.add_type('application/mp4', '.m4a')
mimetypes.add_type('application/mp4', '.m4b')
mimetypes.add_type('application/ogg', '.ogg')
mimetypes.add_type('audio/ogg', '.ogg')
mimetypes.add_type('application/ogg', '.oga')
mimetypes.add_type('text/css', '.css')
mimetypes.add_type('application/x-ms-reader', '.lit')

View File

@ -23,7 +23,7 @@
import os
from datetime import datetime
import json
from shutil import copyfile
from shutil import copyfile, move
from uuid import uuid4
from markupsafe import escape, Markup # dependency of flask
from functools import wraps
@ -118,14 +118,13 @@ def edit_book(book_id):
# handle book title change
title_change = handle_title_on_edit(book, to_save["book_title"])
# handle book author change
input_authors, author_change, renamed = handle_author_on_edit(book, to_save["author_name"])
input_authors, author_change = handle_author_on_edit(book, to_save["author_name"])
if author_change or title_change:
edited_books_id = book.id
modify_date = True
title_author_error = helper.update_dir_structure(edited_books_id,
config.get_book_path(),
input_authors[0],
renamed_author=renamed)
input_authors[0])
if title_author_error:
flash(title_author_error, category="error")
calibre_db.session.rollback()
@ -251,7 +250,7 @@ def upload():
if error:
return error
db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta)
db_book, input_authors, title_dir = create_book_on_upload(modify_date, meta)
# Comments need book id therefore only possible after flush
modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
@ -261,7 +260,6 @@ def upload():
if config.config_use_google_drive:
helper.upload_new_file_gdrive(book_id,
input_authors[0],
renamed_authors,
title,
title_dir,
meta.file_path,
@ -271,8 +269,7 @@ def upload():
config.get_book_path(),
input_authors[0],
meta.file_path,
title_dir + meta.extension.lower(),
renamed_author=renamed_authors)
title_dir + meta.extension.lower())
move_coverfile(meta, db_book)
@ -405,9 +402,8 @@ def edit_list_book(param):
ret = Response(json.dumps({'success': True, 'newValue': book.comments[0].text}),
mimetype='application/json')
elif param == 'authors':
input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0],
renamed_author=renamed)
input_authors, __ = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0])
if not rename_error:
ret = Response(json.dumps({
'success': True,
@ -543,7 +539,7 @@ def table_xchange_author_title():
author_names.append(authr.name.replace('|', ','))
title_change = handle_title_on_edit(book, " ".join(author_names))
input_authors, author_change, renamed = handle_author_on_edit(book, authors)
input_authors, author_change = handle_author_on_edit(book, authors)
if author_change or title_change:
edited_books_id = book.id
modify_date = True
@ -553,8 +549,7 @@ def table_xchange_author_title():
if edited_books_id:
# toDo: Handle error
edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0],
renamed_author=renamed)
edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0])
if modify_date:
book.last_modified = datetime.utcnow()
calibre_db.set_metadata_dirty(book.id)
@ -602,7 +597,9 @@ def identifier_list(to_save, book):
return result
def prepare_authors(authr):
def prepare_authors(authr, calibre_path, gdrive=False):
if gdrive:
calibre_path = ""
# handle authors
input_authors = authr.split('&')
# handle_authors(input_authors)
@ -614,20 +611,44 @@ def prepare_authors(authr):
if input_authors == ['']:
input_authors = [_('Unknown')] # prevent empty Author
renamed = list()
for index,in_aut in enumerate(input_authors):
# renamed_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == in_aut).first()
for in_aut in input_authors:
renamed_author = calibre_db.session.query(db.Authors).filter(func.lower(db.Authors.name).ilike(in_aut)).first()
if renamed_author and in_aut != renamed_author.name:
renamed.append(renamed_author.name)
old_author_name = renamed_author.name
# rename author in Database
create_objects_for_addition(renamed_author, in_aut,"author")
# rename all Books with this author as first author:
# rename all book author_sort strings with the new author name
all_books = calibre_db.session.query(db.Books) \
.filter(db.Books.authors.any(db.Authors.name == renamed_author.name)).all()
sorted_renamed_author = helper.get_sorted_author(renamed_author.name)
sorted_old_author = helper.get_sorted_author(in_aut)
for one_book in all_books:
one_book.author_sort = one_book.author_sort.replace(sorted_renamed_author, sorted_old_author)
input_authors[index] = renamed_author.name
return input_authors, renamed
# ToDo: check
sorted_old_author = helper.get_sorted_author(old_author_name)
sorted_renamed_author = helper.get_sorted_author(in_aut)
# change author sort path
try:
author_index = one_book.author_sort.index(sorted_old_author)
one_book.author_sort = one_book.author_sort.replace(sorted_old_author, sorted_renamed_author)
except ValueError:
log.error("Sorted author {} not found in database".format(sorted_old_author))
author_index = -1
# change book path if changed author is first author -> match on first position
if author_index == 0:
one_titledir = one_book.path.split('/')[1]
one_old_authordir = one_book.path.split('/')[0]
# rename author path only once per renamed author -> search all books with author name in book.path
# das muss einmal geschehen aber pro Buch geprüft werden ansonsten habe ich das Problem das vlt. 2 gleiche Ordner bis auf Groß/Kleinschreibung vorhanden sind im Umzug
new_author_dir = helper.rename_author_path(in_aut, one_old_authordir, renamed_author.name, calibre_path, gdrive)
one_book.path = os.path.join(new_author_dir, one_titledir).replace('\\', '/')
# rename all books in book data with the new author name and move corresponding files to new locations
# old_path = os.path.join(calibre_path, new_author_dir, one_titledir)
new_path = os.path.join(calibre_path, new_author_dir, one_titledir)
all_new_name = helper.get_valid_filename(one_book.title, chars=42) + ' - ' \
+ helper.get_valid_filename(renamed_author.name, chars=42)
# change location in database to new author/title path
helper.rename_all_files_on_change(one_book, new_path, new_path, all_new_name, gdrive)
return input_authors
def prepare_authors_on_upload(title, authr):
@ -638,7 +659,7 @@ def prepare_authors_on_upload(title, authr):
flash(_("Uploaded book probably exists in the library, consider to change before upload new: ")
+ Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")
input_authors, renamed = prepare_authors(authr)
input_authors = prepare_authors(authr, config.get_book_path(), config.config_use_google_drive)
sort_authors_list = list()
db_author = None
@ -657,13 +678,13 @@ def prepare_authors_on_upload(title, authr):
sort_author = stored_author.sort
sort_authors_list.append(sort_author)
sort_authors = ' & '.join(sort_authors_list)
return sort_authors, input_authors, db_author, renamed
return sort_authors, input_authors, db_author
def create_book_on_upload(modify_date, meta):
title = meta.title
authr = meta.author
sort_authors, input_authors, db_author, renamed_authors = prepare_authors_on_upload(title, authr)
sort_authors, input_authors, db_author = prepare_authors_on_upload(title, authr)
title_dir = helper.get_valid_filename(title, chars=96)
author_dir = helper.get_valid_filename(db_author.name, chars=96)
@ -720,14 +741,14 @@ def create_book_on_upload(modify_date, meta):
flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning")
modify_date |= modification
return db_book, input_authors, title_dir, renamed_authors
return db_book, input_authors, title_dir
def file_handling_on_upload(requested_file):
# check if file extension is correct
allowed_extensions = config.config_upload_formats.split(',')
if requested_file:
if config.config_check_extensions:
if config.config_check_extensions and allowed_extensions != ['']:
if not validate_mime_type(requested_file, allowed_extensions):
flash(_("File type isn't allowed to be uploaded to this server"), category="error")
return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
@ -1163,7 +1184,7 @@ def upload_single_file(file_request, book, book_id):
requested_file = file_request.files.get('btn-upload-format', None)
allowed_extensions = config.config_upload_formats.split(',')
if requested_file:
if config.config_check_extensions:
if config.config_check_extensions and allowed_extensions != ['']:
if not validate_mime_type(requested_file, allowed_extensions):
flash(_("File type isn't allowed to be uploaded to this server"), category="error")
return False
@ -1262,7 +1283,7 @@ def handle_title_on_edit(book, book_title):
def handle_author_on_edit(book, author_name, update_stored=True):
change = False
# handle author(s)
input_authors, renamed = prepare_authors(author_name)
input_authors = prepare_authors(author_name, config.get_book_path(), config.config_use_google_drive)
# change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
# Search for each author if author is in database, if not, author name and sorted author name is generated new
@ -1282,7 +1303,7 @@ def handle_author_on_edit(book, author_name, update_stored=True):
change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
return input_authors, change, renamed
return input_authors, change
def search_objects_remove(db_book_object, db_type, input_elements):

View File

@ -581,7 +581,7 @@ def get_cover_via_gdrive(cover_path):
session.add(permissionAdded)
try:
session.commit()
except OperationalError as ex:
except (OperationalError, IntegrityError) as ex:
log.error_or_exception('Database error: {}'.format(ex))
session.rollback()
return df.metadata.get('webContentLink')

View File

@ -388,41 +388,26 @@ def delete_book_file(book, calibrepath, book_format=None):
id=book.id,
path=book.path)
def clean_author_database(renamed_author, calibre_path="", local_book=None, gdrive=None):
valid_filename_authors = [get_valid_filename(r, chars=96) for r in renamed_author]
for r in renamed_author:
if local_book:
all_books = [local_book]
else:
all_books = calibre_db.session.query(db.Books) \
.filter(db.Books.authors.any(db.Authors.name == r)).all()
for book in all_books:
book_author_path = book.path.split('/')[0]
if book_author_path in valid_filename_authors or local_book:
new_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == r).first()
all_new_authordir = get_valid_filename(new_author.name, chars=96)
all_titledir = book.path.split('/')[1]
all_new_path = os.path.join(calibre_path, all_new_authordir, all_titledir)
all_new_name = get_valid_filename(book.title, chars=42) + ' - ' \
+ get_valid_filename(new_author.name, chars=42)
# change location in database to new author/title path
book.path = os.path.join(all_new_authordir, all_titledir).replace('\\', '/')
for file_format in book.data:
def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdrive=False):
for file_format in one_book.data:
if not gdrive:
shutil.move(os.path.normcase(os.path.join(all_new_path,
file_format.name + '.' + file_format.format.lower())),
os.path.normcase(os.path.join(all_new_path,
all_new_name + '.' + file_format.format.lower())))
if not os.path.exists(new_path):
os.makedirs(new_path)
shutil.move(os.path.normcase(
os.path.join(old_path, file_format.name + '.' + file_format.format.lower())),
os.path.normcase(
os.path.join(new_path, all_new_name + '.' + file_format.format.lower())))
else:
g_file = gd.getFileFromEbooksFolder(all_new_path,
g_file = gd.getFileFromEbooksFolder(old_path,
file_format.name + '.' + file_format.format.lower())
if g_file:
gd.moveGdriveFileRemote(g_file, all_new_name + '.' + file_format.format.lower())
gd.updateDatabaseOnEdit(g_file['id'], all_new_name + '.' + file_format.format.lower())
else:
log.error("File {} not found on gdrive"
.format(all_new_path, file_format.name + '.' + file_format.format.lower()))
.format(old_path, file_format.name + '.' + file_format.format.lower()))
# change name in Database
file_format.name = all_new_name
@ -455,8 +440,32 @@ def rename_all_authors(first_author, renamed_author, calibre_path="", localbook=
return new_authordir
def rename_author_path(first_author, old_author_dir, renamed_author, calibre_path="", gdrive=False):
# Create new_author_dir from parameter or from database
# Create new title_dir from database and add id
new_authordir = get_valid_filename(first_author, chars=96)
# new_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == renamed_author).first()
# old_author_dir = get_valid_filename(old_author_name, chars=96)
new_author_rename_dir = get_valid_filename(renamed_author, chars=96)
if gdrive:
g_file = gd.getFileFromEbooksFolder(None, old_author_dir)
if g_file:
gd.moveGdriveFolderRemote(g_file, new_author_rename_dir)
else:
if os.path.isdir(os.path.join(calibre_path, old_author_dir)):
old_author_path = os.path.join(calibre_path, old_author_dir)
new_author_path = os.path.join(calibre_path, new_author_rename_dir)
try:
shutil.move(os.path.normcase(old_author_path), os.path.normcase(new_author_path))
except OSError as ex:
log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex)
log.debug(ex, exc_info=True)
return _("Rename author from: '%(src)s' to '%(dest)s' failed with error: %(error)s",
src=old_author_path, dest=new_author_path, error=str(ex))
return new_authordir
# Moves files in file storage during author/title rename, or from temp dir to file storage
def update_dir_structure_file(book_id, calibre_path, first_author, original_filepath, db_filename, renamed_author):
def update_dir_structure_file(book_id, calibre_path, original_filepath, db_filename):
# get book database entry from id, if original path overwrite source with original_filepath
local_book = calibre_db.get_book(book_id)
if original_filepath:
@ -468,49 +477,47 @@ def update_dir_structure_file(book_id, calibre_path, first_author, original_file
author_dir = local_book.path.split('/')[0]
title_dir = local_book.path.split('/')[1]
# Create new_author_dir from parameter or from database
# Create new title_dir from database and add id
new_author_dir = rename_all_authors(first_author, renamed_author, calibre_path, local_book)
if first_author:
if first_author.lower() in [r.lower() for r in renamed_author]:
if os.path.isdir(os.path.join(calibre_path, new_author_dir)):
path = os.path.join(calibre_path, new_author_dir, title_dir)
new_title_dir = get_valid_filename(local_book.title, chars=96) + " (" + str(book_id) + ")"
if title_dir != new_title_dir or author_dir != new_author_dir or original_filepath:
if title_dir != new_title_dir or original_filepath:
error = move_files_on_change(calibre_path,
new_author_dir,
author_dir,
new_title_dir,
local_book,
db_filename,
original_filepath,
path)
new_path = os.path.join(calibre_path, author_dir, new_title_dir).replace('\\', '/')
all_new_name = get_valid_filename(local_book.title, chars=42) + ' - ' \
+ get_valid_filename(author_dir, chars=42)
# Book folder already moved, only files need to be renamed
rename_all_files_on_change(local_book, new_path, new_path, all_new_name)
if error:
return error
# Rename all files from old names to new names
return rename_files_on_change(first_author, renamed_author, local_book, original_filepath, path, calibre_path)
return False
def upload_new_file_gdrive(book_id, first_author, renamed_author, title, title_dir, original_filepath, filename_ext):
def upload_new_file_gdrive(book_id, first_author, title, title_dir, original_filepath, filename_ext):
book = calibre_db.get_book(book_id)
file_name = get_valid_filename(title, chars=42) + ' - ' + \
get_valid_filename(first_author, chars=42) + filename_ext
rename_all_authors(first_author, renamed_author, gdrive=True)
gdrive_path = os.path.join(get_valid_filename(first_author, chars=96),
title_dir + " (" + str(book_id) + ")")
book.path = gdrive_path.replace("\\", "/")
gd.uploadFileToEbooksFolder(os.path.join(gdrive_path, file_name).replace("\\", "/"), original_filepath)
return rename_files_on_change(first_author, renamed_author, local_book=book, gdrive=True)
return False # rename_files_on_change(first_author, renamed_author, local_book=book, gdrive=True)
def update_dir_structure_gdrive(book_id, first_author, renamed_author):
def update_dir_structure_gdrive(book_id):
book = calibre_db.get_book(book_id)
authordir = book.path.split('/')[0]
titledir = book.path.split('/')[1]
new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True)
# new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True)
# new_authordir = get_valid_filename(book.title, chars=96)
new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")"
if titledir != new_titledir:
@ -522,21 +529,25 @@ def update_dir_structure_gdrive(book_id, first_author, renamed_author):
else:
return _('File %(file)s not found on Google Drive', file=book.path) # file not found
if authordir != new_authordir and authordir not in renamed_author:
'''if authordir != new_authordir:
g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), new_titledir)
if g_file:
gd.moveGdriveFolderRemote(g_file, new_authordir)
book.path = new_authordir + '/' + book.path.split('/')[1]
gd.updateDatabaseOnEdit(g_file['id'], book.path)
else:
return _('File %(file)s not found on Google Drive', file=authordir) # file not found
return _('File %(file)s not found on Google Drive', file=authordir) # file not found'''
if titledir != new_titledir:
all_new_name = get_valid_filename(book.title, chars=42) + ' - ' \
+ get_valid_filename(authordir, chars=42)
rename_all_files_on_change(book, book.path, book.path, all_new_name, gdrive=True) # todo: Move filenames on gdrive
# change location in database to new author/title path
book.path = os.path.join(new_authordir, new_titledir).replace('\\', '/')
return rename_files_on_change(first_author, renamed_author, book, gdrive=True)
# book.path = os.path.join(authordir, new_titledir).replace('\\', '/')
return False
def move_files_on_change(calibre_path, new_authordir, new_titledir, localbook, db_filename, original_filepath, path):
new_authordir = get_valid_filename(new_authordir, chars=96)
new_path = os.path.join(calibre_path, new_authordir, new_titledir)
new_name = get_valid_filename(localbook.title, chars=96) + ' - ' + new_authordir
try:
@ -575,15 +586,15 @@ def rename_files_on_change(first_author,
calibre_path="",
gdrive=False):
# Rename all files from old names to new names
try:
clean_author_database(renamed_author, calibre_path, gdrive=gdrive)
if first_author and first_author not in renamed_author:
clean_author_database([first_author], calibre_path, local_book, gdrive)
if not gdrive and not renamed_author and not original_filepath and len(os.listdir(os.path.dirname(path))) == 0:
shutil.rmtree(os.path.dirname(path))
except (OSError, FileNotFoundError) as ex:
log.error_or_exception("Error in rename file in path {}".format(ex))
return _("Error in rename file in path: {}".format(str(ex)))
#try:
#clean_author_database(renamed_author, calibre_path, gdrive=gdrive)
#if first_author and first_author not in renamed_author:
# clean_author_database([first_author], calibre_path, local_book, gdrive)
#if not gdrive and not renamed_author and not original_filepath and len(os.listdir(os.path.dirname(path))) == 0:
# shutil.rmtree(os.path.dirname(path))
#except (OSError, FileNotFoundError) as ex:
# log.error_or_exception("Error in rename file in path {}".format(ex))
# return _("Error in rename file in path: {}".format(str(ex)))
return False
@ -648,12 +659,6 @@ def generate_random_password(min_length):
return ''.join(password)
'''def generate_random_password(min_length):
s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?"
passlen = min_length
return "".join(s[c % len(s)] for c in os.urandom(passlen))'''
def uniq(inpt):
output = []
inpt = [" ".join(inp.split()) for inp in inpt]
@ -717,17 +722,14 @@ def update_dir_structure(book_id,
calibre_path,
first_author=None, # change author of book to this author
original_filepath=None,
db_filename=None,
renamed_author=None):
renamed_author = renamed_author or []
db_filename=None):
if config.config_use_google_drive:
return update_dir_structure_gdrive(book_id, first_author, renamed_author)
return update_dir_structure_gdrive(book_id, first_author)
else:
return update_dir_structure_file(book_id,
calibre_path,
first_author,
original_filepath,
db_filename, renamed_author)
db_filename)
def delete_book(book, calibrepath, book_format):

File diff suppressed because it is too large Load Diff