NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'Develop' (Fix for #3058, refactored rename author and improved duplicate check for tags/publishers, series, ...)

Browse Source
This commit is contained in:
Ozzie Isaacs 2024-07-03 15:39:05 +02:00
commit ab2620a265
6 changed files with 438 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
cps/gdriveutils.py
View File

@ -22,7 +22,9 @@ import shutil
import chardet import chardet
import ssl import ssl
import sqlite3 import sqlite3
import mimetypes
from werkzeug.datastructures import Headers
from flask import Response, stream_with_context from flask import Response, stream_with_context
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy import Column, UniqueConstraint from sqlalchemy import Column, UniqueConstraint
@ -64,7 +66,7 @@ except ImportError as err:
importError = err importError = err
gdrive_support = False gdrive_support = False
from . import logger, cli_param, config from . import logger, cli_param, config, db
from .constants import CONFIG_DIR as _CONFIG_DIR from .constants import CONFIG_DIR as _CONFIG_DIR
@ -265,7 +267,7 @@ def getFile(pathId, fileName, drive, nocase):
if fileList.__len__() == 0: if fileList.__len__() == 0:
return None return None
if nocase: if nocase:
return fileList[0] return fileList[0] if db.lcase(fileList[0]['title']) == db.lcase(fileName) else None
for f in fileList: for f in fileList:
if f['title'] == fileName: if f['title'] == fileName:
return f return f
@ -273,8 +275,6 @@ def getFile(pathId, fileName, drive, nocase):
def getFolderId(path, drive): def getFolderId(path, drive):
# drive = getDrive(drive)
log.info(f"GetFolder: {path}")
currentFolderId = None currentFolderId = None
try: try:
currentFolderId = getEbooksFolderId(drive) currentFolderId = getEbooksFolderId(drive)
@ -348,16 +348,23 @@ def moveGdriveFolderRemote(origin_file, target_folder, single_book=False):
previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')]) previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')])
children = drive.auth.service.children().list(folderId=previous_parents).execute() children = drive.auth.service.children().list(folderId=previous_parents).execute()
if single_book: if single_book:
# gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True)
gFileTargetDir = drive.CreateFile( if gFileTargetDir:
{'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}], # Move the file to the new folder
"mimeType": "application/vnd.google-apps.folder"}) drive.auth.service.files().update(fileId=origin_file['id'],
gFileTargetDir.Upload() addParents=gFileTargetDir['id'],
# Move the file to the new folder removeParents=previous_parents,
drive.auth.service.files().update(fileId=origin_file['id'], fields='id, parents').execute()
addParents=gFileTargetDir['id'], else:
removeParents=previous_parents, gFileTargetDir = drive.CreateFile(
fields='id, parents').execute() {'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}],
"mimeType": "application/vnd.google-apps.folder"})
gFileTargetDir.Upload()
# Move the file to the new folder
drive.auth.service.files().update(fileId=origin_file['id'],
addParents=gFileTargetDir['id'],
removeParents=previous_parents,
fields='id, parents').execute()
elif origin_file['title'] != target_folder: elif origin_file['title'] != target_folder:
#gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) #gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True)
#if gFileTargetDir: #if gFileTargetDir:
@ -366,12 +373,7 @@ def moveGdriveFolderRemote(origin_file, target_folder, single_book=False):
drive.auth.service.files().patch(fileId=origin_file['id'], drive.auth.service.files().patch(fileId=origin_file['id'],
body={'title': target_folder}, body={'title': target_folder},
fields='title').execute() fields='title').execute()
'''else:
# Move the file to the new folder
drive.auth.service.files().update(fileId=origin_file['id'],
addParents=gFileTargetDir['id'],
removeParents=previous_parents,
fields='id, parents').execute()'''
# if previous_parents has no children anymore, delete original fileparent # if previous_parents has no children anymore, delete original fileparent
if len(children['items']) == 1: if len(children['items']) == 1:
deleteDatabaseEntry(previous_parents) deleteDatabaseEntry(previous_parents)
@ -600,7 +602,10 @@ def get_cover_via_gdrive(cover_path):
except (OperationalError, IntegrityError) as ex: except (OperationalError, IntegrityError) as ex:
log.error_or_exception('Database error: {}'.format(ex)) log.error_or_exception('Database error: {}'.format(ex))
session.rollback() session.rollback()
return df.metadata.get('webContentLink') headers = Headers()
headers["Content-Type"] = 'image/jpeg'
resp, content = df.auth.Get_Http_Object().request(df.metadata.get('downloadUrl'), headers=headers)
return content
else: else:
return None return None

27
cps/helper.py
View File

@ -30,7 +30,7 @@ import requests
import unidecode import unidecode
from uuid import uuid4 from uuid import uuid4
from flask import send_from_directory, make_response, redirect, abort, url_for from flask import send_from_directory, make_response, abort, url_for, Response
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_babel import lazy_gettext as N_ from flask_babel import lazy_gettext as N_
from flask_babel import get_locale from flask_babel import get_locale
@ -393,10 +393,8 @@ def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdriv
if not gdrive: if not gdrive:
if not os.path.exists(new_path): if not os.path.exists(new_path):
os.makedirs(new_path) os.makedirs(new_path)
shutil.move(os.path.normcase( shutil.move(os.path.join(old_path, file_format.name + '.' + file_format.format.lower()),
os.path.join(old_path, file_format.name + '.' + file_format.format.lower())), os.path.join(new_path, all_new_name + '.' + file_format.format.lower()))
os.path.normcase(
os.path.join(new_path, all_new_name + '.' + file_format.format.lower())))
else: else:
g_file = gd.getFileFromEbooksFolder(old_path, g_file = gd.getFileFromEbooksFolder(old_path,
file_format.name + '.' + file_format.format.lower()) file_format.name + '.' + file_format.format.lower())
@ -457,7 +455,7 @@ def rename_author_path(first_author, old_author_dir, renamed_author, calibre_pat
old_author_path = os.path.join(calibre_path, old_author_dir) old_author_path = os.path.join(calibre_path, old_author_dir)
new_author_path = os.path.join(calibre_path, new_author_rename_dir) new_author_path = os.path.join(calibre_path, new_author_rename_dir)
try: try:
shutil.move(os.path.normcase(old_author_path), os.path.normcase(new_author_path)) shutil.move(old_author_path, new_author_path)
except OSError as ex: except OSError as ex:
log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex) log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex)
log.debug(ex, exc_info=True) log.debug(ex, exc_info=True)
@ -527,7 +525,7 @@ def update_dir_structure_gdrive(book_id, first_author):
new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")" new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")"
if titledir != new_titledir: if titledir != new_titledir:
g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), titledir) g_file = gd.getFileFromEbooksFolder(authordir, titledir)
if g_file: if g_file:
gd.moveGdriveFileRemote(g_file, new_titledir) gd.moveGdriveFileRemote(g_file, new_titledir)
book.path = book.path.split('/')[0] + '/' + new_titledir book.path = book.path.split('/')[0] + '/' + new_titledir
@ -559,21 +557,20 @@ def move_files_on_change(calibre_path, new_author_dir, new_titledir, localbook,
if original_filepath: if original_filepath:
if not os.path.isdir(new_path): if not os.path.isdir(new_path):
os.makedirs(new_path) os.makedirs(new_path)
shutil.move(os.path.normcase(original_filepath), os.path.normcase(os.path.join(new_path, db_filename))) shutil.move(original_filepath, os.path.join(new_path, db_filename))
log.debug("Moving title: %s to %s/%s", original_filepath, new_path) log.debug("Moving title: %s to %s/%s", original_filepath, new_path)
else: else:
# Check new path is not valid path # Check new path is not valid path
if not os.path.exists(new_path): if not os.path.exists(new_path):
# move original path to new path # move original path to new path
log.debug("Moving title: %s to %s", path, new_path) log.debug("Moving title: %s to %s", path, new_path)
shutil.move(os.path.normcase(path), os.path.normcase(new_path)) shutil.move(path, new_path)
else: # path is valid copy only files to new location (merge) else: # path is valid copy only files to new location (merge)
log.info("Moving title: %s into existing: %s", path, new_path) log.info("Moving title: %s into existing: %s", path, new_path)
# Take all files and subfolder from old path (strange command) # Take all files and subfolder from old path (strange command)
for dir_name, __, file_list in os.walk(path): for dir_name, __, file_list in os.walk(path):
for file in file_list: for file in file_list:
shutil.move(os.path.normcase(os.path.join(dir_name, file)), shutil.move(os.path.join(dir_name, file), os.path.join(new_path + dir_name[len(path):], file))
os.path.normcase(os.path.join(new_path + dir_name[len(path):], file)))
if not os.listdir(os.path.split(path)[0]): if not os.listdir(os.path.split(path)[0]):
try: try:
shutil.rmtree(os.path.split(path)[0]) shutil.rmtree(os.path.split(path)[0])
@ -615,7 +612,7 @@ def delete_book_gdrive(book, book_format):
for entry in book.data: for entry in book.data:
if entry.format.upper() == book_format: if entry.format.upper() == book_format:
name = entry.name + '.' + book_format name = entry.name + '.' + book_format
g_file = gd.getFileFromEbooksFolder(book.path, name) g_file = gd.getFileFromEbooksFolder(book.path, name, nocase=True)
else: else:
g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), book.path.split('/')[1]) g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), book.path.split('/')[1])
if g_file: if g_file:
@ -790,9 +787,9 @@ def get_book_cover_internal(book, resolution=None):
try: try:
if not gd.is_gdrive_ready(): if not gd.is_gdrive_ready():
return get_cover_on_failure() return get_cover_on_failure()
path = gd.get_cover_via_gdrive(book.path) cover_file = gd.get_cover_via_gdrive(book.path)
if path: if cover_file:
return redirect(path) return Response(cover_file, mimetype='image/jpeg')
else: else:
log.error('{}/cover.jpg not found on Google Drive'.format(book.path)) log.error('{}/cover.jpg not found on Google Drive'.format(book.path))
return get_cover_on_failure() return get_cover_on_failure()

9
cps/tasks/thumbnail.py
View File

@ -19,6 +19,7 @@
import os import os
from shutil import copyfile, copyfileobj from shutil import copyfile, copyfileobj
from urllib.request import urlopen from urllib.request import urlopen
from io import BytesIO
from .. import constants from .. import constants
from cps import config, db, fs, gdriveutils, logger, ub from cps import config, db, fs, gdriveutils, logger, ub
@ -182,13 +183,11 @@ class TaskGenerateCoverThumbnails(CalibreTask):
if not gdriveutils.is_gdrive_ready(): if not gdriveutils.is_gdrive_ready():
raise Exception('Google Drive is configured but not ready') raise Exception('Google Drive is configured but not ready')
web_content_link = gdriveutils.get_cover_via_gdrive(book.path) content = gdriveutils.get_cover_via_gdrive(book.path)
if not web_content_link: if not content:
raise Exception('Google Drive cover url not found') raise Exception('Google Drive cover url not found')
stream = None
try: try:
stream = urlopen(web_content_link) stream = BytesIO(content)
with Image(file=stream) as img: with Image(file=stream) as img:
filename = self.cache.get_cache_file_path(thumbnail.filename, filename = self.cache.get_cache_file_path(thumbnail.filename,
constants.CACHE_TYPE_THUMBNAILS) constants.CACHE_TYPE_THUMBNAILS)

4
optional-requirements.txt
View File

@ -1,5 +1,5 @@
# GDrive Integration # GDrive Integration
google-api-python-client>=1.7.11,<2.120.0 google-api-python-client>=1.7.11,<2.200.0
gevent>20.6.0,<24.3.0 gevent>20.6.0,<24.3.0
greenlet>=0.4.17,<3.1.0 greenlet>=0.4.17,<3.1.0
httplib2>=0.9.2,<0.23.0 httplib2>=0.9.2,<0.23.0
@ -13,7 +13,7 @@ rsa>=3.4.2,<4.10.0
# Gmail # Gmail
google-auth-oauthlib>=0.4.3,<1.3.0 google-auth-oauthlib>=0.4.3,<1.3.0
google-api-python-client>=1.7.11,<2.120.0 google-api-python-client>=1.7.11,<2.200.0
# goodreads # goodreads
goodreads>=0.3.2,<0.4.0 goodreads>=0.3.2,<0.4.0

10
setup.cfg
View File

@ -46,19 +46,19 @@ install_requires =
Flask-Principal>=0.3.2,<0.5.1 Flask-Principal>=0.3.2,<0.5.1
Flask>=1.0.2,<3.1.0 Flask>=1.0.2,<3.1.0
iso-639>=0.4.5,<0.5.0 iso-639>=0.4.5,<0.5.0
PyPDF>=3.15.6,<4.1.0 PyPDF>=3.15.6,<4.3.0
pytz>=2016.10 pytz>=2016.10
requests>=2.28.0,<2.32.0 requests>=2.28.0,<2.32.0
SQLAlchemy>=1.3.0,<2.1.0 SQLAlchemy>=1.3.0,<2.1.0
tornado>=6.3,<6.5 tornado>=6.3,<6.5
Wand>=0.4.4,<0.7.0 Wand>=0.4.4,<0.7.0
unidecode>=0.04.19,<1.4.0 unidecode>=0.04.19,<1.4.0
lxml>=4.9.1,<5.2.0 lxml>=4.9.1,<5.3.0
flask-wtf>=0.14.2,<1.3.0 flask-wtf>=0.14.2,<1.3.0
chardet>=3.0.0,<4.1.0 chardet>=3.0.0,<4.1.0
advocate>=1.0.0,<1.1.0 advocate>=1.0.0,<1.1.0
Flask-Limiter>=2.3.0,<3.6.0 Flask-Limiter>=2.3.0,<3.6.0
regex>=2022.3.2,<2024.2.25 regex>=2022.3.2,<2024.6.25
bleach>=6.0.0,<6.2.0 bleach>=6.0.0,<6.2.0
python-magic>=0.4.27,<0.5.0 python-magic>=0.4.27,<0.5.0
@ -69,7 +69,7 @@ include = cps/services*
[options.extras_require] [options.extras_require]
gdrive = gdrive =
google-api-python-client>=1.7.11,<2.120.0 google-api-python-client>=1.7.11,<2.200.0
gevent>20.6.0,<24.3.0 gevent>20.6.0,<24.3.0
greenlet>=0.4.17,<3.1.0 greenlet>=0.4.17,<3.1.0
httplib2>=0.9.2,<0.23.0 httplib2>=0.9.2,<0.23.0
@ -82,7 +82,7 @@ gdrive =
rsa>=3.4.2,<4.10.0 rsa>=3.4.2,<4.10.0
gmail = gmail =
google-auth-oauthlib>=0.4.3,<1.3.0 google-auth-oauthlib>=0.4.3,<1.3.0
google-api-python-client>=1.7.11,<2.120.0 google-api-python-client>=1.7.11,<2.200.0
goodreads = goodreads =
goodreads>=0.3.2,<0.4.0 goodreads>=0.3.2,<0.4.0
python-Levenshtein>=0.12.0,<0.26.0 python-Levenshtein>=0.12.0,<0.26.0

851
test/Calibre-Web TestSummary_Linux.html
View File

File diff suppressed because it is too large Load Diff