From ca212c8737f8ec02f4e179199d34da9f4a7fa029 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Mon, 4 Jan 2021 18:45:25 +0100 Subject: [PATCH 01/22] Kobo sync limit set to 100 Remove python2 support from readme --- README.md | 6 +++--- cps/kobo.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4ee209f8..3f0792bf 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d ## Quick start -1. Install dependencies by running `pip3 install --target vendor -r requirements.txt` (python3.x) or `pip install --target vendor -r requirements.txt` (python2.7). -2. Execute the command: `python cps.py` (or `nohup python cps.py` - recommended if you want to exit the terminal window) +1. Install dependencies by running `pip3 install --target vendor -r requirements.txt` (python3.x). Alternativly set up a python virtual environment. +2. Execute the command: `python3 cps.py` (or `nohup python3 cps.py` - recommended if you want to exit the terminal window) 3. Point your browser to `http://localhost:8083` or `http://localhost:8083/opds` for the OPDS catalog 4. Set `Location of Calibre database` to the path of the folder where your Calibre library (metadata.db) lives, push "submit" button\ Optionally a Google Drive can be used to host the calibre library [-> Using Google Drive integration](https://github.com/janeczku/calibre-web/wiki/Configuration#using-google-drive-integration) @@ -48,7 +48,7 @@ Please note that running the above install command can fail on some versions of ## Requirements -python 3.x+, (Python 2.7+) +python 3.x+ Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata: diff --git a/cps/kobo.py b/cps/kobo.py index 06e89112..3210a450 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -58,7 +58,7 @@ KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]} KOBO_STOREAPI_URL = "https://storeapi.kobo.com" KOBO_IMAGEHOST_URL = "https://kbimages1-a.akamaihd.net" -SYNC_ITEM_LIMIT = 5 +SYNC_ITEM_LIMIT = 100 kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) From 54a78d556588912418bebe3118fb7b751ac2cea5 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Thu, 7 Jan 2021 17:40:18 +0100 Subject: [PATCH 02/22] Remove duplicate id from the search form --- cps/templates/search_form.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cps/templates/search_form.html b/cps/templates/search_form.html index e61fe067..050028ac 100644 --- a/cps/templates/search_form.html +++ b/cps/templates/search_form.html @@ -98,7 +98,7 @@
- {% for extension in extensions %} {% endfor %} @@ -106,7 +106,7 @@
- {% for extension in extensions %} {% endfor %} From b8ab66369e72a1f1706e679ffa0109d2644357ef Mon Sep 17 00:00:00 2001 From: jvoisin Date: Thu, 7 Jan 2021 17:59:08 +0100 Subject: [PATCH 03/22] Remove some unused imports --- cps.py | 2 +- cps/helper.py | 2 +- cps/kobo.py | 1 - cps/kobo_auth.py | 1 - cps/oauth_bb.py | 1 - cps/web.py | 2 +- 6 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cps.py b/cps.py index 50ab0076..737b0d97 100755 --- a/cps.py +++ b/cps.py @@ -31,7 +31,7 @@ else: sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'vendor')) -from cps import create_app, config +from cps import create_app from cps import web_server from cps.opds import opds from cps.web import web diff --git a/cps/helper.py b/cps/helper.py index 506afe71..50e24636 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -32,7 +32,7 @@ from tempfile import gettempdir import requests from babel.dates import format_datetime from babel.units import format_unit -from flask import send_from_directory, make_response, redirect, abort, url_for, send_file +from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ from flask_login import current_user from sqlalchemy.sql.expression import true, false, and_, text diff --git a/cps/kobo.py b/cps/kobo.py index 3210a450..39032f3b 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -43,7 +43,6 @@ from flask_login import current_user from werkzeug.datastructures import Headers from sqlalchemy import func from sqlalchemy.sql.expression import and_, or_ -from sqlalchemy.exc import OperationalError from sqlalchemy.orm import load_only from sqlalchemy.exc import StatementError import requests diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index ba35c78b..1ede03a4 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -66,7 +66,6 @@ from os import urandom from flask import g, Blueprint, url_for, abort, request from flask_login import login_user, login_required from flask_babel import gettext as _ -from sqlalchemy.exc import OperationalError from . import logger, ub, lm from .render_template import render_title_template diff --git a/cps/oauth_bb.py b/cps/oauth_bb.py index b6474758..b17d38c7 100644 --- a/cps/oauth_bb.py +++ b/cps/oauth_bb.py @@ -32,7 +32,6 @@ from flask_dance.contrib.github import make_github_blueprint, github from flask_dance.contrib.google import make_google_blueprint, google from flask_login import login_user, current_user, login_required from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy.exc import OperationalError from . import constants, logger, config, app, ub diff --git a/cps/web.py b/cps/web.py index 82378de8..c85c0175 100644 --- a/cps/web.py +++ b/cps/web.py @@ -47,7 +47,7 @@ from werkzeug.security import generate_password_hash, check_password_hash from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app -from . import calibre_db, shelf +from . import calibre_db from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import check_valid_domain, render_task_status, \ get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ From 4a9b01e93bab675756c78c699bfc0c88c6f32407 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 10 Jan 2021 10:23:14 +0100 Subject: [PATCH 04/22] Added IntegrityError to catched error --- cps/editbooks.py | 6 +++--- cps/web.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index b20ec5c8..2c045500 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -30,7 +30,7 @@ from uuid import uuid4 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response from flask_babel import gettext as _ from flask_login import current_user, login_required -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, IntegrityError from . import constants, logger, isoLanguages, gdriveutils, uploader, helper from . import config, get_locale, ub, db @@ -570,7 +570,7 @@ def upload_single_file(request, book, book_id): calibre_db.session.add(db_format) calibre_db.session.commit() calibre_db.update_title_sort(config) - except OperationalError as e: + except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() log.error('Database error: %s', e) flash(_(u"Database error: %(error)s.", error=e), category="error") @@ -925,7 +925,7 @@ def upload(): else: resp = {"location": url_for('web.show_book', book_id=book_id)} return Response(json.dumps(resp), mimetype='application/json') - except OperationalError as e: + except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() log.error("Database error: %s", e) flash(_(u"Database error: %(error)s.", error=e), category="error") diff --git a/cps/web.py b/cps/web.py index c85c0175..94cd8708 100644 --- a/cps/web.py +++ b/cps/web.py @@ -184,7 +184,7 @@ def toggle_read(book_id): calibre_db.session.commit() except (KeyError, AttributeError): log.error(u"Custom Column No.%d is not exisiting in calibre database", config.config_read_column) - except OperationalError as e: + except (OperationalError, OperationalError) as e: calibre_db.session.rollback() log.error(u"Read status could not set: %e", e) From 6f9e52792ac66f129833afc102ece8f16d87d181 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 10 Jan 2021 11:01:54 +0100 Subject: [PATCH 05/22] No autoflush on metadata.db change --- cps/db.py | 16 +++++++++++----- cps/gdriveutils.py | 3 ++- cps/tasks/convert.py | 1 + 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cps/db.py b/cps/db.py index f30fb609..dbb90ff1 100644 --- a/cps/db.py +++ b/cps/db.py @@ -402,7 +402,10 @@ class AlchemyEncoder(json.JSONEncoder): el.append(ele.get()) else: el.append(json.dumps(ele, cls=AlchemyEncoder)) - data = ",".join(el) + if field == 'authors': + data = " & ".join(el) + else: + data = ",".join(el) if data == '[]': data = "" else: @@ -542,7 +545,7 @@ class CalibreDB(): backref='books')) cls.session_factory = scoped_session(sessionmaker(autocommit=False, - autoflush=True, + autoflush=False, bind=cls.engine)) for inst in cls.instances: inst.initSession() @@ -619,9 +622,12 @@ class CalibreDB(): .join(*join, isouter=True) \ .filter(db_filter) \ .filter(self.common_filters(allow_show_archived)) - pagination = Pagination(page, pagesize, - len(query.all())) - entries = query.order_by(*order).offset(off).limit(pagesize).all() + try: + pagination = Pagination(page, pagesize, + len(query.all())) + entries = query.order_by(*order).offset(off).limit(pagesize).all() + except Exception as e: + logger.debug(e) for book in entries: book = self.order_authors(book) return entries, randm, pagination diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 6f78ab47..79587b79 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -394,7 +394,8 @@ def uploadFileToEbooksFolder(destFile, f): if len(existingFiles) > 0: driveFile = existingFiles[0] else: - driveFile = drive.CreateFile({'title': x, 'parents': [{"kind": "drive#fileLink", 'id': parent['id']}],}) + driveFile = drive.CreateFile({'title': x, + 'parents': [{"kind": "drive#fileLink", 'id': parent['id']}], }) driveFile.SetContentFile(f) driveFile.Upload() else: diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index d3e74569..93f964f9 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -123,6 +123,7 @@ class TaskConvert(CalibreTask): book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) try: local_db.session.merge(new_format) + local_db.session.flush() local_db.session.commit() except SQLAlchemyError as e: local_db.session.rollback() From 263a8f90482483f8cea5ac7f744d819a4c242a7b Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 10 Jan 2021 14:57:54 +0100 Subject: [PATCH 06/22] Added catching of missing invalid_request --- cps/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/web.py b/cps/web.py index 94cd8708..45e03224 100644 --- a/cps/web.py +++ b/cps/web.py @@ -184,7 +184,7 @@ def toggle_read(book_id): calibre_db.session.commit() except (KeyError, AttributeError): log.error(u"Custom Column No.%d is not exisiting in calibre database", config.config_read_column) - except (OperationalError, OperationalError) as e: + except (OperationalError, InvalidRequestError) as e: calibre_db.session.rollback() log.error(u"Read status could not set: %e", e) From e5f754ed0ee96857bcdcb6d30d9ea93c16e23d5c Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 10 Jan 2021 15:02:04 +0100 Subject: [PATCH 07/22] improved session handling --- cps/db.py | 2 +- cps/editbooks.py | 14 +- cps/tasks/convert.py | 1 - test/Calibre-Web TestSummary_Linux.html | 596 +++++++++++++++--------- 4 files changed, 392 insertions(+), 221 deletions(-) diff --git a/cps/db.py b/cps/db.py index dbb90ff1..ea865f80 100644 --- a/cps/db.py +++ b/cps/db.py @@ -545,7 +545,7 @@ class CalibreDB(): backref='books')) cls.session_factory = scoped_session(sessionmaker(autocommit=False, - autoflush=False, + autoflush=True, bind=cls.engine)) for inst in cls.instances: inst.initSession() diff --git a/cps/editbooks.py b/cps/editbooks.py index 2c045500..1b1e13a2 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -31,7 +31,7 @@ from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, R from flask_babel import gettext as _ from flask_login import current_user, login_required from sqlalchemy.exc import OperationalError, IntegrityError - +from sqlite3 import OperationalError as sqliteOperationalError from . import constants, logger, isoLanguages, gdriveutils, uploader, helper from . import config, get_locale, ub, db from . import calibre_db @@ -310,7 +310,6 @@ def delete_book(book_id, book_format, jsonResponse): def render_edit_book(book_id): - calibre_db.update_title_sort(config) cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) if not book: @@ -607,12 +606,19 @@ def upload_cover(request, book): @edit_required def edit_book(book_id): modif_date = False + + # create the function for sorting... + try: + calibre_db.update_title_sort(config) + except sqliteOperationalError as e: + log.debug_or_exception(e) + calibre_db.session.rollback() + # Show form if request.method != 'POST': return render_edit_book(book_id) - # create the function for sorting... - calibre_db.update_title_sort(config) + book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) # Book not found diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 93f964f9..d3e74569 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -123,7 +123,6 @@ class TaskConvert(CalibreTask): book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) try: local_db.session.merge(new_format) - local_db.session.flush() local_db.session.commit() except SQLAlchemyError as e: local_db.session.rollback() diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 027a3e46..5949373d 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2021-01-02 07:53:24

+

Start Time: 2021-01-10 18:12:04

-

Stop Time: 2021-01-02 10:58:22

+

Stop Time: 2021-01-10 20:39:46

-

Duration: 2h 37 min

+

Duration: 1h 59 min

@@ -611,15 +611,15 @@ - + TestEbookConvertGDriveKepubify + 5 + 1 + 1 3 - 3 - 0 - 0 0 - Detail + Detail @@ -634,20 +634,126 @@ - +
TestEbookConvertGDriveKepubify - test_convert_only
- PASS + +
+ FAIL +
+ + + + - + + +
TestEbookConvertGDriveKepubify - test_convert_only
+ + +
+ ERROR +
+ + + + + + + + +
TestEbookConvertGDriveKepubify - test_convert_wrong_excecutable
- PASS + +
+ ERROR +
+ + + + + + + + + + +
TestEbookConvertGDriveKepubify - test_convert_wrong_excecutable
+ + +
+ ERROR +
+ + + + @@ -1190,11 +1296,11 @@ - + TestEditBooksOnGdrive 20 - 19 - 1 + 20 + 0 0 0 @@ -1339,33 +1445,11 @@ - +
TestEditBooksOnGdrive - test_edit_title
- -
- FAIL -
- - - - + PASS @@ -1509,13 +1593,13 @@ AssertionError: 'No Cover' != 'The camicdemo' - + TestSetupGdrive 1 + 0 + 0 1 0 - 0 - 0 Detail @@ -1523,11 +1607,39 @@ AssertionError: 'No Cover' != 'The camicdemo' - +
TestSetupGdrive - test_config_gdrive
- PASS + +
+ ERROR +
+ + + + @@ -1803,6 +1915,60 @@ AssertionError: 'No Cover' != 'The camicdemo' + + _ErrorHolder + 1 + 0 + 0 + 1 + 0 + + Detail + + + + + + + +
tearDownClass (test_ldap)
+ + +
+ ERROR +
+ + + + + + + + + TestLogging 7 @@ -1811,13 +1977,13 @@ AssertionError: 'No Cover' != 'The camicdemo' 0 1 - Detail + Detail - +
TestLogging - test_access_log_recover
@@ -1826,7 +1992,7 @@ AssertionError: 'No Cover' != 'The camicdemo' - +
TestLogging - test_debug_log
@@ -1835,7 +2001,7 @@ AssertionError: 'No Cover' != 'The camicdemo' - +
TestLogging - test_failed_login
@@ -1844,19 +2010,19 @@ AssertionError: 'No Cover' != 'The camicdemo' - +
TestLogging - test_failed_register
- SKIP + SKIP
-