From e8e4d87d398a4aa2c366fce48bbb399de21ae7e9 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 30 Apr 2022 08:26:00 +0200 Subject: [PATCH] Bugfix session commit Bugfix get_locale Bugfix reconnect database schedule Bugfix no permission error message logging Bugfix updater --- cps/admin.py | 3 +- cps/db.py | 5 +- cps/fs.py | 12 +- cps/helper.py | 3 +- cps/schedule.py | 2 +- cps/tasks/convert.py | 6 +- cps/tasks/thumbnail.py | 39 +- cps/templates/schedule_edit.html | 2 +- cps/updater.py | 18 +- test/Calibre-Web TestSummary_Linux.html | 2596 +++++------------------ 10 files changed, 585 insertions(+), 2101 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 91a16126..76275922 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -185,7 +185,6 @@ def update_thumbnails(): @login_required @admin_required def admin(): - locale = get_locale() version = updater_thread.get_current_version_info() if version is False: commit = _(u'Unknown') @@ -1460,7 +1459,7 @@ def download_debug(): def get_update_status(): if feature_support['updater']: log.info(u"Update status requested") - return updater_thread.get_available_updates(request.method, locale=get_locale()) + return updater_thread.get_available_updates(request.method) else: return '' diff --git a/cps/db.py b/cps/db.py index 21104622..f4dc4ccb 100644 --- a/cps/db.py +++ b/cps/db.py @@ -441,10 +441,13 @@ class CalibreDB: # instances alive once they reach the end of their respective scopes instances = WeakSet() - def __init__(self): + def __init__(self, expire_on_commit=True, init=False): """ Initialize a new CalibreDB session """ self.session = None + if init: + self.init_db(expire_on_commit) + def init_db(self, expire_on_commit=True): if self._init: diff --git a/cps/fs.py b/cps/fs.py index 2f83378d..996499c3 100644 --- a/cps/fs.py +++ b/cps/fs.py @@ -39,7 +39,7 @@ class FileSystem: makedirs(self._cache_dir) except OSError: self.log.info(f'Failed to create path {self._cache_dir} (Permission denied).') - return False + raise path = join(self._cache_dir, cache_type) if cache_type and not isdir(path): @@ -47,7 +47,7 @@ class FileSystem: makedirs(path) except OSError: self.log.info(f'Failed to create path {path} (Permission denied).') - return False + raise return path if cache_type else self._cache_dir @@ -58,7 +58,7 @@ class FileSystem: makedirs(path) except OSError: self.log.info(f'Failed to create path {path} (Permission denied).') - return False + raise return path @@ -75,7 +75,7 @@ class FileSystem: rmtree(self._cache_dir) except OSError: self.log.info(f'Failed to delete path {self._cache_dir} (Permission denied).') - return False + raise path = join(self._cache_dir, cache_type) if cache_type and isdir(path): @@ -83,7 +83,7 @@ class FileSystem: rmtree(path) except OSError: self.log.info(f'Failed to delete path {path} (Permission denied).') - return False + raise def delete_cache_file(self, filename, cache_type=None): path = self.get_cache_file_path(filename, cache_type) @@ -92,4 +92,4 @@ class FileSystem: remove(path) except OSError: self.log.info(f'Failed to delete path {path} (Permission denied).') - return False + raise diff --git a/cps/helper.py b/cps/helper.py index 7362eb9e..757e48da 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -1015,8 +1015,7 @@ def replace_cover_thumbnail_cache(book_id): def delete_thumbnail_cache(): - if config.schedule_generate_book_covers: - WorkerThread.add(None, TaskClearCoverThumbnailCache(-1)) + WorkerThread.add(None, TaskClearCoverThumbnailCache(-1)) def add_book_to_thumbnail_cache(book_id): diff --git a/cps/schedule.py b/cps/schedule.py index 9bc248f1..faadfb7e 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -62,7 +62,7 @@ def register_scheduled_tasks(reconnect=True): duration = config.schedule_duration # Register scheduled tasks - scheduler.schedule_tasks(tasks=get_scheduled_tasks(), trigger='cron', hour=start) + scheduler.schedule_tasks(tasks=get_scheduled_tasks(reconnect), trigger='cron', hour=start) end_time = calclulate_end_time(start, duration) scheduler.schedule(func=end_scheduled_tasks, trigger='cron', name="end scheduled task", hour=end_time.hour, minute=end_time.minute) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index bc7e928a..3062850d 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -55,8 +55,7 @@ class TaskConvert(CalibreTask): def run(self, worker_thread): self.worker_thread = worker_thread if config.config_use_google_drive: - worker_db = db.CalibreDB() - worker_db.init_db(expire_on_commit=False) + worker_db = db.CalibreDB(expire_on_commit=False, init=True) cur_book = worker_db.get_book(self.book_id) self.title = cur_book.title data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) @@ -105,8 +104,7 @@ class TaskConvert(CalibreTask): def _convert_ebook_format(self): error_message = None - local_db = db.CalibreDB() - local_db.init_db(expire_on_commit=False) + local_db = db.CalibreDB(expire_on_commit=False, init=True) file_path = self.file_path book_id = self.book_id format_old_ext = u'.' + self.settings['old_book_format'].lower() diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 2ecc57c8..ace9cecc 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -68,8 +68,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.log = logger.create() self.book_id = book_id self.app_db_session = ub.get_new_session_instance() - self.calibre_db = db.CalibreDB() - self.calibre_db.init_db(expire_on_commit=False) + # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) self.cache = fs.FileSystem() self.resolutions = [ constants.COVER_THUMBNAIL_SMALL, @@ -77,7 +76,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): ] def run(self, worker_thread): - if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: + if use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: self.message = 'Scanning Books' books_with_covers = self.get_books_with_covers(self.book_id) count = len(books_with_covers) @@ -112,11 +111,10 @@ class TaskGenerateCoverThumbnails(CalibreTask): def get_books_with_covers(self, book_id=-1): filter_exp = (db.Books.id == book_id) if book_id != -1 else True - return self.calibre_db.session \ - .query(db.Books) \ - .filter(db.Books.has_cover == 1) \ - .filter(filter_exp) \ - .all() + calibre_db = db.CalibreDB(expire_on_commit=False, init=True) + books_cover = calibre_db.session.query(db.Books).filter(db.Books.has_cover == 1).filter(filter_exp).all() + calibre_db.session.close() + return books_cover def get_book_cover_thumbnails(self, book_id): return self.app_db_session \ @@ -160,7 +158,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.app_db_session.commit() self.generate_book_thumbnail(book, thumbnail) except Exception as ex: - self.log.info('Error creating book thumbnail: ' + str(ex)) + self.log.debug('Error creating book thumbnail: ' + str(ex)) self._handleError('Error creating book thumbnail: ' + str(ex)) self.app_db_session.rollback() @@ -172,7 +170,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) self.generate_book_thumbnail(book, thumbnail) except Exception as ex: - self.log.info('Error updating book thumbnail: ' + str(ex)) + self.log.debug('Error updating book thumbnail: ' + str(ex)) self._handleError('Error updating book thumbnail: ' + str(ex)) self.app_db_session.rollback() @@ -200,7 +198,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): img.save(filename=filename) except Exception as ex: # Bubble exception to calling function - self.log.info('Error generating thumbnail file: ' + str(ex)) + self.log.debug('Error generating thumbnail file: ' + str(ex)) raise ex finally: if stream is not None: @@ -239,8 +237,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask): super(TaskGenerateSeriesThumbnails, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() - self.calibre_db = db.CalibreDB() - self.calibre_db.init_db(expire_on_commit=False) + self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) self.cache = fs.FileSystem() self.resolutions = [ constants.COVER_THUMBNAIL_SMALL, @@ -337,7 +334,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask): self.app_db_session.commit() self.generate_series_thumbnail(series_books, thumbnail) except Exception as ex: - self.log.info('Error creating book thumbnail: ' + str(ex)) + self.log.debug('Error creating book thumbnail: ' + str(ex)) self._handleError('Error creating book thumbnail: ' + str(ex)) self.app_db_session.rollback() @@ -349,7 +346,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask): self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) self.generate_series_thumbnail(series_books, thumbnail) except Exception as ex: - self.log.info('Error updating book thumbnail: ' + str(ex)) + self.log.debug('Error updating book thumbnail: ' + str(ex)) self._handleError('Error updating book thumbnail: ' + str(ex)) self.app_db_session.rollback() @@ -393,7 +390,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask): canvas.composite(img, left, top) except Exception as ex: - self.log.info('Error generating thumbnail file: ' + str(ex)) + self.log.debug('Error generating thumbnail file: ' + str(ex)) raise ex finally: if stream is not None: @@ -450,18 +447,18 @@ class TaskClearCoverThumbnailCache(CalibreTask): super(TaskClearCoverThumbnailCache, self).__init__(task_message) self.log = logger.create() self.book_id = book_id - self.calibre_db = db.CalibreDB() - self.calibre_db.init_db(expire_on_commit=False) self.app_db_session = ub.get_new_session_instance() self.cache = fs.FileSystem() def run(self, worker_thread): if self.app_db_session: if self.book_id == 0: # delete superfluous thumbnails - thumbnails = (self.calibre_db.session.query(ub.Thumbnail) + calibre_db = db.CalibreDB(expire_on_commit=False, init=True) + thumbnails = (calibre_db.session.query(ub.Thumbnail) .join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True) .filter(db.Books.id == None) .all()) + calibre_db.session.close() elif self.book_id > 0: # make sure single book is selected thumbnails = self.get_thumbnails_for_book(self.book_id) if self.book_id < 0: @@ -489,7 +486,7 @@ class TaskClearCoverThumbnailCache(CalibreTask): .delete() self.app_db_session.commit() except Exception as ex: - self.log.info('Error deleting book thumbnail: ' + str(ex)) + self.log.debug('Error deleting book thumbnail: ' + str(ex)) self._handleError('Error deleting book thumbnail: ' + str(ex)) def delete_all_thumbnails(self): @@ -498,7 +495,7 @@ class TaskClearCoverThumbnailCache(CalibreTask): self.app_db_session.commit() self.cache.delete_cache_dir(constants.CACHE_TYPE_THUMBNAILS) except Exception as ex: - self.log.info('Error deleting thumbnail directory: ' + str(ex)) + self.log.debug('Error deleting thumbnail directory: ' + str(ex)) self._handleError('Error deleting thumbnail directory: ' + str(ex)) @property diff --git a/cps/templates/schedule_edit.html b/cps/templates/schedule_edit.html index 55658ce7..83dd6c68 100644 --- a/cps/templates/schedule_edit.html +++ b/cps/templates/schedule_edit.html @@ -33,7 +33,7 @@
- +
diff --git a/cps/updater.py b/cps/updater.py index b47b80f5..757e43fc 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -67,10 +67,10 @@ class Updater(threading.Thread): return self._stable_version_info() return self._nightly_version_info() - def get_available_updates(self, request_method, locale): + def get_available_updates(self, request_method): if self.config.config_updatechannel == constants.UPDATE_STABLE: return self._stable_available_updates(request_method) - return self._nightly_available_updates(request_method, locale) + return self._nightly_available_updates(request_method) def do_work(self): try: @@ -335,7 +335,7 @@ class Updater(threading.Thread): print("\n*** Finished ***") @staticmethod - def _populate_parent_commits(update_data, status, locale, tz, parents): + def _populate_parent_commits(update_data, status, tz, parents): try: parent_commit = update_data['parents'][0] # limit the maximum search depth @@ -360,7 +360,7 @@ class Updater(threading.Thread): parent_commit_date = datetime.datetime.strptime( parent_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz parent_commit_date = format_datetime( - parent_commit_date, format='short', locale=locale) + parent_commit_date, format='short') parents.append([parent_commit_date, parent_data['message'].replace('\r\n', '

').replace('\n', '

')]) @@ -418,7 +418,7 @@ class Updater(threading.Thread): log_function("Excluded file list for updater not found, or not accessible") return excluded_files - def _nightly_available_updates(self, request_method, locale): + def _nightly_available_updates(self, request_method): tz = datetime.timedelta(seconds=time.timezone if (time.localtime().tm_isdst == 0) else time.altzone) if request_method == "GET": repository_url = _REPOSITORY_API_URL @@ -459,14 +459,14 @@ class Updater(threading.Thread): update_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz parents.append( [ - format_datetime(new_commit_date, format='short', locale=locale), + format_datetime(new_commit_date, format='short'), update_data['message'], update_data['sha'] ] ) # it only makes sense to analyze the parents if we know the current commit hash if status['current_commit_hash'] != '': - parents = self._populate_parent_commits(update_data, status, locale, tz, parents) + parents = self._populate_parent_commits(update_data, status, tz, parents) status['history'] = parents[::-1] except (IndexError, KeyError): status['success'] = False @@ -595,7 +595,7 @@ class Updater(threading.Thread): return json.dumps(status) def _get_request_path(self): - if config.config_updatechannel == constants.UPDATE_STABLE: + if self.config.config_updatechannel == constants.UPDATE_STABLE: return self.updateFile return _REPOSITORY_API_URL + '/zipball/master' @@ -623,7 +623,7 @@ class Updater(threading.Thread): status['message'] = _(u'HTTP Error') + ': ' + commit['message'] else: status['message'] = _(u'HTTP Error') + ': ' + str(e) - except requests.exceptions.ConnectionError: + except requests.exceptions.ConnectionError as e: status['message'] = _(u'Connection error') except requests.exceptions.Timeout: status['message'] = _(u'Timeout while establishing connection') diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index cf3e893d..913913b1 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@

-

Start Time: 2022-04-27 19:55:57

+

Start Time: 2022-04-29 21:17:05

-

Stop Time: 2022-04-28 01:28:47

+

Stop Time: 2022-04-30 03:15:20

-

Duration: 4h 34 min

+

Duration: 5h 5 min

@@ -600,8 +600,8 @@ TestEbookConvertCalibreGDrive 7 - 6 - 1 + 5 + 2 0 0 @@ -611,11 +611,31 @@ - +
TestEbookConvertCalibreGDrive - test_convert_email
- PASS + +
+ FAIL +
+ + + + @@ -682,9 +702,9 @@ 
Traceback (most recent call last):
-  File "/home/ozzie/Development/calibre-web-test/test/test_ebook_convert_gdrive.py", line 441, in test_thumbnail_cache
-    self.assertTrue(os.path.exists(thumbnail_cache_path))
-AssertionError: False is not true
+ File "/home/ozzie/Development/calibre-web-test/test/test_ebook_convert_gdrive.py", line 442, in test_thumbnail_cache + self.assertEqual(10*2, count_files(thumbnail_cache_path)) +AssertionError: 20 != 0
@@ -1642,11 +1662,11 @@ AssertionError: False is not true - + TestEditBooksOnGdrive 18 - 18 - 0 + 17 + 1 0 0 @@ -1809,11 +1829,31 @@ AssertionError: False is not true - +
TestEditBooksOnGdrive - test_watch_metadata
- PASS + +
+ FAIL +
+ + + + @@ -2145,12 +2185,12 @@ AssertionError: False is not true - + TestKoboSync 11 + 11 + 0 0 - 1 - 10 0 Detail @@ -2159,1610 +2199,190 @@ AssertionError: False is not true - +
TestKoboSync - test_book_download
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_kobo_about
- -
- FAIL -
- - - - + PASS - +
TestKoboSync - test_kobo_sync_selected_shelfs
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_kobo_upload_book
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_shelves_add_remove_books
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_sync_changed_book
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_sync_invalid
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_sync_reading_state
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_sync_shelf
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_sync_unchanged
- -
- ERROR -
- - - - + PASS - +
TestKoboSync - test_sync_upload
- -
- ERROR -
- - - - + PASS - - _ErrorHolder - 5 - 0 - 0 - 5 - 0 - - Detail - - - - - - - -
tearDownClass (test_kobo_sync)
- - -
- ERROR -
- - - - - - - - - - -
tearDownClass (test_kobo_sync_big)
- - -
- ERROR -
- - - - - - - - - - -
tearDownClass (test_reverse_proxy)
- - -
- ERROR -
- - - - - - - - - - -
setUpClass (test_updater)
- - -
- ERROR -
- - - - - - - - - - -
setUpClass (test_zz_helper)
- - -
- ERROR -
- - - - - - - - - - + TestKoboSyncBig 6 - 0 - 0 6 0 + 0 + 0 - Detail + Detail - +
TestKoboSyncBig - test_download_cover
- -
- ERROR -
- - - - + PASS - +
TestKoboSyncBig - test_kobo_sync_multi_user
- -
- ERROR -
- - - - + PASS - +
TestKoboSyncBig - test_kobo_sync_selected_shelves
- -
- ERROR -
- - - - + PASS - +
TestKoboSyncBig - test_sync_changed_book
- -
- ERROR -
- - - - + PASS - +
TestKoboSyncBig - test_sync_reading_state
- -
- ERROR -
- - - - + PASS - +
TestKoboSyncBig - test_sync_shelf
- -
- ERROR -
- - - - + PASS - + TestLdapLogin 13 - 10 + 13 + 0 0 - 3 0 - Detail + Detail - +
TestLdapLogin - test_LDAP_SSL
@@ -3771,7 +2391,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_LDAP_SSL_CERTIFICATE
@@ -3780,7 +2400,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_LDAP_STARTTLS
@@ -3789,7 +2409,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_LDAP_fallback_Login
@@ -3798,7 +2418,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_LDAP_import
@@ -3807,7 +2427,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_LDAP_import_memberfield
@@ -3816,7 +2436,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_LDAP_login
@@ -3825,7 +2445,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_invalid_LDAP
@@ -3834,7 +2454,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_ldap_about
@@ -3843,7 +2463,7 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_ldap_authentication
@@ -3852,109 +2472,29 @@ TypeError: unsupported operand type(s) for +: 'NoneType' and 'str - +
TestLdapLogin - test_ldap_kobo_sync
- -
- ERROR -
- - - - + PASS - +
TestLdapLogin - test_ldap_opds_anonymous
- -
- ERROR -
- - - - + PASS - +
TestLdapLogin - test_ldap_opds_download_book
- -
- ERROR -
- - - - + PASS @@ -3968,13 +2508,13 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:160 0 - Detail + Detail - +
TestCalibreWebListOrders - test_author_sort
@@ -3983,7 +2523,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_download_sort
@@ -3992,7 +2532,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_format_sort
@@ -4001,7 +2541,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_lang_sort
@@ -4010,7 +2550,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_order_authors_all_links
@@ -4019,7 +2559,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_order_series_all_links
@@ -4028,7 +2568,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_publisher_sort
@@ -4037,7 +2577,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_ratings_sort
@@ -4046,7 +2586,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_series_sort
@@ -4055,7 +2595,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestCalibreWebListOrders - test_tags_sort
@@ -4065,21 +2605,21 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 + TestLogging 9 - 7 - 1 + 8 + 0 0 1 - Detail + Detail - +
TestLogging - test_access_log_recover
@@ -4088,7 +2628,7 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestLogging - test_debug_log
@@ -4097,38 +2637,16 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestLogging - test_debuginfo_download
- -
- FAIL -
- - - - + PASS - +
TestLogging - test_failed_login
@@ -4137,19 +2655,19 @@ AssertionError: 'text/html; charset=utf-8' != 'application/zip' - +
TestLogging - test_failed_register
- SKIP + SKIP
-