From b75497231e5818ae45d664c17e760a49ec7b8d0d Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Thu, 21 May 2020 09:28:35 +0200 Subject: [PATCH] Additional fix for #1407 (metadata.db is now held in memory, app.db is attached to it -> joins between both databases possible -> book_read_link is joined for getting result) --- cps/__init__.py | 2 +- cps/admin.py | 4 ++-- cps/db.py | 15 ++++++++++----- cps/gdrive.py | 7 ++++--- cps/kobo.py | 2 +- cps/ub.py | 5 +++++ cps/web.py | 25 ++++++++++++++----------- 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index 99682737..5effb3d6 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -98,7 +98,7 @@ def create_app(): app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) web_server.init_app(app, config) - db.setup_db(config) + db.setup_db(config, cli.settingspath) babel.init_app(app) _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) diff --git a/cps/admin.py b/cps/admin.py index 6163d1b1..49bf309c 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -99,7 +99,7 @@ def shutdown(): if task == 2: log.warning("reconnecting to calibre database") - db.setup_db(config) + db.setup_db(config, ub.app_DB_path) showtext['text'] = _(u'Reconnect successful') return json.dumps(showtext) @@ -688,7 +688,7 @@ def _configuration_update_helper(): return _configuration_result('%s' % e, gdriveError) if db_change: - if not db.setup_db(config): + if not db.setup_db(config, ub.app_DB_path): return _configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdriveError) if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK): flash(_(u"DB is not Writeable"), category="warning") diff --git a/cps/db.py b/cps/db.py index 882c792e..e58b2534 100755 --- a/cps/db.py +++ b/cps/db.py @@ -24,7 +24,7 @@ import re import ast from datetime import datetime -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy import Table, Column, ForeignKey, CheckConstraint from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float from sqlalchemy.orm import relationship, sessionmaker, scoped_session @@ -329,7 +329,7 @@ def update_title_sort(config, conn=None): conn.create_function("title_sort", 1, _title_sort) -def setup_db(config): +def setup_db(config, app_db_path): dispose() global engine @@ -343,10 +343,14 @@ def setup_db(config): return False try: - engine = create_engine('sqlite:///{0}'.format(dbpath), + #engine = create_engine('sqlite:///{0}'.format(dbpath), + engine = create_engine('sqlite://', echo=False, isolation_level="SERIALIZABLE", connect_args={'check_same_thread': False}) + engine.execute("attach database '{}' as calibre;".format(dbpath)) + engine.execute("attach database '{}' as app_settings;".format(app_db_path)) + conn = engine.connect() # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 except Exception as e: @@ -438,7 +442,8 @@ def dispose(): if table is not None: Base.metadata.remove(table) -def reconnect_db(config): + +def reconnect_db(config, app_db_path): session.close() engine.dispose() - setup_db(config) + setup_db(config, app_db_path) diff --git a/cps/gdrive.py b/cps/gdrive.py index 23a36a91..7610795f 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -39,7 +39,7 @@ try: except ImportError: pass -from . import logger, gdriveutils, config, db +from . import logger, gdriveutils, config, db, ub from .web import admin_required @@ -145,7 +145,8 @@ def on_received_watch_confirmation(): dbpath = os.path.join(config.config_calibre_dir, "metadata.db") else: dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() - if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != hashlib.md5(dbpath): + if not response['deleted'] and response['file']['title'] == 'metadata.db' \ + and response['file']['md5Checksum'] != hashlib.md5(dbpath): tmpDir = tempfile.gettempdir() log.info('Database file updated') copyfile(dbpath, os.path.join(tmpDir, "metadata.db_" + str(current_milli_time()))) @@ -154,7 +155,7 @@ def on_received_watch_confirmation(): log.info('Setting up new DB') # prevent error on windows, as os.rename does on exisiting files move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath) - db.setup_db(config) + db.setup_db(config, ub.app_DB_path) except Exception as e: log.exception(e) updateMetaData() diff --git a/cps/kobo.py b/cps/kobo.py index e8c14088..58f43f06 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -143,7 +143,7 @@ def HandleSyncRequest(): # We reload the book database so that the user get's a fresh view of the library # in case of external changes (e.g: adding a book through Calibre). - db.reconnect_db(config) + db.reconnect_db(config, ub.app_DB_path) archived_books = ( ub.session.query(ub.ArchivedBook) diff --git a/cps/ub.py b/cps/ub.py index 57aee968..3dfb5347 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -49,6 +49,7 @@ from . import constants session = None +app_DB_path = None Base = declarative_base() @@ -664,8 +665,12 @@ def create_admin_user(session): def init_db(app_db_path): # Open session for database connection global session + global app_DB_path + app_DB_path = app_db_path engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False) + # engine.execute("attach database '{0}' as app_settings;".format(app_db_path)) + Session = sessionmaker() Session.configure(bind=engine) diff --git a/cps/web.py b/cps/web.py index ea93998f..887b3620 100644 --- a/cps/web.py +++ b/cps/web.py @@ -970,7 +970,7 @@ def get_tasks_status(): @app.route("/reconnect") def reconnect(): - db.reconnect_db(config) + db.reconnect_db(config, ub.app_DB_path) return json.dumps({}) @@ -1154,30 +1154,33 @@ def advanced_search(): def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs): order = order or [] if not config.config_read_column: - readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\ - .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all() - readBookIds = [x.book_id for x in readBooks] if are_read: - db_filter = db.Books.id.in_(readBookIds) + db_filter = and_(ub.ReadBook.user_id == int(current_user.id), + ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED) else: - db_filter = ~db.Books.id.in_(readBookIds) - entries, random, pagination = fill_indexpage(page, db.Books, db_filter, order) + db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED + entries, random, pagination = fill_indexpage(page, db.Books, + db_filter, + order, + ub.ReadBook, db.Books.id==ub.ReadBook.book_id) else: try: if are_read: db_filter = db.cc_classes[config.config_read_column].value == True else: db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True - # book_count = db.session.query(func.count(db.Books.id)).filter(common_filters()).filter(db_filter).scalar() entries, random, pagination = fill_indexpage(page, db.Books, db_filter, order, db.cc_classes[config.config_read_column]) except KeyError: log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column) - book_count = 0 - - + if not as_xml: + flash(_("Custom Column No.%(column)d is not existing in calibre database", + column=config.config_read_column), + category="error") + return redirect(url_for("web.index")) + # ToDo: Handle error Case for opds if as_xml: return entries, pagination else: