NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'worker/db' into Develop

Browse Source 
# Conflicts:
#	cps/tasks/convert.py
This commit is contained in:
OzzieIsaacs 2020-10-06 21:40:47 +02:00
commit 3b8e5ddfb3
4 changed files with 880 additions and 457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cps/__init__.py
View File

@ -83,6 +83,8 @@ log = logger.create()
from . import services from . import services
db.CalibreDB.setup_db(config, cli.settingspath)
calibre_db = db.CalibreDB() calibre_db = db.CalibreDB()
def create_app(): def create_app():
@ -91,7 +93,7 @@ def create_app():
if sys.version_info < (3, 0): if sys.version_info < (3, 0):
app.static_folder = app.static_folder.decode('utf-8') app.static_folder = app.static_folder.decode('utf-8')
app.root_path = app.root_path.decode('utf-8') app.root_path = app.root_path.decode('utf-8')
app.instance_path = app.instance_path .decode('utf-8') app.instance_path = app.instance_path.decode('utf-8')
cache_buster.init_cache_busting(app) cache_buster.init_cache_busting(app)
@ -101,7 +103,6 @@ def create_app():
app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
web_server.init_app(app, config) web_server.init_app(app, config)
calibre_db.setup_db(config, cli.settingspath)
babel.init_app(app) babel.init_app(app)
_BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

160
cps/db.py
View File

@ -42,48 +42,49 @@ from flask_babel import gettext as _
from . import logger, ub, isoLanguages from . import logger, ub, isoLanguages
from .pagination import Pagination from .pagination import Pagination
from weakref import WeakSet
try: try:
import unidecode import unidecode
use_unidecode = True use_unidecode = True
except ImportError: except ImportError:
use_unidecode = False use_unidecode = False
Session = None
cc_exceptions = ['datetime', 'comments', 'composite', 'series'] cc_exceptions = ['datetime', 'comments', 'composite', 'series']
cc_classes = {} cc_classes = {}
Base = declarative_base() Base = declarative_base()
books_authors_link = Table('books_authors_link', Base.metadata, books_authors_link = Table('books_authors_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('author', Integer, ForeignKey('authors.id'), primary_key=True) Column('author', Integer, ForeignKey('authors.id'), primary_key=True)
) )
books_tags_link = Table('books_tags_link', Base.metadata, books_tags_link = Table('books_tags_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) Column('tag', Integer, ForeignKey('tags.id'), primary_key=True)
) )
books_series_link = Table('books_series_link', Base.metadata, books_series_link = Table('books_series_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('series', Integer, ForeignKey('series.id'), primary_key=True) Column('series', Integer, ForeignKey('series.id'), primary_key=True)
) )
books_ratings_link = Table('books_ratings_link', Base.metadata, books_ratings_link = Table('books_ratings_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True)
) )
books_languages_link = Table('books_languages_link', Base.metadata, books_languages_link = Table('books_languages_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True)
) )
books_publishers_link = Table('books_publishers_link', Base.metadata, books_publishers_link = Table('books_publishers_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True)
) )
class Identifiers(Base): class Identifiers(Base):
@ -99,7 +100,7 @@ class Identifiers(Base):
self.type = id_type self.type = id_type
self.book = book self.book = book
#def get(self): # def get(self):
# return {self.type: self.val} # return {self.type: self.val}
def formatType(self): def formatType(self):
@ -290,7 +291,7 @@ class Publishers(Base):
class Data(Base): class Data(Base):
__tablename__ = 'data' __tablename__ = 'data'
__table_args__ = {'schema':'calibre'} __table_args__ = {'schema': 'calibre'}
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
book = Column(Integer, ForeignKey('books.id'), nullable=False) book = Column(Integer, ForeignKey('books.id'), nullable=False)
@ -315,7 +316,7 @@ class Data(Base):
class Books(Base): class Books(Base):
__tablename__ = 'books' __tablename__ = 'books'
DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00") DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00")
id = Column(Integer, primary_key=True, autoincrement=True) id = Column(Integer, primary_key=True, autoincrement=True)
title = Column(String(collation='NOCASE'), nullable=False, default='Unknown') title = Column(String(collation='NOCASE'), nullable=False, default='Unknown')
@ -354,7 +355,7 @@ class Books(Base):
self.path = path self.path = path
self.has_cover = (has_cover != None) self.has_cover = (has_cover != None)
#def as_dict(self): # def as_dict(self):
# return {c.name: getattr(self, c.name) for c in self.__table__.columns} # return {c.name: getattr(self, c.name) for c in self.__table__.columns}
def __repr__(self): def __repr__(self):
@ -366,6 +367,7 @@ class Books(Base):
def atom_timestamp(self): def atom_timestamp(self):
return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '') return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '')
class Custom_Columns(Base): class Custom_Columns(Base):
__tablename__ = 'custom_columns' __tablename__ = 'custom_columns'
@ -385,6 +387,7 @@ class Custom_Columns(Base):
display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']] display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']]
return display_dict return display_dict
class AlchemyEncoder(json.JSONEncoder): class AlchemyEncoder(json.JSONEncoder):
def default(self, obj): def default(self, obj):
@ -397,15 +400,15 @@ class AlchemyEncoder(json.JSONEncoder):
data = obj.__getattribute__(field) data = obj.__getattribute__(field)
try: try:
if isinstance(data, str): if isinstance(data, str):
data = data.replace("'","\'") data = data.replace("'", "\'")
elif isinstance(data, InstrumentedList): elif isinstance(data, InstrumentedList):
el =list() el = list()
for ele in data: for ele in data:
if ele.get: if ele.get:
el.append(ele.get()) el.append(ele.get())
else: else:
el.append(json.dumps(ele, cls=AlchemyEncoder)) el.append(json.dumps(ele, cls=AlchemyEncoder))
data =",".join(el) data = ",".join(el)
if data == '[]': if data == '[]':
data = "" data = ""
else: else:
@ -420,17 +423,32 @@ class AlchemyEncoder(json.JSONEncoder):
class CalibreDB(): class CalibreDB():
_init = False
engine = None
config = None
session_factory = None
# This is a WeakSet so that references here don't keep other CalibreDB
# instances alive once they reach the end of their respective scopes
instances = WeakSet()
def __init__(self): def __init__(self):
self.engine = None """ Initialize a new CalibreDB session
"""
self.session = None self.session = None
self.log = None if self._init:
self.config = None self.initSession()
def setup_db(self, config, app_db_path): self.instances.add(self)
self.config = config
self.dispose()
global Session def initSession(self):
self.session = self.session_factory()
self.update_title_sort(self.config)
@classmethod
def setup_db(cls, config, app_db_path):
cls.config = config
cls.dispose()
if not config.config_calibre_dir: if not config.config_calibre_dir:
config.invalidate() config.invalidate()
@ -442,22 +460,21 @@ class CalibreDB():
return False return False
try: try:
self.engine = create_engine('sqlite://', cls.engine = create_engine('sqlite://',
echo=False, echo=False,
isolation_level="SERIALIZABLE", isolation_level="SERIALIZABLE",
connect_args={'check_same_thread': False}, connect_args={'check_same_thread': False},
poolclass=StaticPool) poolclass=StaticPool)
self.engine.execute("attach database '{}' as calibre;".format(dbpath)) cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
self.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
conn = self.engine.connect() conn = cls.engine.connect()
# conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
except Exception as e: except Exception as e:
config.invalidate(e) config.invalidate(e)
return False return False
config.db_configured = True config.db_configured = True
self.update_title_sort(config, conn.connection)
if not cc_classes: if not cc_classes:
cc = conn.execute("SELECT id, datatype FROM custom_columns") cc = conn.execute("SELECT id, datatype FROM custom_columns")
@ -472,12 +489,12 @@ class CalibreDB():
'book': Column(Integer, ForeignKey('books.id'), 'book': Column(Integer, ForeignKey('books.id'),
primary_key=True), primary_key=True),
'map_value': Column('value', Integer, 'map_value': Column('value', Integer,
ForeignKey('custom_column_' + ForeignKey('custom_column_' +
str(row.id) + '.id'), str(row.id) + '.id'),
primary_key=True), primary_key=True),
'extra': Column(Float), 'extra': Column(Float),
'asoc' : relationship('custom_column_' + str(row.id), uselist=False), 'asoc': relationship('custom_column_' + str(row.id), uselist=False),
'value' : association_proxy('asoc', 'value') 'value': association_proxy('asoc', 'value')
} }
books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'), books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'),
(Base,), dicttable) (Base,), dicttable)
@ -513,7 +530,7 @@ class CalibreDB():
'custom_column_' + str(cc_id[0]), 'custom_column_' + str(cc_id[0]),
relationship(cc_classes[cc_id[0]], relationship(cc_classes[cc_id[0]],
primaryjoin=( primaryjoin=(
Books.id == cc_classes[cc_id[0]].book), Books.id == cc_classes[cc_id[0]].book),
backref='books')) backref='books'))
elif (cc_id[1] == 'series'): elif (cc_id[1] == 'series'):
setattr(Books, setattr(Books,
@ -527,17 +544,20 @@ class CalibreDB():
secondary=books_custom_column_links[cc_id[0]], secondary=books_custom_column_links[cc_id[0]],
backref='books')) backref='books'))
Session = scoped_session(sessionmaker(autocommit=False, cls.session_factory = scoped_session(sessionmaker(autocommit=False,
autoflush=True, autoflush=True,
bind=self.engine)) bind=cls.engine))
self.session = Session() for inst in cls.instances:
inst.initSession()
cls._init = True
return True return True
def get_book(self, book_id): def get_book(self, book_id):
return self.session.query(Books).filter(Books.id == book_id).first() return self.session.query(Books).filter(Books.id == book_id).first()
def get_filtered_book(self, book_id, allow_show_archived=False): def get_filtered_book(self, book_id, allow_show_archived=False):
return self.session.query(Books).filter(Books.id == book_id).\ return self.session.query(Books).filter(Books.id == book_id). \
filter(self.common_filters(allow_show_archived)).first() filter(self.common_filters(allow_show_archived)).first()
def get_book_by_uuid(self, book_uuid): def get_book_by_uuid(self, book_uuid):
@ -587,7 +607,8 @@ class CalibreDB():
def fill_indexpage(self, page, pagesize, database, db_filter, order, *join): def fill_indexpage(self, page, pagesize, database, db_filter, order, *join):
return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join) return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join)
def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, *join): def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived,
*join):
pagesize = pagesize or self.config.config_books_per_page pagesize = pagesize or self.config.config_books_per_page
if current_user.show_detail_random(): if current_user.show_detail_random():
randm = self.session.query(Books) \ randm = self.session.query(Books) \
@ -642,7 +663,7 @@ class CalibreDB():
for authorterm in authorterms: for authorterm in authorterms:
q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
return self.session.query(Books)\ return self.session.query(Books) \
.filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
# read search results from calibre-database and return it (function is used for feed and simple search # read search results from calibre-database and return it (function is used for feed and simple search
@ -702,17 +723,23 @@ class CalibreDB():
conn = conn or self.session.connection().connection.connection conn = conn or self.session.connection().connection.connection
conn.create_function("title_sort", 1, _title_sort) conn.create_function("title_sort", 1, _title_sort)
def dispose(self): @classmethod
def dispose(cls):
# global session # global session
old_session = self.session for inst in cls.instances:
self.session = None old_session = inst.session
if old_session: inst.session = None
try: old_session.close() if old_session:
except: pass try:
if old_session.bind: old_session.close()
try: old_session.bind.dispose() except:
except Exception: pass pass
if old_session.bind:
try:
old_session.bind.dispose()
except Exception:
pass
for attr in list(Books.__dict__.keys()): for attr in list(Books.__dict__.keys()):
if attr.startswith("custom_column_"): if attr.startswith("custom_column_"):
@ -729,10 +756,11 @@ class CalibreDB():
Base.metadata.remove(table) Base.metadata.remove(table)
def reconnect_db(self, config, app_db_path): def reconnect_db(self, config, app_db_path):
self.session.close() self.dispose()
self.engine.dispose() self.engine.dispose()
self.setup_db(config, app_db_path) self.setup_db(config, app_db_path)
def lcase(s): def lcase(s):
try: try:
return unidecode.unidecode(s.lower()) return unidecode.unidecode(s.lower())

3
cps/tasks/convert.py
View File

@ -70,7 +70,8 @@ class TaskConvert(CalibreTask):
return self._handleError(str(e)) return self._handleError(str(e))
def _convert_ebook_format(self): def _convert_ebook_format(self):
local_session = db.Session() error_message = None
local_session = db.CalibreDB()
file_path = self.file_path file_path = self.file_path
book_id = self.bookid book_id = self.bookid
format_old_ext = u'.' + self.settings['old_book_format'].lower() format_old_ext = u'.' + self.settings['old_book_format'].lower()

1169
test/Calibre-Web TestSummary_Linux.html
View File

File diff suppressed because it is too large Load Diff