diff --git a/cps/__init__.py b/cps/__init__.py index b14fb445..d557649c 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -37,6 +37,7 @@ from . import config_sql, logger, cache_buster, cli, ub, db from .reverseproxy import ReverseProxied from .server import WebServer + mimetypes.init() mimetypes.add_type('application/xhtml+xml', '.xhtml') mimetypes.add_type('application/epub+zip', '.epub') @@ -59,7 +60,7 @@ app = Flask(__name__) app.config.update( SESSION_COOKIE_HTTPONLY=True, SESSION_COOKIE_SAMESITE='Lax', - REMEMBER_COOKIE_SAMESITE='Lax', + REMEMBER_COOKIE_SAMESITE='Lax', # will be available in flask-login 0.5.1 earliest ) @@ -82,6 +83,8 @@ log = logger.create() from . import services +calibre_db = db.CalibreDB() + def create_app(): app.wsgi_app = ReverseProxied(app.wsgi_app) # For python2 convert path to unicode @@ -98,7 +101,8 @@ def create_app(): app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) web_server.init_app(app, config) - db.setup_db(config) + calibre_db.setup_db(config, cli.settingspath) + calibre_db.start() babel.init_app(app) _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) diff --git a/cps/about.py b/cps/about.py index 2f1d4b43..676d91db 100644 --- a/cps/about.py +++ b/cps/about.py @@ -30,7 +30,7 @@ import babel, pytz, requests, sqlalchemy import werkzeug, flask, flask_login, flask_principal, jinja2 from flask_babel import gettext as _ -from . import db, converter, uploader, server, isoLanguages, constants +from . import db, calibre_db, converter, uploader, server, isoLanguages, constants from .web import render_title_template try: from flask_login import __version__ as flask_loginVersion @@ -85,10 +85,12 @@ _VERSIONS.update(uploader.get_versions()) @about.route("/stats") @flask_login.login_required def stats(): - counter = db.session.query(db.Books).count() - authors = db.session.query(db.Authors).count() - categorys = db.session.query(db.Tags).count() - series = db.session.query(db.Series).count() - _VERSIONS['ebook converter'] = _(converter.get_version()) + counter = calibre_db.session.query(db.Books).count() + authors = calibre_db.session.query(db.Authors).count() + categorys = calibre_db.session.query(db.Tags).count() + series = calibre_db.session.query(db.Series).count() + _VERSIONS['ebook converter'] = _(converter.get_calibre_version()) + _VERSIONS['unrar'] = _(converter.get_unrar_version()) + _VERSIONS['kepubify'] = _(converter.get_kepubify_version()) return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=_VERSIONS, categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat") diff --git a/cps/admin.py b/cps/admin.py index aa46e8b6..947d0087 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -22,6 +22,7 @@ from __future__ import division, print_function, unicode_literals import os +import re import base64 import json import time @@ -37,8 +38,8 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.sql.expression import func from . import constants, logger, helper, services -from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils -from .helper import speaking_language, check_valid_domain, send_test_mail, reset_password, generate_password_hash +from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils +from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash from .gdriveutils import is_gdrive_ready, gdrive_support from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano @@ -85,7 +86,7 @@ def shutdown(): showtext = {} if task in (0, 1): # valid commandos received # close all database connections - db.dispose() + calibre_db.dispose() ub.dispose() if task == 0: @@ -98,7 +99,7 @@ def shutdown(): if task == 2: log.warning("reconnecting to calibre database") - db.setup_db(config) + calibre_db.setup_db(config, ub.app_DB_path) showtext['text'] = _(u'Reconnect successful') return json.dumps(showtext) @@ -147,10 +148,10 @@ def configuration(): @login_required @admin_required def view_configuration(): - readColumn = db.session.query(db.Custom_Columns)\ - .filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all() - restrictColumns= db.session.query(db.Custom_Columns)\ - .filter(and_(db.Custom_Columns.datatype == 'text',db.Custom_Columns.mark_for_delete == 0)).all() + readColumn = calibre_db.session.query(db.Custom_Columns)\ + .filter(and_(db.Custom_Columns.datatype == 'bool', db.Custom_Columns.mark_for_delete == 0)).all() + restrictColumns= calibre_db.session.query(db.Custom_Columns)\ + .filter(and_(db.Custom_Columns.datatype == 'text', db.Custom_Columns.mark_for_delete == 0)).all() return render_title_template("config_view_edit.html", conf=config, readColumns=readColumn, restrictColumns=restrictColumns, title=_(u"UI Configuration"), page="uiconfig") @@ -168,7 +169,6 @@ def update_view_configuration(): _config_string("config_calibre_web_title") _config_string("config_columns_to_ignore") - # _config_string("config_mature_content_tags") reboot_required |= _config_string("config_title_regex") _config_int("config_read_column") @@ -426,7 +426,6 @@ def delete_restriction(res_type): return "" -#@admi.route("/ajax/listrestriction//", defaults={'user_id': '0'}) @admi.route("/ajax/listrestriction/") @login_required @admin_required @@ -472,6 +471,7 @@ def list_restriction(res_type): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @admi.route("/config", methods=["GET", "POST"]) @unconfigured def basic_configuration(): @@ -481,19 +481,23 @@ def basic_configuration(): return _configuration_result() -def _configuration_update_helper(): - reboot_required = False - db_change = False - to_save = request.form.to_dict() +def _config_int(to_save, x, func=int): + return config.set_from_dictionary(to_save, x, func) - _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) - _config_int = lambda x: config.set_from_dictionary(to_save, x, int) - _config_checkbox = lambda x: config.set_from_dictionary(to_save, x, lambda y: y == "on", False) - _config_checkbox_int = lambda x: config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0) - db_change |= _config_string("config_calibre_dir") +def _config_checkbox(to_save, x): + return config.set_from_dictionary(to_save, x, lambda y: y == "on", False) - # Google drive setup + +def _config_checkbox_int(to_save, x): + return config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0) + + +def _config_string(to_save, x): + return config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) + + +def _configuration_gdrive_helper(to_save): if not os.path.isfile(gdriveutils.SETTINGS_YAML): config.config_use_google_drive = False @@ -512,144 +516,173 @@ def _configuration_update_helper(): # always show google drive settings, but in case of error deny support config.config_use_google_drive = (not gdriveError) and ("config_use_google_drive" in to_save) - if _config_string("config_google_drive_folder"): + if _config_string(to_save, "config_google_drive_folder"): gdriveutils.deleteDatabaseOnChange() + return gdriveError - reboot_required |= _config_int("config_port") +def _configuration_oauth_helper(to_save): + active_oauths = 0 + reboot_required = False + for element in oauthblueprints: + if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \ + or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']: + reboot_required = True + element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"] + element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"] + if to_save["config_" + str(element['id']) + "_oauth_client_id"] \ + and to_save["config_" + str(element['id']) + "_oauth_client_secret"]: + active_oauths += 1 + element["active"] = 1 + else: + element["active"] = 0 + ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update( + {"oauth_client_id": to_save["config_" + str(element['id']) + "_oauth_client_id"], + "oauth_client_secret": to_save["config_" + str(element['id']) + "_oauth_client_secret"], + "active": element["active"]}) + return reboot_required - reboot_required |= _config_string("config_keyfile") +def _configuration_logfile_helper(to_save, gdriveError): + reboot_required = False + reboot_required |= _config_int(to_save, "config_log_level") + reboot_required |= _config_string(to_save, "config_logfile") + if not logger.is_valid_logfile(config.config_logfile): + return reboot_required, _configuration_result(_('Logfile Location is not Valid, Please Enter Correct Path'), gdriveError) + + reboot_required |= _config_checkbox_int(to_save, "config_access_log") + reboot_required |= _config_string(to_save, "config_access_logfile") + if not logger.is_valid_logfile(config.config_access_logfile): + return reboot_required, _configuration_result(_('Access Logfile Location is not Valid, Please Enter Correct Path'), gdriveError) + return reboot_required, None + +def _configuration_ldap_helper(to_save, gdriveError): + reboot_required = False + reboot_required |= _config_string(to_save, "config_ldap_provider_url") + reboot_required |= _config_int(to_save, "config_ldap_port") + reboot_required |= _config_int(to_save, "config_ldap_authentication") + reboot_required |= _config_string(to_save, "config_ldap_dn") + reboot_required |= _config_string(to_save, "config_ldap_serv_username") + reboot_required |= _config_string(to_save, "config_ldap_user_object") + reboot_required |= _config_string(to_save, "config_ldap_group_object_filter") + reboot_required |= _config_string(to_save, "config_ldap_group_members_field") + reboot_required |= _config_checkbox(to_save, "config_ldap_openldap") + reboot_required |= _config_int(to_save, "config_ldap_encryption") + reboot_required |= _config_string(to_save, "config_ldap_cert_path") + _config_string(to_save, "config_ldap_group_name") + if "config_ldap_serv_password" in to_save and to_save["config_ldap_serv_password"] != "": + reboot_required |= 1 + config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8') + config.save() + + if not config.config_ldap_provider_url \ + or not config.config_ldap_port \ + or not config.config_ldap_dn \ + or not config.config_ldap_user_object: + return reboot_required, _configuration_result(_('Please Enter a LDAP Provider, ' + 'Port, DN and User Object Identifier'), gdriveError) + + if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS: + if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE: + if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password): + return reboot_required, _configuration_result('Please Enter a LDAP Service Account and Password', gdriveError) + else: + if not config.config_ldap_serv_username: + return reboot_required, _configuration_result('Please Enter a LDAP Service Account', gdriveError) + + if config.config_ldap_group_object_filter: + if config.config_ldap_group_object_filter.count("%s") != 1: + return reboot_required, _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'), + gdriveError) + if config.config_ldap_group_object_filter.count("(") != config.config_ldap_group_object_filter.count(")"): + return reboot_required, _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'), + gdriveError) + + if config.config_ldap_user_object.count("%s") != 1: + return reboot_required, _configuration_result(_('LDAP User Object Filter needs to Have One "%s" Format Identifier'), + gdriveError) + if config.config_ldap_user_object.count("(") != config.config_ldap_user_object.count(")"): + return reboot_required, _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'), + gdriveError) + + if config.config_ldap_cert_path and not os.path.isdir(config.config_ldap_cert_path): + return reboot_required, _configuration_result(_('LDAP Certificate Location is not Valid, Please Enter Correct Path'), + gdriveError) + return reboot_required, None + + +def _configuration_update_helper(): + reboot_required = False + db_change = False + to_save = request.form.to_dict() + + to_save['config_calibre_dir'] = re.sub('[\\/]metadata\.db$', '', to_save['config_calibre_dir'], flags=re.IGNORECASE) + db_change |= _config_string(to_save, "config_calibre_dir") + + # Google drive setup + gdriveError = _configuration_gdrive_helper(to_save) + + reboot_required |= _config_int(to_save, "config_port") + + reboot_required |= _config_string(to_save, "config_keyfile") if config.config_keyfile and not os.path.isfile(config.config_keyfile): return _configuration_result(_('Keyfile Location is not Valid, Please Enter Correct Path'), gdriveError) - reboot_required |= _config_string("config_certfile") + reboot_required |= _config_string(to_save, "config_certfile") if config.config_certfile and not os.path.isfile(config.config_certfile): return _configuration_result(_('Certfile Location is not Valid, Please Enter Correct Path'), gdriveError) - _config_checkbox_int("config_uploading") - _config_checkbox_int("config_anonbrowse") - _config_checkbox_int("config_public_reg") - reboot_required |= _config_checkbox_int("config_kobo_sync") - _config_checkbox_int("config_kobo_proxy") + _config_checkbox_int(to_save, "config_uploading") + _config_checkbox_int(to_save, "config_anonbrowse") + _config_checkbox_int(to_save, "config_public_reg") + _config_checkbox_int(to_save, "config_register_email") + reboot_required |= _config_checkbox_int(to_save, "config_kobo_sync") + _config_checkbox_int(to_save, "config_kobo_proxy") + _config_string(to_save, "config_upload_formats") + constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in config.config_upload_formats.split(',')] - _config_int("config_ebookconverter") - _config_string("config_calibre") - _config_string("config_converterpath") + _config_string(to_save, "config_calibre") + _config_string(to_save, "config_converterpath") + _config_string(to_save, "config_kepubifypath") - reboot_required |= _config_int("config_login_type") + reboot_required |= _config_int(to_save, "config_login_type") #LDAP configurator, if config.config_login_type == constants.LOGIN_LDAP: - reboot_required |= _config_string("config_ldap_provider_url") - reboot_required |= _config_int("config_ldap_port") - reboot_required |= _config_int("config_ldap_authentication") - reboot_required |= _config_string("config_ldap_dn") - reboot_required |= _config_string("config_ldap_serv_username") - reboot_required |= _config_string("config_ldap_user_object") - reboot_required |= _config_string("config_ldap_group_object_filter") - reboot_required |= _config_string("config_ldap_group_members_field") - reboot_required |= _config_checkbox("config_ldap_openldap") - reboot_required |= _config_int("config_ldap_encryption") - reboot_required |= _config_string("config_ldap_cert_path") - _config_string("config_ldap_group_name") - if "config_ldap_serv_password" in to_save and to_save["config_ldap_serv_password"] != "": - reboot_required |= 1 - config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8') - config.save() - - if not config.config_ldap_provider_url \ - or not config.config_ldap_port \ - or not config.config_ldap_dn \ - or not config.config_ldap_user_object: - return _configuration_result(_('Please Enter a LDAP Provider, ' - 'Port, DN and User Object Identifier'), gdriveError) - - if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS: - if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE: - if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password): - return _configuration_result('Please Enter a LDAP Service Account and Password', gdriveError) - else: - if not config.config_ldap_serv_username: - return _configuration_result('Please Enter a LDAP Service Account', gdriveError) - - #_config_checkbox("config_ldap_use_ssl") - #_config_checkbox("config_ldap_use_tls") - # reboot_required |= _config_checkbox("config_ldap_openldap") - # _config_checkbox("config_ldap_require_cert") - - if config.config_ldap_group_object_filter: - if config.config_ldap_group_object_filter.count("%s") != 1: - return _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'), - gdriveError) - if config.config_ldap_group_object_filter.count("(") != config.config_ldap_group_object_filter.count(")"): - return _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'), - gdriveError) - - if config.config_ldap_user_object.count("%s") != 1: - return _configuration_result(_('LDAP User Object Filter needs to Have One "%s" Format Identifier'), - gdriveError) - if config.config_ldap_user_object.count("(") != config.config_ldap_user_object.count(")"): - return _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'), - gdriveError) - - if config.config_ldap_cert_path and not os.path.isdir(config.config_ldap_cert_path): - return _configuration_result(_('LDAP Certificate Location is not Valid, Please Enter Correct Path'), - gdriveError) + reboot, message = _configuration_ldap_helper(to_save, gdriveError) + if message: + return message + reboot_required |= reboot # Remote login configuration - _config_checkbox("config_remote_login") + _config_checkbox(to_save, "config_remote_login") if not config.config_remote_login: ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.token_type==0).delete() # Goodreads configuration - _config_checkbox("config_use_goodreads") - _config_string("config_goodreads_api_key") - _config_string("config_goodreads_api_secret") + _config_checkbox(to_save, "config_use_goodreads") + _config_string(to_save, "config_goodreads_api_key") + _config_string(to_save, "config_goodreads_api_secret") if services.goodreads_support: services.goodreads_support.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) - _config_int("config_updatechannel") + _config_int(to_save, "config_updatechannel") # Reverse proxy login configuration - _config_checkbox("config_allow_reverse_proxy_header_login") - _config_string("config_reverse_proxy_login_header_name") + _config_checkbox(to_save, "config_allow_reverse_proxy_header_login") + _config_string(to_save, "config_reverse_proxy_login_header_name") - # GitHub OAuth configuration + # OAuth configuration if config.config_login_type == constants.LOGIN_OAUTH: - active_oauths = 0 - - for element in oauthblueprints: - if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \ - or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']: - reboot_required = True - element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"] - element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"] - if to_save["config_"+str(element['id'])+"_oauth_client_id"] \ - and to_save["config_"+str(element['id'])+"_oauth_client_secret"]: - active_oauths += 1 - element["active"] = 1 - else: - element["active"] = 0 - ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update( - {"oauth_client_id":to_save["config_"+str(element['id'])+"_oauth_client_id"], - "oauth_client_secret":to_save["config_"+str(element['id'])+"_oauth_client_secret"], - "active":element["active"]}) - - - reboot_required |= _config_int("config_log_level") - reboot_required |= _config_string("config_logfile") - if not logger.is_valid_logfile(config.config_logfile): - return _configuration_result(_('Logfile Location is not Valid, Please Enter Correct Path'), gdriveError) - - reboot_required |= _config_checkbox_int("config_access_log") - reboot_required |= _config_string("config_access_logfile") - if not logger.is_valid_logfile(config.config_access_logfile): - return _configuration_result(_('Access Logfile Location is not Valid, Please Enter Correct Path'), gdriveError) + reboot_required |= _configuration_oauth_helper(to_save) + reboot, message = _configuration_logfile_helper(to_save, gdriveError) + if message: + return message + reboot_required |= reboot # Rarfile Content configuration - _config_string("config_rarfile_location") + _config_string(to_save, "config_rarfile_location") unrar_status = helper.check_unrar(config.config_rarfile_location) if unrar_status: return _configuration_result(unrar_status, gdriveError) @@ -663,9 +696,10 @@ def _configuration_update_helper(): return _configuration_result('%s' % e, gdriveError) if db_change: - # reload(db) - if not db.setup_db(config): + if not calibre_db.setup_db(config, ub.app_DB_path): return _configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdriveError) + if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK): + flash(_(u"DB is not Writeable"), category="warning") config.save() flash(_(u"Calibre-Web configuration updated"), category="success") @@ -701,62 +735,155 @@ def _configuration_result(error_flash=None, gdriveError=None): title=_(u"Basic Configuration"), page="config") +def _handle_new_user(to_save, content,languages, translations, kobo_support): + content.default_language = to_save["default_language"] + # content.mature_content = "Show_mature_content" in to_save + content.locale = to_save.get("locale", content.locale) + + content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_')) + if "show_detail_random" in to_save: + content.sidebar_view |= constants.DETAIL_RANDOM + + content.role = constants.selected_roles(to_save) + + if not to_save["nickname"] or not to_save["email"] or not to_save["password"]: + flash(_(u"Please fill out all fields!"), category="error") + return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, + registered_oauth=oauth_check, kobo_support=kobo_support, + title=_(u"Add new user")) + content.password = generate_password_hash(to_save["password"]) + existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower()) \ + .first() + existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ + .first() + if not existing_user and not existing_email: + content.nickname = to_save["nickname"] + if config.config_public_reg and not check_valid_domain(to_save["email"]): + flash(_(u"E-mail is not from valid domain"), category="error") + return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, + registered_oauth=oauth_check, kobo_support=kobo_support, + title=_(u"Add new user")) + else: + content.email = to_save["email"] + else: + flash(_(u"Found an existing account for this e-mail address or nickname."), category="error") + return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, + languages=languages, title=_(u"Add new user"), page="newuser", + kobo_support=kobo_support, registered_oauth=oauth_check) + try: + content.allowed_tags = config.config_allowed_tags + content.denied_tags = config.config_denied_tags + content.allowed_column_value = config.config_allowed_column_value + content.denied_column_value = config.config_denied_column_value + ub.session.add(content) + ub.session.commit() + flash(_(u"User '%(user)s' created", user=content.nickname), category="success") + return redirect(url_for('admin.admin')) + except IntegrityError: + ub.session.rollback() + flash(_(u"Found an existing account for this e-mail address or nickname."), category="error") + + +def _handle_edit_user(to_save, content,languages, translations, kobo_support, downloads): + if "delete" in to_save: + if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, + ub.User.id != content.id).count(): + ub.session.query(ub.User).filter(ub.User.id == content.id).delete() + ub.session.commit() + flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success") + return redirect(url_for('admin.admin')) + else: + flash(_(u"No admin user remaining, can't delete user", nick=content.nickname), category="error") + return redirect(url_for('admin.admin')) + else: + if not ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, + ub.User.id != content.id).count() and \ + not 'admin_role' in to_save: + flash(_(u"No admin user remaining, can't remove admin role", nick=content.nickname), category="error") + return redirect(url_for('admin.admin')) + + if "password" in to_save and to_save["password"]: + content.password = generate_password_hash(to_save["password"]) + anonymous = content.is_anonymous + content.role = constants.selected_roles(to_save) + if anonymous: + content.role |= constants.ROLE_ANONYMOUS + else: + content.role &= ~constants.ROLE_ANONYMOUS + + val = [int(k[5:]) for k in to_save if k.startswith('show_')] + sidebar = ub.get_sidebar_config() + for element in sidebar: + value = element['visibility'] + if value in val and not content.check_visibility(value): + content.sidebar_view |= value + elif not value in val and content.check_visibility(value): + content.sidebar_view &= ~value + + if "Show_detail_random" in to_save: + content.sidebar_view |= constants.DETAIL_RANDOM + else: + content.sidebar_view &= ~constants.DETAIL_RANDOM + + if "default_language" in to_save: + content.default_language = to_save["default_language"] + if "locale" in to_save and to_save["locale"]: + content.locale = to_save["locale"] + if to_save["email"] and to_save["email"] != content.email: + existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ + .first() + if not existing_email: + content.email = to_save["email"] + else: + flash(_(u"Found an existing account for this e-mail address."), category="error") + return render_title_template("user_edit.html", + translations=translations, + languages=languages, + mail_configured=config.get_mail_server_configured(), + kobo_support=kobo_support, + new_user=0, + content=content, + downloads=downloads, + registered_oauth=oauth_check, + title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") + if "nickname" in to_save and to_save["nickname"] != content.nickname: + # Query User nickname, if not existing, change + if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar(): + content.nickname = to_save["nickname"] + else: + flash(_(u"This username is already taken"), category="error") + return render_title_template("user_edit.html", + translations=translations, + languages=languages, + mail_configured=config.get_mail_server_configured(), + new_user=0, content=content, + downloads=downloads, + registered_oauth=oauth_check, + kobo_support=kobo_support, + title=_(u"Edit User %(nick)s", nick=content.nickname), + page="edituser") + + if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: + content.kindle_mail = to_save["kindle_mail"] + try: + ub.session.commit() + flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success") + except IntegrityError: + ub.session.rollback() + flash(_(u"An unknown error occured."), category="error") + + @admi.route("/admin/user/new", methods=["GET", "POST"]) @login_required @admin_required def new_user(): content = ub.User() - languages = speaking_language() + languages = calibre_db.speaking_language() translations = [LC('en')] + babel.list_translations() kobo_support = feature_support['kobo'] and config.config_kobo_sync if request.method == "POST": to_save = request.form.to_dict() - content.default_language = to_save["default_language"] - # content.mature_content = "Show_mature_content" in to_save - content.locale = to_save.get("locale", content.locale) - - content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_')) - if "show_detail_random" in to_save: - content.sidebar_view |= constants.DETAIL_RANDOM - - content.role = constants.selected_roles(to_save) - - if not to_save["nickname"] or not to_save["email"] or not to_save["password"]: - flash(_(u"Please fill out all fields!"), category="error") - return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, - registered_oauth=oauth_check, kobo_support=kobo_support, - title=_(u"Add new user")) - content.password = generate_password_hash(to_save["password"]) - existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower())\ - .first() - existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower())\ - .first() - if not existing_user and not existing_email: - content.nickname = to_save["nickname"] - if config.config_public_reg and not check_valid_domain(to_save["email"]): - flash(_(u"E-mail is not from valid domain"), category="error") - return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, - registered_oauth=oauth_check, kobo_support=kobo_support, - title=_(u"Add new user")) - else: - content.email = to_save["email"] - else: - flash(_(u"Found an existing account for this e-mail address or nickname."), category="error") - return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, - languages=languages, title=_(u"Add new user"), page="newuser", - kobo_support=kobo_support, registered_oauth=oauth_check) - try: - content.allowed_tags = config.config_allowed_tags - content.denied_tags = config.config_denied_tags - content.allowed_column_value = config.config_allowed_column_value - content.denied_column_value = config.config_denied_column_value - ub.session.add(content) - ub.session.commit() - flash(_(u"User '%(user)s' created", user=content.nickname), category="success") - return redirect(url_for('admin.admin')) - except IntegrityError: - ub.session.rollback() - flash(_(u"Found an existing account for this e-mail address or nickname."), category="error") + _handle_new_user(to_save, content, languages, translations, kobo_support) else: content.role = config.config_default_role content.sidebar_view = config.config_default_show @@ -770,8 +897,7 @@ def new_user(): @admin_required def edit_mailsettings(): content = config.get_mail_settings() - # log.debug("edit_mailsettings %r", content) - return render_title_template("email_edit.html", content=content, title=_(u"Edit e-mail server settings"), + return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"), page="mailset") @@ -780,17 +906,15 @@ def edit_mailsettings(): @admin_required def update_mailsettings(): to_save = request.form.to_dict() - log.debug("update_mailsettings %r", to_save) + # log.debug("update_mailsettings %r", to_save) - _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) - _config_int = lambda x: config.set_from_dictionary(to_save, x, int) - - _config_string("mail_server") - _config_int("mail_port") - _config_int("mail_use_ssl") - _config_string("mail_login") - _config_string("mail_password") - _config_string("mail_from") + _config_string(to_save, "mail_server") + _config_int(to_save, "mail_port") + _config_int(to_save, "mail_use_ssl") + _config_string(to_save, "mail_login") + _config_string(to_save, "mail_password") + _config_string(to_save, "mail_from") + _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024) config.save() if to_save.get("test"): @@ -818,105 +942,18 @@ def edit_user(user_id): flash(_(u"User not found"), category="error") return redirect(url_for('admin.admin')) downloads = list() - languages = speaking_language() + languages = calibre_db.speaking_language() translations = babel.list_translations() + [LC('en')] kobo_support = feature_support['kobo'] and config.config_kobo_sync for book in content.downloads: - downloadbook = db.session.query(db.Books).filter(db.Books.id == book.book_id).first() + downloadbook = calibre_db.get_book(book.book_id) if downloadbook: downloads.append(downloadbook) else: ub.delete_download(book.book_id) - # ub.session.query(ub.Downloads).filter(book.book_id == ub.Downloads.book_id).delete() - # ub.session.commit() if request.method == "POST": to_save = request.form.to_dict() - if "delete" in to_save: - if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, - ub.User.id != content.id).count(): - ub.session.query(ub.User).filter(ub.User.id == content.id).delete() - ub.session.commit() - flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success") - return redirect(url_for('admin.admin')) - else: - flash(_(u"No admin user remaining, can't delete user", nick=content.nickname), category="error") - return redirect(url_for('admin.admin')) - else: - if not ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, - ub.User.id != content.id).count() and \ - not 'admin_role' in to_save: - flash(_(u"No admin user remaining, can't remove admin role", nick=content.nickname), category="error") - return redirect(url_for('admin.admin')) - - if "password" in to_save and to_save["password"]: - content.password = generate_password_hash(to_save["password"]) - anonymous = content.is_anonymous - content.role = constants.selected_roles(to_save) - if anonymous: - content.role |= constants.ROLE_ANONYMOUS - else: - content.role &= ~constants.ROLE_ANONYMOUS - - val = [int(k[5:]) for k in to_save if k.startswith('show_')] - sidebar = ub.get_sidebar_config() - for element in sidebar: - value = element['visibility'] - if value in val and not content.check_visibility(value): - content.sidebar_view |= value - elif not value in val and content.check_visibility(value): - content.sidebar_view &= ~value - - if "Show_detail_random" in to_save: - content.sidebar_view |= constants.DETAIL_RANDOM - else: - content.sidebar_view &= ~constants.DETAIL_RANDOM - - if "default_language" in to_save: - content.default_language = to_save["default_language"] - if "locale" in to_save and to_save["locale"]: - content.locale = to_save["locale"] - if to_save["email"] and to_save["email"] != content.email: - existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ - .first() - if not existing_email: - content.email = to_save["email"] - else: - flash(_(u"Found an existing account for this e-mail address."), category="error") - return render_title_template("user_edit.html", - translations=translations, - languages=languages, - mail_configured = config.get_mail_server_configured(), - kobo_support=kobo_support, - new_user=0, - content=content, - downloads=downloads, - registered_oauth=oauth_check, - title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") - if "nickname" in to_save and to_save["nickname"] != content.nickname: - # Query User nickname, if not existing, change - if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar(): - content.nickname = to_save["nickname"] - else: - flash(_(u"This username is already taken"), category="error") - return render_title_template("user_edit.html", - translations=translations, - languages=languages, - mail_configured=config.get_mail_server_configured(), - new_user=0, content=content, - downloads=downloads, - registered_oauth=oauth_check, - kobo_support=kobo_support, - title=_(u"Edit User %(nick)s", nick=content.nickname), - page="edituser") - - if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: - content.kindle_mail = to_save["kindle_mail"] - try: - ub.session.commit() - flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success") - except IntegrityError: - ub.session.rollback() - flash(_(u"An unknown error occured."), category="error") + _handle_edit_user(to_save, content, languages, translations, kobo_support, downloads) return render_title_template("user_edit.html", translations=translations, languages=languages, diff --git a/cps/comic.py b/cps/comic.py index 2ee182c4..f6bf84b2 100644 --- a/cps/comic.py +++ b/cps/comic.py @@ -18,10 +18,17 @@ from __future__ import division, print_function, unicode_literals import os +import io from . import logger, isoLanguages from .constants import BookMeta +try: + from PIL import Image as PILImage + use_PIL = True +except ImportError as e: + use_PIL = False + log = logger.create() @@ -29,18 +36,43 @@ log = logger.create() try: from comicapi.comicarchive import ComicArchive, MetaDataStyle use_comic_meta = True + try: + from comicapi import __version__ as comic_version + except (ImportError): + comic_version = '' except ImportError as e: - log.debug('cannot import comicapi, extracting comic metadata will not work: %s', e) + log.debug('Cannot import comicapi, extracting comic metadata will not work: %s', e) import zipfile import tarfile try: import rarfile use_rarfile = True except ImportError as e: - log.debug('cannot import rarfile, extracting cover files from rar files will not work: %s', e) + log.debug('Cannot import rarfile, extracting cover files from rar files will not work: %s', e) use_rarfile = False use_comic_meta = False +def _cover_processing(tmp_file_name, img, extension): + if use_PIL: + # convert to jpg because calibre only supports jpg + if extension in ('.png', '.webp'): + imgc = PILImage.open(io.BytesIO(img)) + im = imgc.convert('RGB') + tmp_bytesio = io.BytesIO() + im.save(tmp_bytesio, format='JPEG') + img = tmp_bytesio.getvalue() + + prefix = os.path.dirname(tmp_file_name) + if img: + tmp_cover_name = prefix + '/cover.jpg' + image = open(tmp_cover_name, 'wb') + image.write(img) + image.close() + else: + tmp_cover_name = None + return tmp_cover_name + + def _extractCover(tmp_file_name, original_file_extension, rarExceutable): cover_data = extension = None @@ -50,7 +82,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension == '.jpg' or extension == '.jpeg': + if extension in ('.jpg', '.jpeg', '.png', '.webp'): cover_data = archive.getPage(index) break else: @@ -60,7 +92,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension == '.jpg' or extension == '.jpeg': + if extension in ('.jpg', '.jpeg', '.png', '.webp'): cover_data = cf.read(name) break elif original_file_extension.upper() == '.CBT': @@ -69,7 +101,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension == '.jpg' or extension == '.jpeg': + if extension in ('.jpg', '.jpeg', '.png', '.webp'): cover_data = cf.extractfile(name).read() break elif original_file_extension.upper() == '.CBR' and use_rarfile: @@ -80,21 +112,12 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension == '.jpg' or extension == '.jpeg': + if extension in ('.jpg', '.jpeg', '.png', '.webp'): cover_data = cf.read(name) break except Exception as e: log.debug('Rarfile failed with error: %s', e) - - prefix = os.path.dirname(tmp_file_name) - if cover_data: - tmp_cover_name = prefix + '/cover' + extension - image = open(tmp_cover_name, 'wb') - image.write(cover_data) - image.close() - else: - tmp_cover_name = None - return tmp_cover_name + return _cover_processing(tmp_file_name, cover_data, extension) def get_comic_info(tmp_file_path, original_file_name, original_file_extension, rarExceutable): diff --git a/cps/config_sql.py b/cps/config_sql.py index bb88b7a3..1135516d 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -53,6 +53,7 @@ class _Settings(_Base): mail_login = Column(String, default='mail@example.com') mail_password = Column(String, default='mypassword') mail_from = Column(String, default='automailer ') + mail_size = Column(Integer, default=25*1024*1024) config_calibre_dir = Column(String) config_port = Column(Integer, default=constants.DEFAULT_PORT) @@ -96,7 +97,7 @@ class _Settings(_Base): config_use_goodreads = Column(Boolean, default=False) config_goodreads_api_key = Column(String) config_goodreads_api_secret = Column(String) - + config_register_email = Column(Boolean, default=False) config_login_type = Column(Integer, default=0) config_kobo_proxy = Column(Boolean, default=False) @@ -116,10 +117,11 @@ class _Settings(_Base): config_ldap_group_members_field = Column(String, default='memberUid') config_ldap_group_name = Column(String, default='calibreweb') - config_ebookconverter = Column(Integer, default=0) - config_converterpath = Column(String) + config_kepubifypath = Column(String, default=None) + config_converterpath = Column(String, default=None) config_calibre = Column(String) - config_rarfile_location = Column(String) + config_rarfile_location = Column(String, default=None) + config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) @@ -140,6 +142,22 @@ class _ConfigSQL(object): self.config_calibre_dir = None self.load() + change = False + if self.config_converterpath == None: + change = True + self.config_converterpath = autodetect_calibre_binary() + + if self.config_kepubifypath == None: + change = True + self.config_kepubifypath = autodetect_kepubify_binary() + + if self.config_rarfile_location == None: + change = True + self.config_rarfile_location = autodetect_unrar_binary() + if change: + self.save() + + def _read_from_storage(self): if self._settings is None: log.debug("_ConfigSQL._read_from_storage") @@ -264,7 +282,8 @@ class _ConfigSQL(object): setattr(self, k, v) if self.config_google_drive_watch_changes_response: - self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response) + self.config_google_drive_watch_changes_response = \ + json.loads(self.config_google_drive_watch_changes_response) have_metadata_db = bool(self.config_calibre_dir) if have_metadata_db: @@ -272,8 +291,13 @@ class _ConfigSQL(object): db_file = os.path.join(self.config_calibre_dir, 'metadata.db') have_metadata_db = os.path.isfile(db_file) self.db_configured = have_metadata_db - - logger.setup(self.config_logfile, self.config_log_level) + constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in self.config_upload_formats.split(',')] + logfile = logger.setup(self.config_logfile, self.config_log_level) + if logfile != self.config_logfile: + log.warning("Log path %s not valid, falling back to default", self.config_logfile) + self.config_logfile = logfile + self._session.merge(s) + self._session.commit() def save(self): '''Apply all configuration values to the underlying storage.''' @@ -334,17 +358,41 @@ def _migrate_table(session, orm_class): if changed: session.commit() + def autodetect_calibre_binary(): if sys.platform == "win32": - calibre_path = ["C:\\program files\calibre\calibre-convert.exe", - "C:\\program files(x86)\calibre\calibre-convert.exe"] + calibre_path = ["C:\\program files\calibre\ebook-convert.exe", + "C:\\program files(x86)\calibre\ebook-convert.exe", + "C:\\program files(x86)\calibre2\ebook-convert.exe", + "C:\\program files\calibre2\ebook-convert.exe"] else: calibre_path = ["/opt/calibre/ebook-convert"] for element in calibre_path: if os.path.isfile(element) and os.access(element, os.X_OK): return element - return None + return "" +def autodetect_unrar_binary(): + if sys.platform == "win32": + calibre_path = ["C:\\program files\\WinRar\\unRAR.exe", + "C:\\program files(x86)\\WinRar\\unRAR.exe"] + else: + calibre_path = ["/usr/bin/unrar"] + for element in calibre_path: + if os.path.isfile(element) and os.access(element, os.X_OK): + return element + return "" + +def autodetect_kepubify_binary(): + if sys.platform == "win32": + calibre_path = ["C:\\program files\\kepubify\\kepubify-windows-64Bit.exe", + "C:\\program files(x86)\\kepubify\\kepubify-windows-64Bit.exe"] + else: + calibre_path = ["/opt/kepubify/kepubify-linux-64bit", "/opt/kepubify/kepubify-linux-32bit"] + for element in calibre_path: + if os.path.isfile(element) and os.access(element, os.X_OK): + return element + return "" def _migrate_database(session): # make sure the table is created, if it does not exist diff --git a/cps/constants.py b/cps/constants.py index 1698ef91..4649f5d9 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -81,6 +81,7 @@ SIDEBAR_PUBLISHER = 1 << 12 SIDEBAR_RATING = 1 << 13 SIDEBAR_FORMAT = 1 << 14 SIDEBAR_ARCHIVED = 1 << 15 +# SIDEBAR_LIST = 1 << 16 ADMIN_USER_ROLES = sum(r for r in ALL_ROLES.values()) & ~ROLE_ANONYMOUS ADMIN_USER_SIDEBAR = (SIDEBAR_ARCHIVED << 1) - 1 @@ -111,11 +112,9 @@ del env_CALIBRE_PORT EXTENSIONS_AUDIO = {'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac'} -EXTENSIONS_CONVERT = {'pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt'} -EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx', +EXTENSIONS_CONVERT = ['pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt'] +EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac'} -# EXTENSIONS_READER = set(['txt', 'pdf', 'epub', 'zip', 'cbz', 'tar', 'cbt'] + -# (['rar','cbr'] if feature_support['rar'] else [])) def has_flag(value, bit_flag): diff --git a/cps/converter.py b/cps/converter.py index d3482e5f..01a6fbc7 100644 --- a/cps/converter.py +++ b/cps/converter.py @@ -19,6 +19,7 @@ from __future__ import division, print_function, unicode_literals import os import re +import sys from flask_babel import gettext as _ from . import config, logger @@ -29,8 +30,8 @@ log = logger.create() # _() necessary to make babel aware of string for translation _NOT_CONFIGURED = _('not configured') -_NOT_INSTALLED = 'not installed' -_EXECUTION_ERROR = 'Execution permissions missing' +_NOT_INSTALLED = _('not installed') +_EXECUTION_ERROR = _('Execution permissions missing') def _get_command_version(path, pattern, argument=None): @@ -48,10 +49,15 @@ def _get_command_version(path, pattern, argument=None): return _NOT_INSTALLED -def get_version(): - version = None - if config.config_ebookconverter == 1: - version = _get_command_version(config.config_converterpath, r'Amazon kindlegen\(') - elif config.config_ebookconverter == 2: - version = _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') - return version or _NOT_CONFIGURED +def get_calibre_version(): + return _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') \ + or _NOT_CONFIGURED + + +def get_unrar_version(): + return _get_command_version(config.config_rarfile_location, r'UNRAR.*\d') or _NOT_CONFIGURED + +def get_kepubify_version(): + return _get_command_version(config.config_kepubifypath, r'kepubify\s','--version') or _NOT_CONFIGURED + + diff --git a/cps/db.py b/cps/db.py old mode 100755 new mode 100644 index 6d31e599..d309da89 --- a/cps/db.py +++ b/cps/db.py @@ -22,18 +22,34 @@ import sys import os import re import ast +import json +from datetime import datetime +import threading from sqlalchemy import create_engine -from sqlalchemy import Table, Column, ForeignKey -from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float, DateTime +from sqlalchemy import Table, Column, ForeignKey, CheckConstraint +from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.exc import OperationalError +from flask_login import current_user +from sqlalchemy.sql.expression import and_, true, false, text, func, or_ +from babel import Locale as LC +from babel.core import UnknownLocaleError +from flask_babel import gettext as _ + +from . import logger, ub, isoLanguages +from .pagination import Pagination + +try: + import unidecode + use_unidecode = True +except ImportError: + use_unidecode = False -session = None cc_exceptions = ['datetime', 'comments', 'composite', 'series'] cc_classes = {} -engine = None Base = declarative_base() @@ -72,9 +88,9 @@ class Identifiers(Base): __tablename__ = 'identifiers' id = Column(Integer, primary_key=True) - type = Column(String) - val = Column(String) - book = Column(Integer, ForeignKey('books.id')) + type = Column(String(collation='NOCASE'), nullable=False, default="isbn") + val = Column(String(collation='NOCASE'), nullable=False) + book = Column(Integer, ForeignKey('books.id'), nullable=False) def __init__(self, val, id_type, book): self.val = val @@ -126,8 +142,8 @@ class Comments(Base): __tablename__ = 'comments' id = Column(Integer, primary_key=True) - text = Column(String) - book = Column(Integer, ForeignKey('books.id')) + text = Column(String(collation='NOCASE'), nullable=False) + book = Column(Integer, ForeignKey('books.id'), nullable=False) def __init__(self, text, book): self.text = text @@ -141,7 +157,7 @@ class Tags(Base): __tablename__ = 'tags' id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(String) + name = Column(String(collation='NOCASE'), unique=True, nullable=False) def __init__(self, name): self.name = name @@ -154,9 +170,9 @@ class Authors(Base): __tablename__ = 'authors' id = Column(Integer, primary_key=True) - name = Column(String) - sort = Column(String) - link = Column(String) + name = Column(String(collation='NOCASE'), unique=True, nullable=False) + sort = Column(String(collation='NOCASE')) + link = Column(String, nullable=False, default="") def __init__(self, name, sort, link): self.name = name @@ -171,8 +187,8 @@ class Series(Base): __tablename__ = 'series' id = Column(Integer, primary_key=True) - name = Column(String) - sort = Column(String) + name = Column(String(collation='NOCASE'), unique=True, nullable=False) + sort = Column(String(collation='NOCASE')) def __init__(self, name, sort): self.name = name @@ -186,7 +202,7 @@ class Ratings(Base): __tablename__ = 'ratings' id = Column(Integer, primary_key=True) - rating = Column(Integer) + rating = Column(Integer, CheckConstraint('rating>-1 AND rating<11'), unique=True) def __init__(self, rating): self.rating = rating @@ -199,7 +215,7 @@ class Languages(Base): __tablename__ = 'languages' id = Column(Integer, primary_key=True) - lang_code = Column(String) + lang_code = Column(String(collation='NOCASE'), nullable=False, unique=True) def __init__(self, lang_code): self.lang_code = lang_code @@ -212,8 +228,8 @@ class Publishers(Base): __tablename__ = 'publishers' id = Column(Integer, primary_key=True) - name = Column(String) - sort = Column(String) + name = Column(String(collation='NOCASE'), nullable=False, unique=True) + sort = Column(String(collation='NOCASE')) def __init__(self, name, sort): self.name = name @@ -225,12 +241,13 @@ class Publishers(Base): class Data(Base): __tablename__ = 'data' + __table_args__ = {'schema':'calibre'} id = Column(Integer, primary_key=True) - book = Column(Integer, ForeignKey('books.id')) - format = Column(String) - uncompressed_size = Column(Integer) - name = Column(String) + book = Column(Integer, ForeignKey('books.id'), nullable=False) + format = Column(String(collation='NOCASE'), nullable=False) + uncompressed_size = Column(Integer, nullable=False) + name = Column(String, nullable=False) def __init__(self, book, book_format, uncompressed_size, name): self.book = book @@ -247,17 +264,20 @@ class Books(Base): DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00" - id = Column(Integer, primary_key=True) - title = Column(String) - sort = Column(String) - author_sort = Column(String) - timestamp = Column(TIMESTAMP) - pubdate = Column(String) - series_index = Column(String) - last_modified = Column(TIMESTAMP) - path = Column(String) - has_cover = Column(Integer) + id = Column(Integer, primary_key=True, autoincrement=True) + title = Column(String(collation='NOCASE'), nullable=False, default='Unknown') + sort = Column(String(collation='NOCASE')) + author_sort = Column(String(collation='NOCASE')) + timestamp = Column(TIMESTAMP, default=datetime.utcnow) + pubdate = Column(String) # , default=datetime.utcnow) + series_index = Column(String, nullable=False, default="1.0") + last_modified = Column(TIMESTAMP, default=datetime.utcnow) + path = Column(String, default="", nullable=False) + has_cover = Column(Integer, default=0) uuid = Column(String) + isbn = Column(String(collation='NOCASE'), default="") + # Iccn = Column(String(collation='NOCASE'), default="") + flags = Column(Integer, nullable=False, default=1) authors = relationship('Authors', secondary=books_authors_link, backref='books') tags = relationship('Tags', secondary=books_tags_link, backref='books',order_by="Tags.name") @@ -310,131 +330,324 @@ class Custom_Columns(Base): return display_dict -def update_title_sort(config, conn=None): - # user defined sort function for calibre databases (Series, etc.) - def _title_sort(title): - # calibre sort stuff - title_pat = re.compile(config.config_title_regex, re.IGNORECASE) - match = title_pat.search(title) - if match: - prep = match.group(1) - title = title[len(prep):] + ', ' + prep - return title.strip() +class CalibreDB(threading.Thread): - conn = conn or session.connection().connection.connection - conn.create_function("title_sort", 1, _title_sort) + def __init__(self): + threading.Thread.__init__(self) + self.engine = None + self.session = None + self.queue = None + self.log = None + self.config = None + + def add_queue(self,queue): + self.queue = queue + self.log = logger.create() + + def run(self): + while True: + i = self.queue.get() + if i == 'dummy': + self.queue.task_done() + break + if i['task'] == 'add_format': + cur_book = self.session.query(Books).filter(Books.id == i['id']).first() + cur_book.data.append(i['format']) + try: + # db.session.merge(cur_book) + self.session.commit() + except OperationalError as e: + self.session.rollback() + self.log.error("Database error: %s", e) + # self._handleError(_(u"Database error: %(error)s.", error=e)) + # return + self.queue.task_done() -def setup_db(config): - dispose() - global engine + def stop(self): + self.queue.put('dummy') - if not config.config_calibre_dir: - config.invalidate() - return False + def setup_db(self, config, app_db_path): + self.config = config + self.dispose() + # global engine - dbpath = os.path.join(config.config_calibre_dir, "metadata.db") - if not os.path.exists(dbpath): - config.invalidate() - return False + if not config.config_calibre_dir: + config.invalidate() + return False - try: - engine = create_engine('sqlite:///{0}'.format(dbpath), - echo=False, - isolation_level="SERIALIZABLE", - connect_args={'check_same_thread': False}) - conn = engine.connect() - # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 - except Exception as e: - config.invalidate(e) - return False + dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + if not os.path.exists(dbpath): + config.invalidate() + return False - config.db_configured = True - update_title_sort(config, conn.connection) + try: + #engine = create_engine('sqlite:///{0}'.format(dbpath), + self.engine = create_engine('sqlite://', + echo=False, + isolation_level="SERIALIZABLE", + connect_args={'check_same_thread': False}) + self.engine.execute("attach database '{}' as calibre;".format(dbpath)) + self.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) - if not cc_classes: - cc = conn.execute("SELECT id, datatype FROM custom_columns") + conn = self.engine.connect() + # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 + except Exception as e: + config.invalidate(e) + return False - cc_ids = [] - books_custom_column_links = {} - for row in cc: - if row.datatype not in cc_exceptions: - books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), - primary_key=True), - Column('value', Integer, - ForeignKey('custom_column_' + str(row.id) + '.id'), - primary_key=True) - ) - cc_ids.append([row.id, row.datatype]) - if row.datatype == 'bool': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Boolean)} - elif row.datatype == 'int': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Integer)} - elif row.datatype == 'float': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Float)} + config.db_configured = True + self.update_title_sort(config, conn.connection) + + if not cc_classes: + cc = conn.execute("SELECT id, datatype FROM custom_columns") + + cc_ids = [] + books_custom_column_links = {} + for row in cc: + if row.datatype not in cc_exceptions: + books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, + Column('book', Integer, ForeignKey('books.id'), + primary_key=True), + Column('value', Integer, + ForeignKey('custom_column_' + str(row.id) + '.id'), + primary_key=True) + ) + cc_ids.append([row.id, row.datatype]) + if row.datatype == 'bool': + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(Boolean)} + elif row.datatype == 'int': + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(Integer)} + elif row.datatype == 'float': + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(Float)} + else: + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'value': Column(String)} + cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict) + + for cc_id in cc_ids: + if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'): + setattr(Books, + 'custom_column_' + str(cc_id[0]), + relationship(cc_classes[cc_id[0]], + primaryjoin=( + Books.id == cc_classes[cc_id[0]].book), + backref='books')) else: - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'value': Column(String)} - cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict) + setattr(Books, + 'custom_column_' + str(cc_id[0]), + relationship(cc_classes[cc_id[0]], + secondary=books_custom_column_links[cc_id[0]], + backref='books')) - for cc_id in cc_ids: - if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'): - setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], - primaryjoin=( - Books.id == cc_classes[cc_id[0]].book), - backref='books')) - else: - setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], - secondary=books_custom_column_links[cc_id[0]], - backref='books')) + Session = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=self.engine)) + self.session = Session() + return True + def get_book(self, book_id): + return self.session.query(Books).filter(Books.id == book_id).first() - global session - Session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=engine)) - session = Session() - return True + def get_filtered_book(self, book_id, allow_show_archived=False): + return self.session.query(Books).filter(Books.id == book_id).\ + filter(self.common_filters(allow_show_archived)).first() + def get_book_by_uuid(self, book_uuid): + return self.session.query(Books).filter(Books.uuid == book_uuid).first() -def dispose(): - global session + def get_book_format(self, book_id, format): + return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == format).first() - old_session = session - session = None - if old_session: - try: old_session.close() - except: pass - if old_session.bind: - try: old_session.bind.dispose() - except Exception: pass + # Language and content filters for displaying in the UI + def common_filters(self, allow_show_archived=False): + if not allow_show_archived: + archived_books = ( + ub.session.query(ub.ArchivedBook) + .filter(ub.ArchivedBook.user_id == int(current_user.id)) + .filter(ub.ArchivedBook.is_archived == True) + .all() + ) + archived_book_ids = [archived_book.book_id for archived_book in archived_books] + archived_filter = Books.id.notin_(archived_book_ids) + else: + archived_filter = true() - for attr in list(Books.__dict__.keys()): - if attr.startswith("custom_column_"): - setattr(Books, attr, None) + if current_user.filter_language() != "all": + lang_filter = Books.languages.any(Languages.lang_code == current_user.filter_language()) + else: + lang_filter = true() + negtags_list = current_user.list_denied_tags() + postags_list = current_user.list_allowed_tags() + neg_content_tags_filter = false() if negtags_list == [''] else Books.tags.any(Tags.name.in_(negtags_list)) + pos_content_tags_filter = true() if postags_list == [''] else Books.tags.any(Tags.name.in_(postags_list)) + if self.config.config_restricted_column: + pos_cc_list = current_user.allowed_column_value.split(',') + pos_content_cc_filter = true() if pos_cc_list == [''] else \ + getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \ + any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list)) + neg_cc_list = current_user.denied_column_value.split(',') + neg_content_cc_filter = false() if neg_cc_list == [''] else \ + getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \ + any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list)) + else: + pos_content_cc_filter = true() + neg_content_cc_filter = false() + return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter, + pos_content_cc_filter, ~neg_content_cc_filter, archived_filter) - for db_class in cc_classes.values(): - Base.metadata.remove(db_class.__table__) - cc_classes.clear() + # Fill indexpage with all requested data from database + def fill_indexpage(self, page, database, db_filter, order, *join): + return self.fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join) - for table in reversed(Base.metadata.sorted_tables): - name = table.key - if name.startswith("custom_column_") or name.startswith("books_custom_column_"): - if table is not None: - Base.metadata.remove(table) + def fill_indexpage_with_archived_books(self, page, database, db_filter, order, allow_show_archived, *join): + if current_user.show_detail_random(): + randm = self.session.query(Books) \ + .filter(self.common_filters(allow_show_archived)) \ + .order_by(func.random()) \ + .limit(self.config.config_random_books) + else: + randm = false() + off = int(int(self.config.config_books_per_page) * (page - 1)) + query = self.session.query(database) \ + .join(*join, isouter=True) \ + .filter(db_filter) \ + .filter(self.common_filters(allow_show_archived)) + pagination = Pagination(page, self.config.config_books_per_page, + len(query.all())) + entries = query.order_by(*order).offset(off).limit(self.config.config_books_per_page).all() + for book in entries: + book = self.order_authors(book) + return entries, randm, pagination -def reconnect_db(config): - session.close() - engine.dispose() - setup_db(config) + # Orders all Authors in the list according to authors sort + def order_authors(self, entry): + sort_authors = entry.author_sort.split('&') + authors_ordered = list() + error = False + for auth in sort_authors: + # ToDo: How to handle not found authorname + result = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).first() + if not result: + error = True + break + authors_ordered.append(result) + if not error: + entry.authors = authors_ordered + return entry + + def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()): + query = query or '' + self.session.connection().connection.connection.create_function("lower", 1, lcase) + entries = self.session.query(database).filter(tag_filter). \ + filter(func.lower(database.name).ilike("%" + query + "%")).all() + json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries]) + return json_dumps + + def check_exists_book(self, authr, title): + self.session.connection().connection.connection.create_function("lower", 1, lcase) + q = list() + authorterms = re.split(r'\s*&\s*', authr) + for authorterm in authorterms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) + + return self.session.query(Books)\ + .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() + + # read search results from calibre-database and return it (function is used for feed and simple search + def get_search_results(self, term): + term.strip().lower() + self.session.connection().connection.connection.create_function("lower", 1, lcase) + q = list() + authorterms = re.split("[, ]+", term) + for authorterm in authorterms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) + + return self.session.query(Books).filter(self.common_filters()).filter( + or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), + Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), + Books.authors.any(and_(*q)), + Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), + func.lower(Books.title).ilike("%" + term + "%") + )).order_by(Books.sort).all() + + # Creates for all stored languages a translated speaking name in the array for the UI + def speaking_language(self, languages=None): + from . import get_locale + + if not languages: + languages = self.session.query(Languages) \ + .join(books_languages_link) \ + .join(Books) \ + .filter(self.common_filters()) \ + .group_by(text('books_languages_link.lang_code')).all() + for lang in languages: + try: + cur_l = LC.parse(lang.lang_code) + lang.name = cur_l.get_language_name(get_locale()) + except UnknownLocaleError: + lang.name = _(isoLanguages.get(part3=lang.lang_code).name) + return languages + + def update_title_sort(self, config, conn=None): + # user defined sort function for calibre databases (Series, etc.) + def _title_sort(title): + # calibre sort stuff + title_pat = re.compile(config.config_title_regex, re.IGNORECASE) + match = title_pat.search(title) + if match: + prep = match.group(1) + title = title[len(prep):] + ', ' + prep + return title.strip() + + conn = conn or self.session.connection().connection.connection + conn.create_function("title_sort", 1, _title_sort) + + def dispose(self): + # global session + + old_session = self.session + self.session = None + if old_session: + try: old_session.close() + except: pass + if old_session.bind: + try: old_session.bind.dispose() + except Exception: pass + + for attr in list(Books.__dict__.keys()): + if attr.startswith("custom_column_"): + setattr(Books, attr, None) + + for db_class in cc_classes.values(): + Base.metadata.remove(db_class.__table__) + cc_classes.clear() + + for table in reversed(Base.metadata.sorted_tables): + name = table.key + if name.startswith("custom_column_") or name.startswith("books_custom_column_"): + if table is not None: + Base.metadata.remove(table) + + def reconnect_db(self, config, app_db_path): + self.session.close() + self.engine.dispose() + self.setup_db(config, app_db_path) + +def lcase(s): + try: + return unidecode.unidecode(s.lower()) + except Exception as e: + log = logger.create() + log.exception(e) + return s.lower() diff --git a/cps/editbooks.py b/cps/editbooks.py index c7321a2a..e848c8e2 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -24,16 +24,17 @@ from __future__ import division, print_function, unicode_literals import os from datetime import datetime import json -from shutil import move, copyfile +from shutil import copyfile from uuid import uuid4 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response from flask_babel import gettext as _ from flask_login import current_user, login_required +from sqlalchemy.exc import OperationalError from . import constants, logger, isoLanguages, gdriveutils, uploader, helper -from . import config, get_locale, db, ub, worker -from .helper import order_authors, common_filters +from . import config, get_locale, ub, worker, db +from . import calibre_db from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required @@ -174,13 +175,15 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session): @login_required def delete_book(book_id, book_format): if current_user.role_delete_books(): - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + book = calibre_db.get_book(book_id) if book: try: result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper()) if not result: flash(error, category="error") return redirect(url_for('editbook.edit_book', book_id=book_id)) + if error: + flash(error, category="warning") if not book_format: # delete book from Shelfs, Downloads, Read list ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete() @@ -190,13 +193,13 @@ def delete_book(book_id, book_format): # check if only this book links to: # author, language, series, tags, custom columns - modify_database_object([u''], book.authors, db.Authors, db.session, 'author') - modify_database_object([u''], book.tags, db.Tags, db.session, 'tags') - modify_database_object([u''], book.series, db.Series, db.session, 'series') - modify_database_object([u''], book.languages, db.Languages, db.session, 'languages') - modify_database_object([u''], book.publishers, db.Publishers, db.session, 'publishers') + modify_database_object([u''], book.authors, db.Authors, calibre_db.session, 'author') + modify_database_object([u''], book.tags, db.Tags, calibre_db.session, 'tags') + modify_database_object([u''], book.series, db.Series, calibre_db.session, 'series') + modify_database_object([u''], book.languages, db.Languages, calibre_db.session, 'languages') + modify_database_object([u''], book.publishers, db.Publishers, calibre_db.session, 'publishers') - cc = db.session.query(db.Custom_Columns).\ + cc = calibre_db.session.query(db.Custom_Columns).\ filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() for c in cc: cc_string = "custom_column_" + str(c.id) @@ -206,32 +209,32 @@ def delete_book(book_id, book_format): del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) log.debug('remove ' + str(c.id)) - db.session.delete(del_cc) - db.session.commit() + calibre_db.session.delete(del_cc) + calibre_db.session.commit() elif c.datatype == 'rating': del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) if len(del_cc.books) == 0: log.debug('remove ' + str(c.id)) - db.session.delete(del_cc) - db.session.commit() + calibre_db.session.delete(del_cc) + calibre_db.session.commit() else: del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) log.debug('remove ' + str(c.id)) - db.session.delete(del_cc) - db.session.commit() + calibre_db.session.delete(del_cc) + calibre_db.session.commit() else: modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id], - db.session, 'custom') - db.session.query(db.Books).filter(db.Books.id == book_id).delete() + calibre_db.session, 'custom') + calibre_db.session.query(db.Books).filter(db.Books.id == book_id).delete() else: - db.session.query(db.Data).filter(db.Data.book == book.id).\ + calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\ filter(db.Data.format == book_format).delete() - db.session.commit() + calibre_db.session.commit() except Exception as e: log.debug(e) - db.session.rollback() + calibre_db.session.rollback() else: # book not found log.error('Book with id "%s" could not be deleted: not found', book_id) @@ -244,11 +247,9 @@ def delete_book(book_id, book_format): def render_edit_book(book_id): - db.update_title_sort(config) - cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() - book = db.session.query(db.Books)\ - .filter(db.Books.id == book_id).filter(common_filters()).first() - + calibre_db.update_title_sort(config) + cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() + book = calibre_db.get_filtered_book(book_id) if not book: flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error") return redirect(url_for("web.index")) @@ -256,7 +257,7 @@ def render_edit_book(book_id): for lang in book.languages: lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code) - book = order_authors(book) + book = calibre_db.order_authors(book) author_names = [] for authr in book.authors: @@ -264,19 +265,25 @@ def render_edit_book(book_id): # Option for showing convertbook button valid_source_formats=list() - if config.config_ebookconverter == 2: + allowed_conversion_formats = list() + kepub_possible=None + if config.config_converterpath: for file in book.data: if file.format.lower() in constants.EXTENSIONS_CONVERT: valid_source_formats.append(file.format.lower()) + if config.config_kepubifypath and 'epub' in [file.format.lower() for file in book.data]: + kepub_possible = True + if not config.config_converterpath: + valid_source_formats.append('epub') # Determine what formats don't already exist - allowed_conversion_formats = constants.EXTENSIONS_CONVERT.copy() - for file in book.data: - try: - allowed_conversion_formats.remove(file.format.lower()) - except Exception: - log.warning('%s already removed from list.', file.format.lower()) - + if config.config_converterpath: + allowed_conversion_formats = constants.EXTENSIONS_CONVERT.copy() + for file in book.data: + if file.format.lower() in allowed_conversion_formats: + allowed_conversion_formats.remove(file.format.lower()) + if kepub_possible: + allowed_conversion_formats.append('kepub') return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc, title=_(u"edit metadata"), page="editbook", conversion_formats=allowed_conversion_formats, @@ -293,7 +300,7 @@ def edit_book_ratings(to_save, book): ratingx2 = int(float(to_save["rating"]) * 2) if ratingx2 != old_rating: changed = True - is_rating = db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first() + is_rating = calibre_db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first() if is_rating: book.ratings.append(is_rating) else: @@ -307,15 +314,59 @@ def edit_book_ratings(to_save, book): changed = True return changed +def edit_book_tags(tags, book): + input_tags = tags.split(',') + input_tags = list(map(lambda it: it.strip(), input_tags)) + # if input_tags[0] !="": ?? + return modify_database_object(input_tags, book.tags, db.Tags, calibre_db.session, 'tags') -def edit_book_languages(to_save, book): - input_languages = to_save["languages"].split(',') + +def edit_book_series(series, book): + input_series = [series.strip()] + input_series = [x for x in input_series if x != ''] + return modify_database_object(input_series, book.series, db.Series, calibre_db.session, 'series') + + +def edit_book_series_index(series_index, book): + # Add default series_index to book + modif_date = False + series_index = series_index or '1' + #if series_index == '': + # series_index = '1' + if book.series_index != series_index: + book.series_index = series_index + modif_date = True + return modif_date + +# Handle book comments/description +def edit_book_comments(comments, book): + modif_date = False + if len(book.comments): + if book.comments[0].text != comments: + book.comments[0].text = comments + modif_date = True + else: + if comments: + book.comments.append(db.Comments(text=comments, book=book.id)) + modif_date = True + return modif_date + + +def edit_book_languages(languages, book, upload=False): + input_languages = languages.split(',') unknown_languages = [] input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages) for l in unknown_languages: log.error('%s is not a valid language', l) - flash(_(u"%(langname)s is not a valid language", langname=l), category="error") - return modify_database_object(list(input_l), book.languages, db.Languages, db.session, 'languages') + flash(_(u"%(langname)s is not a valid language", langname=l), category="warning") + # ToDo: Not working correct + if upload and len(input_l) == 1: + # If the language of the file is excluded from the users view, it's not imported, to allow the user to view + # the book it's language is set to the filter language + if input_l[0] != current_user.filter_language() and current_user.filter_language() != "all": + input_l[0] = calibre_db.session.query(db.Languages). \ + filter(db.Languages.lang_code == current_user.filter_language()).first() + return modify_database_object(input_l, book.languages, db.Languages, calibre_db.session, 'languages') def edit_book_publisher(to_save, book): @@ -323,15 +374,15 @@ def edit_book_publisher(to_save, book): if to_save["publisher"]: publisher = to_save["publisher"].rstrip().strip() if len(book.publishers) == 0 or (len(book.publishers) > 0 and publisher != book.publishers[0].name): - changed |= modify_database_object([publisher], book.publishers, db.Publishers, db.session, 'publisher') + changed |= modify_database_object([publisher], book.publishers, db.Publishers, calibre_db.session, 'publisher') elif len(book.publishers): - changed |= modify_database_object([], book.publishers, db.Publishers, db.session, 'publisher') + changed |= modify_database_object([], book.publishers, db.Publishers, calibre_db.session, 'publisher') return changed def edit_cc_data(book_id, book, to_save): changed = False - cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() + cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() for c in cc: cc_string = "custom_column_" + str(c.id) if not c.is_multiple: @@ -354,12 +405,12 @@ def edit_cc_data(book_id, book, to_save): else: del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) - db.session.delete(del_cc) + calibre_db.session.delete(del_cc) changed = True else: cc_class = db.cc_classes[c.id] new_cc = cc_class(value=to_save[cc_string], book=book_id) - db.session.add(new_cc) + calibre_db.session.add(new_cc) changed = True else: @@ -371,18 +422,18 @@ def edit_cc_data(book_id, book, to_save): del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) if len(del_cc.books) == 0: - db.session.delete(del_cc) + calibre_db.session.delete(del_cc) changed = True cc_class = db.cc_classes[c.id] - new_cc = db.session.query(cc_class).filter( + new_cc = calibre_db.session.query(cc_class).filter( cc_class.value == to_save[cc_string].strip()).first() # if no cc val is found add it if new_cc is None: new_cc = cc_class(value=to_save[cc_string].strip()) - db.session.add(new_cc) + calibre_db.session.add(new_cc) changed = True - db.session.flush() - new_cc = db.session.query(cc_class).filter( + calibre_db.session.flush() + new_cc = calibre_db.session.query(cc_class).filter( cc_class.value == to_save[cc_string].strip()).first() # add cc value to book getattr(book, cc_string).append(new_cc) @@ -392,13 +443,16 @@ def edit_cc_data(book_id, book, to_save): del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) if not del_cc.books or len(del_cc.books) == 0: - db.session.delete(del_cc) + calibre_db.session.delete(del_cc) changed = True else: input_tags = to_save[cc_string].split(',') input_tags = list(map(lambda it: it.strip(), input_tags)) - changed |= modify_database_object(input_tags, getattr(book, cc_string), db.cc_classes[c.id], db.session, - 'custom') + changed |= modify_database_object(input_tags, + getattr(book, cc_string), + db.cc_classes[c.id], + calibre_db.session, + 'custom') return changed def upload_single_file(request, book, book_id): @@ -435,17 +489,22 @@ def upload_single_file(request, book, book_id): return redirect(url_for('web.show_book', book_id=book.id)) file_size = os.path.getsize(saved_filename) - is_format = db.session.query(db.Data).filter(db.Data.book == book_id).\ - filter(db.Data.format == file_ext.upper()).first() + is_format = calibre_db.get_book_format(book_id, file_ext.upper()) # Format entry already exists, no need to update the database if is_format: log.warning('Book format %s already existing', file_ext.upper()) else: - db_format = db.Data(book_id, file_ext.upper(), file_size, file_name) - db.session.add(db_format) - db.session.commit() - db.update_title_sort(config) + try: + db_format = db.Data(book_id, file_ext.upper(), file_size, file_name) + calibre_db.session.add(db_format) + calibre_db.session.commit() + calibre_db.update_title_sort(config) + except OperationalError as e: + calibre_db.session.rollback() + log.error('Database error: %s', e) + flash(_(u"Database error: %(error)s.", error=e), category="error") + return redirect(url_for('web.show_book', book_id=book.id)) # Queue uploader info uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title) @@ -454,7 +513,7 @@ def upload_single_file(request, book, book_id): return uploader.process( saved_filename, *os.path.splitext(requested_file.filename), - rarExcecutable=config.config_rarfile_location) + rarExecutable=config.config_rarfile_location) def upload_cover(request, book): @@ -481,9 +540,8 @@ def edit_book(book_id): return render_edit_book(book_id) # create the function for sorting... - db.update_title_sort(config) - book = db.session.query(db.Books)\ - .filter(db.Books.id == book_id).filter(common_filters()).first() + calibre_db.update_title_sort(config) + book = calibre_db.get_filtered_book(book_id) # Book not found if not book: @@ -514,13 +572,13 @@ def edit_book(book_id): if input_authors == ['']: input_authors = [_(u'Unknown')] # prevent empty Author - modif_date |= modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author') + modif_date |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author') # Search for each author if author is in database, if not, authorname and sorted authorname is generated new # everything then is assembled for sorted author field in database sort_authors_list = list() for inp in input_authors: - stored_author = db.session.query(db.Authors).filter(db.Authors.name == inp).first() + stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first() if not stored_author: stored_author = helper.get_sorted_author(inp) else: @@ -548,33 +606,21 @@ def edit_book(book_id): else: flash(error, category="error") - if book.series_index != to_save["series_index"]: - book.series_index = to_save["series_index"] - modif_date = True + # Add default series_index to book + modif_date |= edit_book_series_index(to_save["series_index"], book) # Handle book comments/description - if len(book.comments): - if book.comments[0].text != to_save["description"]: - book.comments[0].text = to_save["description"] - modif_date = True - else: - if to_save["description"]: - book.comments.append(db.Comments(text=to_save["description"], book=book.id)) - modif_date = True + modif_date |= edit_book_comments(to_save["description"], book) # Handle identifiers input_identifiers = identifier_list(to_save, book) - modif_date |= modify_identifiers(input_identifiers, book.identifiers, db.session) + modif_date |= modify_identifiers(input_identifiers, book.identifiers, calibre_db.session) # Handle book tags - input_tags = to_save["tags"].split(',') - input_tags = list(map(lambda it: it.strip(), input_tags)) - modif_date |= modify_database_object(input_tags, book.tags, db.Tags, db.session, 'tags') + modif_date |= edit_book_tags(to_save['tags'], book) # Handle book series - input_series = [to_save["series"].strip()] - input_series = [x for x in input_series if x != ''] - modif_date |= modify_database_object(input_series, book.series, db.Series, db.session, 'series') + modif_date |= edit_book_series(to_save["series"], book) if to_save["pubdate"]: try: @@ -588,7 +634,7 @@ def edit_book(book_id): modif_date |= edit_book_publisher(to_save, book) # handle book languages - modif_date |= edit_book_languages(to_save, book) + modif_date |= edit_book_languages(to_save['languages'], book) # handle book ratings modif_date |= edit_book_ratings(to_save, book) @@ -598,7 +644,7 @@ def edit_book(book_id): if modif_date: book.last_modified = datetime.utcnow() - db.session.commit() + calibre_db.session.commit() if config.config_use_google_drive: gdriveutils.updateGdriveCalibreFromLocal() if "detail_view" in to_save: @@ -607,12 +653,12 @@ def edit_book(book_id): flash(_("Metadata successfully updated"), category="success") return render_edit_book(book_id) else: - db.session.rollback() + calibre_db.session.rollback() flash(error, category="error") return render_edit_book(book_id) except Exception as e: log.exception(e) - db.session.rollback() + calibre_db.session.rollback() flash(_("Error editing book, please check logfile for details"), category="error") return redirect(url_for('web.show_book', book_id=book.id)) @@ -652,179 +698,172 @@ def upload(): abort(404) if request.method == 'POST' and 'btn-upload' in request.files: for requested_file in request.files.getlist("btn-upload"): - # create the function for sorting... - db.update_title_sort(config) - db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) - - # check if file extension is correct - if '.' in requested_file.filename: - file_ext = requested_file.filename.rsplit('.', 1)[-1].lower() - if file_ext not in constants.EXTENSIONS_UPLOAD: - flash( - _("File extension '%(ext)s' is not allowed to be uploaded to this server", - ext=file_ext), category="error") - return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - else: - flash(_('File to be uploaded must have an extension'), category="error") - return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - - # extract metadata from file try: - meta = uploader.upload(requested_file, config.config_rarfile_location) - except (IOError, OSError): - log.error("File %s could not saved to temp dir", requested_file.filename) - flash(_(u"File %(filename)s could not saved to temp dir", - filename= requested_file.filename), category="error") - return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - title = meta.title - authr = meta.author - tags = meta.tags - series = meta.series - series_index = meta.series_id - title_dir = helper.get_valid_filename(title) - author_dir = helper.get_valid_filename(authr) - filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir) - saved_filename = os.path.join(filepath, title_dir + meta.extension.lower()) + modif_date = False + # create the function for sorting... + calibre_db.update_title_sort(config) + calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) - if title != _(u'Unknown') and authr != _(u'Unknown'): - entry = helper.check_exists_book(authr, title) - if entry: - log.info("Uploaded book probably exists in library") - flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ") - + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning") - - # check if file path exists, otherwise create it, copy file to calibre path and delete temp file - if not os.path.exists(filepath): - try: - os.makedirs(filepath) - except OSError: - log.error("Failed to create path %s (Permission denied)", filepath) - flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error") + # check if file extension is correct + if '.' in requested_file.filename: + file_ext = requested_file.filename.rsplit('.', 1)[-1].lower() + if file_ext not in constants.EXTENSIONS_UPLOAD: + flash( + _("File extension '%(ext)s' is not allowed to be uploaded to this server", + ext=file_ext), category="error") + return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + else: + flash(_('File to be uploaded must have an extension'), category="error") return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - try: - copyfile(meta.file_path, saved_filename) - os.unlink(meta.file_path) - except OSError as e: - log.error("Failed to move file %s: %s", saved_filename, e) - flash(_(u"Failed to Move File %(file)s: %(error)s", file=saved_filename, error=e), category="error") - return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - if meta.cover is None: - has_cover = 0 - copyfile(os.path.join(constants.STATIC_DIR, 'generic_cover.jpg'), - os.path.join(filepath, "cover.jpg")) - else: - has_cover = 1 + # extract metadata from file try: - copyfile(meta.cover, os.path.join(filepath, "cover.jpg")) - os.unlink(meta.cover) + meta = uploader.upload(requested_file, config.config_rarfile_location) + except (IOError, OSError): + log.error("File %s could not saved to temp dir", requested_file.filename) + flash(_(u"File %(filename)s could not saved to temp dir", + filename= requested_file.filename), category="error") + return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + title = meta.title + authr = meta.author + + if title != _(u'Unknown') and authr != _(u'Unknown'): + entry = calibre_db.check_exists_book(authr, title) + if entry: + log.info("Uploaded book probably exists in library") + flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ") + + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning") + + # handle authors + input_authors = authr.split('&') + # handle_authors(input_authors) + input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors)) + # we have all author names now + if input_authors == ['']: + input_authors = [_(u'Unknown')] # prevent empty Author + + sort_authors_list=list() + db_author = None + for inp in input_authors: + stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first() + if not stored_author: + if not db_author: + db_author = db.Authors(inp, helper.get_sorted_author(inp), "") + calibre_db.session.add(db_author) + calibre_db.session.commit() + sort_author = helper.get_sorted_author(inp) + else: + if not db_author: + db_author = stored_author + sort_author = stored_author.sort + sort_authors_list.append(sort_author) # helper.get_sorted_author(sort_author)) + sort_authors = ' & '.join(sort_authors_list) + + title_dir = helper.get_valid_filename(title) + author_dir = helper.get_valid_filename(db_author.name) + filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir) + saved_filename = os.path.join(filepath, title_dir + meta.extension.lower()) + + # check if file path exists, otherwise create it, copy file to calibre path and delete temp file + if not os.path.exists(filepath): + try: + os.makedirs(filepath) + except OSError: + log.error("Failed to create path %s (Permission denied)", filepath) + flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error") + return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') + try: + copyfile(meta.file_path, saved_filename) + os.unlink(meta.file_path) except OSError as e: - log.error("Failed to move cover file %s: %s", os.path.join(filepath, "cover.jpg"), e) - flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=os.path.join(filepath, "cover.jpg"), - error=e), - category="error") + log.error("Failed to move file %s: %s", saved_filename, e) + flash(_(u"Failed to Move File %(file)s: %(error)s", file=saved_filename, error=e), category="error") return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - # handle authors - is_author = db.session.query(db.Authors).filter(db.Authors.name == authr).first() - if is_author: - db_author = is_author - else: - db_author = db.Authors(authr, helper.get_sorted_author(authr), "") - db.session.add(db_author) - - # handle series - db_series = None - is_series = db.session.query(db.Series).filter(db.Series.name == series).first() - if is_series: - db_series = is_series - elif series != '': - db_series = db.Series(series, "") - db.session.add(db_series) - - # add language actually one value in list - input_language = meta.languages - db_language = None - if input_language != "": - input_language = isoLanguages.get(name=input_language).part3 - hasLanguage = db.session.query(db.Languages).filter(db.Languages.lang_code == input_language).first() - if hasLanguage: - db_language = hasLanguage + if meta.cover is None: + has_cover = 0 + copyfile(os.path.join(constants.STATIC_DIR, 'generic_cover.jpg'), + os.path.join(filepath, "cover.jpg")) else: - db_language = db.Languages(input_language) - db.session.add(db_language) + has_cover = 1 - # If the language of the file is excluded from the users view, it's not imported, to allow the user to view - # the book it's language is set to the filter language - if db_language != current_user.filter_language() and current_user.filter_language() != "all": - db_language = db.session.query(db.Languages).\ - filter(db.Languages.lang_code == current_user.filter_language()).first() + # combine path and normalize path from windows systems + path = os.path.join(author_dir, title_dir).replace('\\', '/') + # Calibre adds books with utc as timezone + db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1), + '1', datetime.utcnow(), path, has_cover, db_author, [], "") - # combine path and normalize path from windows systems - path = os.path.join(author_dir, title_dir).replace('\\', '/') - # Calibre adds books with utc as timezone - db_book = db.Books(title, "", db_author.sort, datetime.utcnow(), datetime(101, 1, 1), - series_index, datetime.utcnow(), path, has_cover, db_author, [], db_language) - db_book.authors.append(db_author) - if db_series: - db_book.series.append(db_series) - if db_language is not None: - db_book.languages.append(db_language) - file_size = os.path.getsize(saved_filename) - db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir) + modif_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session, + 'author') - # handle tags - input_tags = tags.split(',') - input_tags = list(map(lambda it: it.strip(), input_tags)) - if input_tags[0] !="": - modify_database_object(input_tags, db_book.tags, db.Tags, db.session, 'tags') + # Add series_index to book + modif_date |= edit_book_series_index(meta.series_id, db_book) - # flush content, get db_book.id available - db_book.data.append(db_data) - db.session.add(db_book) - db.session.flush() + # add languages + modif_date |= edit_book_languages(meta.languages, db_book, upload=True) - # add comment - book_id = db_book.id - upload_comment = Markup(meta.description).unescape() - if upload_comment != "": - db.session.add(db.Comments(upload_comment, book_id)) + # handle tags + modif_date |= edit_book_tags(meta.tags, db_book) - # save data to database, reread data - db.session.commit() - db.update_title_sort(config) - # Reread book. It's important not to filter the result, as it could have language which hide it from - # current users view (tags are not stored/extracted from metadata and could also be limited) - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - # upload book to gdrive if nesseccary and add "(bookid)" to folder name - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - error = helper.update_dir_stucture(book.id, config.config_calibre_dir) - db.session.commit() - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - if error: - flash(error, category="error") - uploadText=_(u"File %(file)s uploaded", file=book.title) - worker.add_upload(current_user.nickname, - "" + uploadText + "") + # handle series + modif_date |= edit_book_series(meta.series, db_book) - # create data for displaying display Full language name instead of iso639.part3language - if db_language is not None: - book.languages[0].language_name = _(meta.languages) - author_names = [] - for author in db_book.authors: - author_names.append(author.name) - if len(request.files.getlist("btn-upload")) < 2: - if current_user.role_edit() or current_user.role_admin(): - resp = {"location": url_for('editbook.edit_book', book_id=db_book.id)} - return Response(json.dumps(resp), mimetype='application/json') - else: - resp = {"location": url_for('web.show_book', book_id=db_book.id)} - return Response(json.dumps(resp), mimetype='application/json') + # Add file to book + file_size = os.path.getsize(saved_filename) + db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir) + db_book.data.append(db_data) + calibre_db.session.add(db_book) + + # flush content, get db_book.id available + calibre_db.session.flush() + + # Comments needs book id therfore only possiblw after flush + modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book) + + book_id = db_book.id + title = db_book.title + + error = helper.update_dir_stucture(book_id, config.config_calibre_dir, input_authors[0]) + + # move cover to final directory, including book id + if has_cover: + try: + new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg") + copyfile(meta.cover, new_coverpath) + os.unlink(meta.cover) + except OSError as e: + log.error("Failed to move cover file %s: %s", new_coverpath, e) + flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath, + error=e), + category="error") + + # save data to database, reread data + calibre_db.session.commit() + #calibre_db.setup_db(config, ub.app_DB_path) + # Reread book. It's important not to filter the result, as it could have language which hide it from + # current users view (tags are not stored/extracted from metadata and could also be limited) + #book = calibre_db.get_book(book_id) + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + if error: + flash(error, category="error") + uploadText=_(u"File %(file)s uploaded", file=title) + worker.add_upload(current_user.nickname, + "" + uploadText + "") + + if len(request.files.getlist("btn-upload")) < 2: + if current_user.role_edit() or current_user.role_admin(): + resp = {"location": url_for('editbook.edit_book', book_id=book_id)} + return Response(json.dumps(resp), mimetype='application/json') + else: + resp = {"location": url_for('web.show_book', book_id=book_id)} + return Response(json.dumps(resp), mimetype='application/json') + except OperationalError as e: + calibre_db.session.rollback() + log.error("Database error: %s", e) + flash(_(u"Database error: %(error)s.", error=e), category="error") return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') - @editbook.route("/admin/book/convert/", methods=['POST']) @login_required_if_no_ano @edit_required diff --git a/cps/gdrive.py b/cps/gdrive.py index 23a36a91..aa3743d2 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -39,7 +39,7 @@ try: except ImportError: pass -from . import logger, gdriveutils, config, db +from . import logger, gdriveutils, config, ub, calibre_db from .web import admin_required @@ -145,7 +145,8 @@ def on_received_watch_confirmation(): dbpath = os.path.join(config.config_calibre_dir, "metadata.db") else: dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() - if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != hashlib.md5(dbpath): + if not response['deleted'] and response['file']['title'] == 'metadata.db' \ + and response['file']['md5Checksum'] != hashlib.md5(dbpath): tmpDir = tempfile.gettempdir() log.info('Database file updated') copyfile(dbpath, os.path.join(tmpDir, "metadata.db_" + str(current_milli_time()))) @@ -154,7 +155,7 @@ def on_received_watch_confirmation(): log.info('Setting up new DB') # prevent error on windows, as os.rename does on exisiting files move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath) - db.setup_db(config) + calibre_db.setup_db(config, ub.app_DB_path) except Exception as e: log.exception(e) updateMetaData() diff --git a/cps/helper.py b/cps/helper.py index 7858633a..a4715033 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -23,7 +23,6 @@ import os import io import json import mimetypes -import random import re import shutil import time @@ -42,6 +41,7 @@ from flask_login import current_user from sqlalchemy.sql.expression import true, false, and_, or_, text, func from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash +from . import calibre_db try: from urllib.parse import quote @@ -74,8 +74,8 @@ log = logger.create() # Convert existing book entry to new format def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None): - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first() + book = calibre_db.get_book(book_id) + data = calibre_db.get_book_format(book.id, old_book_format) if not data: error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) log.error("convert_book_format: %s", error_message) @@ -142,25 +142,27 @@ def check_send_to_kindle(entry): """ if len(entry.data): bookformats = list() - if config.config_ebookconverter == 0: + if not config.config_converterpath: # no converter - only for mobi and pdf formats for ele in iter(entry.data): - if 'MOBI' in ele.format: - bookformats.append({'format': 'Mobi', - 'convert': 0, - 'text': _('Send %(format)s to Kindle', format='Mobi')}) - if 'PDF' in ele.format: - bookformats.append({'format': 'Pdf', - 'convert': 0, - 'text': _('Send %(format)s to Kindle', format='Pdf')}) - if 'AZW' in ele.format: - bookformats.append({'format': 'Azw', - 'convert': 0, - 'text': _('Send %(format)s to Kindle', format='Azw')}) + if ele.uncompressed_size < config.mail_size: + if 'MOBI' in ele.format: + bookformats.append({'format': 'Mobi', + 'convert': 0, + 'text': _('Send %(format)s to Kindle', format='Mobi')}) + if 'PDF' in ele.format: + bookformats.append({'format': 'Pdf', + 'convert': 0, + 'text': _('Send %(format)s to Kindle', format='Pdf')}) + if 'AZW' in ele.format: + bookformats.append({'format': 'Azw', + 'convert': 0, + 'text': _('Send %(format)s to Kindle', format='Azw')}) else: formats = list() for ele in iter(entry.data): - formats.append(ele.format) + if ele.uncompressed_size < config.mail_size: + formats.append(ele.format) if 'MOBI' in formats: bookformats.append({'format': 'Mobi', 'convert': 0, @@ -173,14 +175,13 @@ def check_send_to_kindle(entry): bookformats.append({'format': 'Pdf', 'convert': 0, 'text': _('Send %(format)s to Kindle', format='Pdf')}) - if config.config_ebookconverter >= 1: + if config.config_converterpath: if 'EPUB' in formats and not 'MOBI' in formats: bookformats.append({'format': 'Mobi', 'convert':1, 'text': _('Convert %(orig)s to %(format)s and send to Kindle', orig='Epub', format='Mobi')}) - if config.config_ebookconverter == 2: if 'AZW3' in formats and not 'MOBI' in formats: bookformats.append({'format': 'Mobi', 'convert': 2, @@ -211,7 +212,7 @@ def check_read_formats(entry): # 3: If Pdf file is existing, it's directly send to kindle email def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): """Send email with attachments""" - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + book = calibre_db.get_book(book_id) if convert == 1: # returns None if success, otherwise errormessage @@ -317,13 +318,13 @@ def delete_book_file(book, calibrepath, book_format=None): return True, None else: log.error("Deleting book %s failed, book path not valid: %s", book.id, book.path) - return False, _("Deleting book %(id)s failed, book path not valid: %(path)s", + return True, _("Deleting book %(id)s, book path not valid: %(path)s", id=book.id, path=book.path) def update_dir_structure_file(book_id, calibrepath, first_author): - localbook = db.session.query(db.Books).filter(db.Books.id == book_id).first() + localbook = calibre_db.get_book(book_id) path = os.path.join(calibrepath, localbook.path) authordir = localbook.path.split('/')[0] @@ -382,7 +383,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author): def update_dir_structure_gdrive(book_id, first_author): error = False - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + book = calibre_db.get_book(book_id) path = book.path authordir = book.path.split('/')[0] @@ -493,18 +494,17 @@ def get_cover_on_failure(use_generic_cover): def get_book_cover(book_id): - book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters(allow_show_archived=True)).first() + book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) return get_book_cover_internal(book, use_generic_cover_on_failure=True) def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): - book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + book = calibre_db.get_book_by_uuid(book_uuid) return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, - use_generic_cover_on_failure): +def get_book_cover_internal(book, use_generic_cover_on_failure): if book and book.has_cover: if config.config_use_google_drive: try: @@ -594,7 +594,8 @@ def save_cover(img, book_path): return save_cover_from_filestorage(os.path.join(config.config_calibre_dir, book_path), "cover.jpg", img) -def do_download_file(book, book_format, data, headers): + +def do_download_file(book, book_format, client, data, headers): if config.config_use_google_drive: startTime = time.time() df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) @@ -608,6 +609,10 @@ def do_download_file(book, book_format, data, headers): if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): # ToDo: improve error handling log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + + if client == "kobo" and book_format == "kepub": + headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") + response = make_response(send_from_directory(filename, data.name + "." + book_format)) # ToDo Check headers parameter for element in headers: @@ -629,11 +634,12 @@ def check_unrar(unrarLocation): unrarLocation = unrarLocation.encode(sys.getfilesystemencoding()) unrarLocation = [unrarLocation] for lines in process_wait(unrarLocation): - value = re.search('UNRAR (.*) freeware', lines) + value = re.search('UNRAR (.*) freeware', lines, re.IGNORECASE) if value: version = value.group(1) log.debug("unrar version %s", version) - except OSError as err: + break + except (OSError, UnicodeDecodeError) as err: log.exception(err) return _('Error excecuting UnRar') @@ -719,66 +725,12 @@ def render_task_status(tasklist): return renderedtasklist -# Language and content filters for displaying in the UI -def common_filters(allow_show_archived=False): - if not allow_show_archived: - archived_books = ( - ub.session.query(ub.ArchivedBook) - .filter(ub.ArchivedBook.user_id == int(current_user.id)) - .filter(ub.ArchivedBook.is_archived == True) - .all() - ) - archived_book_ids = [archived_book.book_id for archived_book in archived_books] - archived_filter = db.Books.id.notin_(archived_book_ids) - else: - archived_filter = true() - - if current_user.filter_language() != "all": - lang_filter = db.Books.languages.any(db.Languages.lang_code == current_user.filter_language()) - else: - lang_filter = true() - negtags_list = current_user.list_denied_tags() - postags_list = current_user.list_allowed_tags() - neg_content_tags_filter = false() if negtags_list == [''] else db.Books.tags.any(db.Tags.name.in_(negtags_list)) - pos_content_tags_filter = true() if postags_list == [''] else db.Books.tags.any(db.Tags.name.in_(postags_list)) - if config.config_restricted_column: - pos_cc_list = current_user.allowed_column_value.split(',') - pos_content_cc_filter = true() if pos_cc_list == [''] else \ - getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\ - any(db.cc_classes[config.config_restricted_column].value.in_(pos_cc_list)) - neg_cc_list = current_user.denied_column_value.split(',') - neg_content_cc_filter = false() if neg_cc_list == [''] else \ - getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\ - any(db.cc_classes[config.config_restricted_column].value.in_(neg_cc_list)) - else: - pos_content_cc_filter = true() - neg_content_cc_filter = false() - return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter, - pos_content_cc_filter, ~neg_content_cc_filter, archived_filter) - - def tags_filters(): negtags_list = current_user.list_denied_tags() postags_list = current_user.list_allowed_tags() neg_content_tags_filter = false() if negtags_list == [''] else db.Tags.name.in_(negtags_list) pos_content_tags_filter = true() if postags_list == [''] else db.Tags.name.in_(postags_list) return and_(pos_content_tags_filter, ~neg_content_tags_filter) - # return ~(false()) if postags_list == [''] else db.Tags.in_(postags_list) - - -# Creates for all stored languages a translated speaking name in the array for the UI -def speaking_language(languages=None): - if not languages: - languages = db.session.query(db.Languages).join(db.books_languages_link).join(db.Books)\ - .filter(common_filters())\ - .group_by(text('books_languages_link.lang_code')).all() - for lang in languages: - try: - cur_l = LC.parse(lang.lang_code) - lang.name = cur_l.get_language_name(get_locale()) - except UnknownLocaleError: - lang.name = _(isoLanguages.get(part3=lang.lang_code).name) - return languages # checks if domain is in database (including wildcards) @@ -795,93 +747,28 @@ def check_valid_domain(domain_text): return not len(result) -# Orders all Authors in the list according to authors sort -def order_authors(entry): - sort_authors = entry.author_sort.split('&') - authors_ordered = list() - error = False - for auth in sort_authors: - # ToDo: How to handle not found authorname - result = db.session.query(db.Authors).filter(db.Authors.sort == auth.lstrip().strip()).first() - if not result: - error = True - break - authors_ordered.append(result) - if not error: - entry.authors = authors_ordered - return entry - - -# Fill indexpage with all requested data from database -def fill_indexpage(page, database, db_filter, order, *join): - return fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join) - - -def fill_indexpage_with_archived_books(page, database, db_filter, order, allow_show_archived, *join): - if current_user.show_detail_random(): - randm = db.session.query(db.Books).filter(common_filters(allow_show_archived))\ - .order_by(func.random()).limit(config.config_random_books) - else: - randm = false() - off = int(int(config.config_books_per_page) * (page - 1)) - query = db.session.query(database).join(*join, isouter=True).\ - filter(db_filter).\ - filter(common_filters(allow_show_archived)) - pagination = Pagination(page, config.config_books_per_page, - len(query.all())) - entries = query.order_by(*order).offset(off).limit(config.config_books_per_page).all() - for book in entries: - book = order_authors(book) - return entries, randm, pagination - - -def get_typeahead(database, query, replace=('', ''), tag_filter=true()): - query = query or '' - db.session.connection().connection.connection.create_function("lower", 1, lcase) - entries = db.session.query(database).filter(tag_filter).\ - filter(func.lower(database.name).ilike("%" + query + "%")).all() - json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries]) - return json_dumps - - -# read search results from calibre-database and return it (function is used for feed and simple search -def get_search_results(term): - db.session.connection().connection.connection.create_function("lower", 1, lcase) - q = list() - authorterms = re.split("[, ]+", term) - for authorterm in authorterms: - q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%"))) - - db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + term + "%")) - - return db.session.query(db.Books).filter(common_filters()).filter( - or_(db.Books.tags.any(func.lower(db.Tags.name).ilike("%" + term + "%")), - db.Books.series.any(func.lower(db.Series.name).ilike("%" + term + "%")), - db.Books.authors.any(and_(*q)), - db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + term + "%")), - func.lower(db.Books.title).ilike("%" + term + "%") - )).order_by(db.Books.sort).all() - - -def get_cc_columns(): - tmpcc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() +def get_cc_columns(filter_config_custom_read=False): + tmpcc = calibre_db.session.query(db.Custom_Columns)\ + .filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() + cc = [] + r = None if config.config_columns_to_ignore: - cc = [] - for col in tmpcc: - r = re.compile(config.config_columns_to_ignore) - if not r.match(col.name): - cc.append(col) - else: - cc = tmpcc + r = re.compile(config.config_columns_to_ignore) + + for col in tmpcc: + if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id: + continue + if r and r.match(col.name): + continue + cc.append(col) + return cc - -def get_download_link(book_id, book_format): +def get_download_link(book_id, book_format, client): book_format = book_format.split(".")[0] - book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() + book = calibre_db.get_filtered_book(book_id) if book: - data1 = db.session.query(db.Data).filter(db.Data.book == book.id)\ - .filter(db.Data.format == book_format.upper()).first() + data1 = data = calibre_db.get_book_format(book.id, book_format.upper()) else: abort(404) if data1: @@ -896,28 +783,6 @@ def get_download_link(book_id, book_format): headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % ( quote(file_name.encode('utf-8')), book_format, quote(file_name.encode('utf-8')), book_format) - return do_download_file(book, book_format, data1, headers) + return do_download_file(book, book_format, client, data1, headers) else: abort(404) - - -def check_exists_book(authr, title): - db.session.connection().connection.connection.create_function("lower", 1, lcase) - q = list() - authorterms = re.split(r'\s*&\s*', authr) - for authorterm in authorterms: - q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%"))) - - return db.session.query(db.Books).filter( - and_(db.Books.authors.any(and_(*q)), - func.lower(db.Books.title).ilike("%" + title + "%") - )).first() - -############### Database Helper functions - - -def lcase(s): - try: - return unidecode.unidecode(s.lower()) - except Exception as e: - log.exception(e) diff --git a/cps/isoLanguages.py b/cps/isoLanguages.py index 3281f98d..d8b7fa00 100644 --- a/cps/isoLanguages.py +++ b/cps/isoLanguages.py @@ -57,12 +57,12 @@ def get_language_name(locale, lang_code): def get_language_codes(locale, language_names, remainder=None): language_names = set(x.strip().lower() for x in language_names if x) - + languages = list() for k, v in get_language_names(locale).items(): v = v.lower() if v in language_names: + languages.append(k) language_names.remove(v) - yield k - if remainder is not None: remainder.extend(language_names) + return languages diff --git a/cps/jinjia.py b/cps/jinjia.py index 2c231582..28c2621a 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -111,10 +111,3 @@ def timestamptodate(date, fmt=None): @jinjia.app_template_filter('yesno') def yesno(value, yes, no): return yes if value else no - - -'''@jinjia.app_template_filter('canread') -def canread(ext): - if isinstance(ext, db.Data): - ext = ext.format - return ext.lower() in EXTENSIONS_READER''' diff --git a/cps/kobo.py b/cps/kobo.py index e8c14088..97d55db0 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -48,7 +48,7 @@ from sqlalchemy.sql.expression import and_, or_ from sqlalchemy.exc import StatementError import requests -from . import config, logger, kobo_auth, db, helper, shelf as shelf_lib, ub +from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub from .services import SyncToken as SyncToken from .web import download_required from .kobo_auth import requires_kobo_auth @@ -143,7 +143,7 @@ def HandleSyncRequest(): # We reload the book database so that the user get's a fresh view of the library # in case of external changes (e.g: adding a book through Calibre). - db.reconnect_db(config) + calibre_db.reconnect_db(config, ub.app_DB_path) archived_books = ( ub.session.query(ub.ArchivedBook) @@ -170,7 +170,7 @@ def HandleSyncRequest(): # It looks like it's treating the db.Books.last_modified field as a string and may fail # the comparison because of the +00:00 suffix. changed_entries = ( - db.session.query(db.Books) + calibre_db.session.query(db.Books) .join(db.Data) .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified, db.Books.id.in_(recently_restored_or_archived_books))) @@ -207,7 +207,7 @@ def HandleSyncRequest(): ub.KoboReadingState.user_id == current_user.id, ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements)))) for kobo_reading_state in changed_reading_states.all(): - book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none() + book = calibre_db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none() if book: sync_results.append({ "ChangedReadingState": { @@ -256,7 +256,7 @@ def HandleMetadataRequest(book_uuid): if not current_app.wsgi_app.is_proxied: log.debug('Kobo: Received unproxied request, changed request port to server port') log.info("Kobo library metadata request received for book %s" % book_uuid) - book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + book = calibre_db.get_book_by_uuid(book_uuid) if not book or not book.data: log.info(u"Book %s not found in database", book_uuid) return redirect_or_proxy_request() @@ -474,7 +474,7 @@ def add_items_to_shelf(items, shelf): items_unknown_to_calibre.append(item) continue - book = db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none() + book = calibre_db.get_book_by_uuid(item["RevisionId"]) if not book: items_unknown_to_calibre.append(item) continue @@ -545,7 +545,7 @@ def HandleTagRemoveItem(tag_id): items_unknown_to_calibre.append(item) continue - book = db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none() + book = calibre_db.get_book_by_uuid(item["RevisionId"]) if not book: items_unknown_to_calibre.append(item) continue @@ -613,7 +613,7 @@ def create_kobo_tag(shelf): "Type": "UserTag" } for book_shelf in shelf.books: - book = db.session.query(db.Books).filter(db.Books.id == book_shelf.book_id).one_or_none() + book = calibre_db.get_book(book_shelf.book_id) if not book: log.info(u"Book (id: %s) in BookShelf (id: %s) not found in book database", book_shelf.book_id, shelf.id) continue @@ -629,7 +629,7 @@ def create_kobo_tag(shelf): @kobo.route("/v1/library//state", methods=["GET", "PUT"]) @login_required def HandleStateRequest(book_uuid): - book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + book = calibre_db.get_book_by_uuid(book_uuid) if not book or not book.data: log.info(u"Book %s not found in database", book_uuid) return redirect_or_proxy_request() @@ -804,7 +804,7 @@ def TopLevelEndpoint(): @login_required def HandleBookDeletionRequest(book_uuid): log.info("Kobo book deletion request received for book %s" % book_uuid) - book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + book = calibre_db.get_book_by_uuid(book_uuid) if not book: log.info(u"Book %s not found in database", book_uuid) return redirect_or_proxy_request() diff --git a/cps/logger.py b/cps/logger.py index 3f850442..f13d75d3 100644 --- a/cps/logger.py +++ b/cps/logger.py @@ -82,7 +82,6 @@ def _absolute_log_file(log_file, default_log_file): if not os.path.dirname(log_file): log_file = os.path.join(_CONFIG_DIR, log_file) return os.path.abspath(log_file) - return default_log_file @@ -115,7 +114,7 @@ def setup(log_file, log_level=None): if previous_handler: # if the log_file has not changed, don't create a new handler if getattr(previous_handler, 'baseFilename', None) == log_file: - return + return "" if log_file == DEFAULT_LOG_FILE else log_file logging.debug("logging to %s level %s", log_file, r.level) if log_file == LOG_TO_STDERR or log_file == LOG_TO_STDOUT: @@ -132,12 +131,14 @@ def setup(log_file, log_level=None): if log_file == DEFAULT_LOG_FILE: raise file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=50000, backupCount=2) + log_file = "" file_handler.setFormatter(FORMATTER) for h in r.handlers: r.removeHandler(h) h.close() r.addHandler(file_handler) + return "" if log_file == DEFAULT_LOG_FILE else log_file def create_access_log(log_file, log_name, formatter): @@ -150,11 +151,18 @@ def create_access_log(log_file, log_name, formatter): access_log = logging.getLogger(log_name) access_log.propagate = False access_log.setLevel(logging.INFO) + try: + file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2) + except IOError: + if log_file == DEFAULT_ACCESS_LOG: + raise + file_handler = RotatingFileHandler(DEFAULT_ACCESS_LOG, maxBytes=50000, backupCount=2) + log_file = "" - file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2) file_handler.setFormatter(formatter) access_log.addHandler(file_handler) - return access_log + return access_log, \ + "" if _absolute_log_file(log_file, DEFAULT_ACCESS_LOG) == DEFAULT_ACCESS_LOG else log_file # Enable logging of smtp lib debug output diff --git a/cps/opds.py b/cps/opds.py index cd83709a..78d5d8ed 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -30,10 +30,10 @@ from flask_login import current_user from sqlalchemy.sql.expression import func, text, or_, and_ from werkzeug.security import check_password_hash -from . import constants, logger, config, db, ub, services, get_locale, isoLanguages -from .helper import fill_indexpage, get_download_link, get_book_cover, speaking_language +from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages +from .helper import get_download_link, get_book_cover from .pagination import Pagination -from .web import common_filters, get_search_results, render_read_books, download_required +from .web import render_read_books, download_required from flask_babel import gettext as _ from babel import Locale as LC from babel.core import UnknownLocaleError @@ -100,15 +100,15 @@ def feed_normal_search(): @requires_basic_auth_if_no_ano def feed_new(): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, True, [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, True, [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @opds.route("/opds/discover") @requires_basic_auth_if_no_ano def feed_discover(): - entries = db.session.query(db.Books).filter(common_filters()).order_by(func.random())\ + entries = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(func.random())\ .limit(config.config_books_per_page) pagination = Pagination(1, config.config_books_per_page, int(config.config_books_per_page)) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -118,9 +118,9 @@ def feed_discover(): @requires_basic_auth_if_no_ano def feed_best_rated(): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, db.Books.ratings.any(db.Ratings.rating > 9), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, db.Books.ratings.any(db.Ratings.rating > 9), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -133,16 +133,13 @@ def feed_hot(): hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - downloadBook = db.session.query(db.Books).filter(db.Books.id == book.Downloads.book_id).first() + downloadBook = calibre_db.get_book(book.Downloads.book_id) if downloadBook: entries.append( - db.session.query(db.Books).filter(common_filters()) - .filter(db.Books.id == book.Downloads.book_id).first() + calibre_db.get_filtered_book(book.Downloads.book_id) ) else: ub.delete_download(book.Downloads.book_id) - # ub.session.query(ub.Downloads).filter(book.Downloads.book_id == ub.Downloads.book_id).delete() - # ub.session.commit() numBooks = entries.__len__() pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, numBooks) @@ -153,11 +150,13 @@ def feed_hot(): @requires_basic_auth_if_no_ano def feed_authorindex(): off = request.args.get("offset") or 0 - entries = db.session.query(db.Authors).join(db.books_authors_link).join(db.Books).filter(common_filters())\ - .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).limit(config.config_books_per_page)\ + entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\ + .filter(calibre_db.common_filters())\ + .group_by(text('books_authors_link.author'))\ + .order_by(db.Authors.sort).limit(config.config_books_per_page)\ .offset(off) pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, - len(db.session.query(db.Authors).all())) + len(calibre_db.session.query(db.Authors).all())) return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_author', pagination=pagination) @@ -165,10 +164,10 @@ def feed_authorindex(): @requires_basic_auth_if_no_ano def feed_author(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.authors.any(db.Authors.id == book_id), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.authors.any(db.Authors.id == book_id), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -176,11 +175,14 @@ def feed_author(book_id): @requires_basic_auth_if_no_ano def feed_publisherindex(): off = request.args.get("offset") or 0 - entries = db.session.query(db.Publishers).join(db.books_publishers_link).join(db.Books).filter(common_filters())\ - .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.sort)\ + entries = calibre_db.session.query(db.Publishers)\ + .join(db.books_publishers_link)\ + .join(db.Books).filter(calibre_db.common_filters())\ + .group_by(text('books_publishers_link.publisher'))\ + .order_by(db.Publishers.sort)\ .limit(config.config_books_per_page).offset(off) pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, - len(db.session.query(db.Publishers).all())) + len(calibre_db.session.query(db.Publishers).all())) return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_publisher', pagination=pagination) @@ -188,10 +190,10 @@ def feed_publisherindex(): @requires_basic_auth_if_no_ano def feed_publisher(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.publishers.any(db.Publishers.id == book_id), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.publishers.any(db.Publishers.id == book_id), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -199,10 +201,16 @@ def feed_publisher(book_id): @requires_basic_auth_if_no_ano def feed_categoryindex(): off = request.args.get("offset") or 0 - entries = db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters())\ - .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).offset(off).limit(config.config_books_per_page) + entries = calibre_db.session.query(db.Tags)\ + .join(db.books_tags_link)\ + .join(db.Books)\ + .filter(calibre_db.common_filters())\ + .group_by(text('books_tags_link.tag'))\ + .order_by(db.Tags.name)\ + .offset(off)\ + .limit(config.config_books_per_page) pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, - len(db.session.query(db.Tags).all())) + len(calibre_db.session.query(db.Tags).all())) return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_category', pagination=pagination) @@ -210,10 +218,10 @@ def feed_categoryindex(): @requires_basic_auth_if_no_ano def feed_category(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.tags.any(db.Tags.id == book_id), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.tags.any(db.Tags.id == book_id), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -221,10 +229,15 @@ def feed_category(book_id): @requires_basic_auth_if_no_ano def feed_seriesindex(): off = request.args.get("offset") or 0 - entries = db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters())\ - .group_by(text('books_series_link.series')).order_by(db.Series.sort).offset(off).all() + entries = calibre_db.session.query(db.Series)\ + .join(db.books_series_link)\ + .join(db.Books)\ + .filter(calibre_db.common_filters())\ + .group_by(text('books_series_link.series'))\ + .order_by(db.Series.sort)\ + .offset(off).all() pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, - len(db.session.query(db.Series).all())) + len(calibre_db.session.query(db.Series).all())) return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_series', pagination=pagination) @@ -232,10 +245,10 @@ def feed_seriesindex(): @requires_basic_auth_if_no_ano def feed_series(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.series.any(db.Series.id == book_id), - [db.Books.series_index]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.series.any(db.Series.id == book_id), + [db.Books.series_index]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -243,10 +256,13 @@ def feed_series(book_id): @requires_basic_auth_if_no_ano def feed_ratingindex(): off = request.args.get("offset") or 0 - entries = db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'), + entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'), (db.Ratings.rating / 2).label('name')) \ - .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \ - .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all() + .join(db.books_ratings_link)\ + .join(db.Books)\ + .filter(calibre_db.common_filters()) \ + .group_by(text('books_ratings_link.rating'))\ + .order_by(db.Ratings.rating).all() pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, len(entries)) @@ -260,10 +276,10 @@ def feed_ratingindex(): @requires_basic_auth_if_no_ano def feed_ratings(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.ratings.any(db.Ratings.id == book_id), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.ratings.any(db.Ratings.id == book_id), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -271,8 +287,10 @@ def feed_ratings(book_id): @requires_basic_auth_if_no_ano def feed_formatindex(): off = request.args.get("offset") or 0 - entries = db.session.query(db.Data).join(db.Books).filter(common_filters()) \ - .group_by(db.Data.format).order_by(db.Data.format).all() + entries = calibre_db.session.query(db.Data).join(db.Books)\ + .filter(calibre_db.common_filters()) \ + .group_by(db.Data.format)\ + .order_by(db.Data.format).all() pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, len(entries)) @@ -286,10 +304,10 @@ def feed_formatindex(): @requires_basic_auth_if_no_ano def feed_format(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.data.any(db.Data.format == book_id.upper()), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.data.any(db.Data.format == book_id.upper()), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -299,13 +317,13 @@ def feed_format(book_id): def feed_languagesindex(): off = request.args.get("offset") or 0 if current_user.filter_language() == u"all": - languages = speaking_language() + languages = calibre_db.speaking_language() else: try: cur_l = LC.parse(current_user.filter_language()) except UnknownLocaleError: cur_l = None - languages = db.session.query(db.Languages).filter( + languages = calibre_db.session.query(db.Languages).filter( db.Languages.lang_code == current_user.filter_language()).all() if cur_l: languages[0].name = cur_l.get_language_name(get_locale()) @@ -320,10 +338,10 @@ def feed_languagesindex(): @requires_basic_auth_if_no_ano def feed_languages(book_id): off = request.args.get("offset") or 0 - entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), - db.Books, - db.Books.languages.any(db.Languages.id == book_id), - [db.Books.timestamp.desc()]) + entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, + db.Books.languages.any(db.Languages.id == book_id), + [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -356,7 +374,7 @@ def feed_shelf(book_id): books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by( ub.BookShelf.order.asc()).all() for book in books_in_shelf: - cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first() + cur_book = calibre_db.get_book(book.book_id) result.append(cur_book) pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, len(result)) @@ -367,14 +385,18 @@ def feed_shelf(book_id): @requires_basic_auth_if_no_ano @download_required def opds_download_link(book_id, book_format): - return get_download_link(book_id, book_format.lower()) + if "Kobo" in request.headers.get('User-Agent'): + client = "kobo" + else: + client = "" + return get_download_link(book_id, book_format.lower(), client) @opds.route("/ajax/book//") @opds.route("/ajax/book/", defaults={'library': ""}) @requires_basic_auth_if_no_ano def get_metadata_calibre_companion(uuid, library): - entry = db.session.query(db.Books).filter(db.Books.uuid.like("%" + uuid + "%")).first() + entry = calibre_db.session.query(db.Books).filter(db.Books.uuid.like("%" + uuid + "%")).first() if entry is not None: js = render_template('json.txt', entry=entry) response = make_response(js) @@ -386,8 +408,7 @@ def get_metadata_calibre_companion(uuid, library): def feed_search(term): if term: - term = term.strip().lower() - entries = get_search_results(term) + entries = calibre_db.get_search_results(term) entriescount = len(entries) if len(entries) > 0 else 1 pagination = Pagination(1, entriescount, entriescount) return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination) diff --git a/cps/server.py b/cps/server.py index d2253ab2..7c2d321d 100644 --- a/cps/server.py +++ b/cps/server.py @@ -72,7 +72,11 @@ class WebServer(object): if config.config_access_log: log_name = "gevent.access" if _GEVENT else "tornado.access" formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO - self.access_logger = logger.create_access_log(config.config_access_logfile, log_name, formatter) + self.access_logger, logfile = logger.create_access_log(config.config_access_logfile, log_name, formatter) + if logfile != config.config_access_logfile: + log.warning("Accesslog path %s not valid, falling back to default", config.config_access_logfile) + config.config_access_logfile = logfile + config.save() else: if not _GEVENT: logger.get('tornado.access').disabled = True @@ -196,6 +200,9 @@ class WebServer(object): def stop(self, restart=False): from . import updater_thread updater_thread.stop() + from . import calibre_db + calibre_db.stop() + log.info("webserver stop (restart=%s)", restart) self.restart = restart diff --git a/cps/services/goodreads_support.py b/cps/services/goodreads_support.py index 55161c7a..58255f3c 100644 --- a/cps/services/goodreads_support.py +++ b/cps/services/goodreads_support.py @@ -20,7 +20,10 @@ from __future__ import division, print_function, unicode_literals import time from functools import reduce -from goodreads.client import GoodreadsClient +try: + from goodreads.client import GoodreadsClient +except ImportError: + from betterreads.client import GoodreadsClient try: import Levenshtein except ImportError: Levenshtein = False @@ -95,8 +98,12 @@ def get_other_books(author_info, library_books=None): for book in author_info.books: if book.isbn in identifiers: continue - if book.gid["#text"] in identifiers: - continue + if isinstance(book.gid, int): + if book.gid in identifiers: + continue + else: + if book.gid["#text"] in identifiers: + continue if Levenshtein and library_titles: goodreads_title = book._book_dict['title_without_series'] diff --git a/cps/shelf.py b/cps/shelf.py index 4d6d5103..b13fe556 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -28,9 +28,8 @@ from flask_babel import gettext as _ from flask_login import login_required, current_user from sqlalchemy.sql.expression import func -from . import logger, ub, searched_ids, db +from . import logger, ub, searched_ids, db, calibre_db from .web import render_title_template -from .helper import common_filters shelf = Blueprint('shelf', __name__) @@ -320,11 +319,11 @@ def show_shelf(shelf_type, shelf_id): books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\ .order_by(ub.BookShelf.order.asc()).all() for book in books_in_shelf: - cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first() + cur_book = calibre_db.get_filtered_book(book.book_id) if cur_book: result.append(cur_book) else: - cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first() + cur_book = calibre_db.get_book(book.book_id) if not cur_book: log.info('Not existing book %s in %s deleted', book.book_id, shelf) ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete() @@ -356,7 +355,7 @@ def order_shelf(shelf_id): books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \ .order_by(ub.BookShelf.order.asc()).all() for book in books_in_shelf2: - cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first() + cur_book = calibre_db.get_filtered_book(book.book_id) if cur_book: result.append({'title': cur_book.title, 'id': cur_book.id, @@ -364,7 +363,7 @@ def order_shelf(shelf_id): 'series': cur_book.series, 'series_index': cur_book.series_index}) else: - cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first() + cur_book = calibre_db.get_book(book.book_id) result.append({'title': _('Hidden Book'), 'id': cur_book.id, 'author': [], diff --git a/cps/static/css/caliBlur_override.css b/cps/static/css/caliBlur_override.css new file mode 100644 index 00000000..05a7c0d8 --- /dev/null +++ b/cps/static/css/caliBlur_override.css @@ -0,0 +1,17 @@ +body.serieslist.grid-view div.container-fluid>div>div.col-sm-10:before{ + display: none; +} + +.cover .badge{ + position: absolute; + top: 0; + right: 0; + background-color: #cc7b19; + border-radius: 0; + padding: 0 8px; + box-shadow: 0 0 4px rgba(0,0,0,.6); + line-height: 24px; +} +.cover{ + box-shadow: 0 0 4px rgba(0,0,0,.6); +} diff --git a/cps/static/css/style.css b/cps/static/css/style.css index 05fd357c..c043f459 100644 --- a/cps/static/css/style.css +++ b/cps/static/css/style.css @@ -109,7 +109,7 @@ a { color: #45b29d; } .container-fluid .book .cover img { border: 1px solid #fff; - box-sizeing: border-box; + box-sizing: border-box; height: 100%; bottom: 0; position: absolute; @@ -165,7 +165,7 @@ span.glyphicon.glyphicon-tags { .container-fluid .single .cover img { border: 1px solid #fff; - box-sizeing: border-box; + box-sizing: border-box; -webkit-box-shadow: 0 5px 8px -6px #777; -moz-box-shadow: 0 5px 8px -6px #777; box-shadow: 0 5px 8px -6px #777; @@ -174,6 +174,12 @@ span.glyphicon.glyphicon-tags { .navbar-default .navbar-toggle .icon-bar {background-color: #000; } .navbar-default .navbar-toggle {border-color: #000; } .cover { margin-bottom: 10px; } +.cover .badge{ + position: absolute; + top: 2px; + left: 2px; + background-color: #777; +} .cover-height { max-height: 100px;} .col-sm-2 a .cover-small { @@ -207,7 +213,7 @@ span.glyphicon.glyphicon-tags { .panel-body {background-color: #f5f5f5; } .spinner {margin: 0 41%; } .spinner2 {margin: 0 41%; } - +.intend-form { margin-left:20px; } table .bg-dark-danger {background-color: #d9534f; color: #fff; } table .bg-dark-danger a {color: #fff; } table .bg-dark-danger:hover {background-color: #c9302c; } @@ -296,3 +302,4 @@ div.log { white-space: nowrap; padding: 0.5em; } + diff --git a/cps/static/js/filter_grid.js b/cps/static/js/filter_grid.js new file mode 100644 index 00000000..457b9055 --- /dev/null +++ b/cps/static/js/filter_grid.js @@ -0,0 +1,54 @@ +/* This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) + * Copyright (C) 2018 OzzieIsaacs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +var $list = $("#list").isotope({ + itemSelector: ".book", + layoutMode: "fitRows", + getSortData: { + title: ".title", + } +}); + +$("#desc").click(function() { + $list.isotope({ + sortBy: "name", + sortAscending: true + }); + return; +}); + +$("#asc").click(function() { + $list.isotope({ + sortBy: "name", + sortAscending: false + }); + return; +}); + +$("#all").click(function() { + // go through all elements and make them visible + $list.isotope({ filter: function() { + return true; + } }) +}); + +$(".char").click(function() { + var character = this.innerText; + $list.isotope({ filter: function() { + return this.attributes["data-id"].value.charAt(0).toUpperCase() == character; + } }) +}); diff --git a/cps/static/js/main.js b/cps/static/js/main.js index f005e3b7..6338be0b 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -323,4 +323,17 @@ $(function() { $(".discover .row").isotope("layout"); }); + $(".update-view").click(function(e) { + var target = $(this).data("target"); + var view = $(this).data("view"); + + e.preventDefault(); + e.stopPropagation(); + var data = {}; + data[target] = view; + console.debug("Updating view data: ", data); + $.post( "/ajax/view", data).done(function( ) { + location.reload(); + }); + }); }); diff --git a/cps/subproc_wrapper.py b/cps/subproc_wrapper.py index 0ad1c1d2..23facbd7 100644 --- a/cps/subproc_wrapper.py +++ b/cps/subproc_wrapper.py @@ -22,7 +22,7 @@ import os import subprocess -def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE): +def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE, newlines=True): # Linux py2.7 encode as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters # windows py2.7 encode as string with quotes empty element for parameters is okay @@ -41,12 +41,13 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro else: exc_command = [x for x in command] - return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=True, env=env) + return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=newlines, env=env) def process_wait(command, serr=subprocess.PIPE): # Run command, wait for process to terminate, and return an iterator over lines of its output. - p = process_open(command, serr=serr) + newlines = os.name != 'nt' + p = process_open(command, serr=serr, newlines=newlines) p.wait() for line in p.stdout.readlines(): if isinstance(line, bytes): diff --git a/cps/templates/admin.html b/cps/templates/admin.html index ab143c59..a21fae48 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -13,11 +13,14 @@ {{_('E-mail Address')}} {{_('Send to Kindle E-mail Address')}} {{_('Downloads')}} - {{_('Admin')}} - {{_('Download')}} - {{_('View Books')}} - {{_('Upload')}} - {{_('Edit')}} + {{_('Admin')}} + {{_('Password')}} + {{_('Upload')}} + {{_('Download')}} + {{_('View Books')}} + {{_('Edit')}} + {{_('Delete')}} + {{_('Public Shelf')}} {% for user in allUser %} {% if not user.role_anonymous() or config.config_anonbrowse %} @@ -27,10 +30,13 @@ {{user.kindle_mail}} {{user.downloads.count()}} {{ display_bool_setting(user.role_admin()) }} - {{ display_bool_setting(user.role_download()) }} - {{ display_bool_setting(user.role_viewer()) }} - {{ display_bool_setting(user.role_upload()) }} - {{ display_bool_setting(user.role_edit()) }} + {{ display_bool_setting(user.role_passwd()) }} + {{ display_bool_setting(user.role_upload()) }} + {{ display_bool_setting(user.role_download()) }} + {{ display_bool_setting(user.role_viewer()) }} + {{ display_bool_setting(user.role_edit()) }} + {{ display_bool_setting(user.role_delete_books()) }} + {{ display_bool_setting(user.role_edit_shelfs()) }} {% endif %} {% endfor %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 0996efcc..14bc590a 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -86,7 +86,7 @@
- +
diff --git a/cps/templates/book_table.html b/cps/templates/book_table.html new file mode 100644 index 00000000..c8149f03 --- /dev/null +++ b/cps/templates/book_table.html @@ -0,0 +1,59 @@ +{% extends "layout.html" %} +{% block body %} +

{{_(title)}}

+ + +
+
+ {% for entry in entries %} + {% if loop.index0 == (loop.length/2+loop.length%2)|int and loop.length > 20 %} +
+ +
+{% endblock %} +{% block js %} + +{% endblock %} diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index f3187011..77a60c1b 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -3,7 +3,7 @@

{{title}}

-
+

@@ -15,10 +15,13 @@

-
- - -
+
+ + + + + +
{% if feature_support['gdrive'] %}
@@ -87,21 +90,25 @@
-
- - + +
+ + + +
-
- - + +
+ + + +
@@ -154,17 +161,29 @@
- +
+
+
+ + +
+
- +
+
+
+ + +
+
@@ -326,30 +345,33 @@
-
-
-
-
-
-
-
+ +
+ + + + +
+
+ + +
+ +
+ + + +
-
-
- - -
-
- - -
+ {% if feature_support['rar'] %} + +
+ + + +
- {% if feature_support['rar'] %} -
- - -
- {% endif %} + {% endif %}
diff --git a/cps/templates/config_view_edit.html b/cps/templates/config_view_edit.html index adc846c6..d87e9f81 100644 --- a/cps/templates/config_view_edit.html +++ b/cps/templates/config_view_edit.html @@ -6,7 +6,7 @@ {% block body %}

{{title}}

- +
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 32b215f9..24ba10c7 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -202,6 +202,7 @@

+ {% if g.user.check_visibility(32768) %}

+ {% endif %}
{% endif %} diff --git a/cps/templates/email_edit.html b/cps/templates/email_edit.html index 2ac9d463..bce3bc21 100644 --- a/cps/templates/email_edit.html +++ b/cps/templates/email_edit.html @@ -6,14 +6,14 @@ {% block body %}

{{title}}

-
+
- +
@@ -35,11 +35,19 @@
+ +
+ + + + +
{{_('Cancel')}}
{% if g.allow_registration %} +

{{_('Allowed Domains (Whitelist)')}}

@@ -74,6 +82,7 @@
+
{% endif %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html new file mode 100644 index 00000000..8fb084fd --- /dev/null +++ b/cps/templates/grid.html @@ -0,0 +1,49 @@ +{% extends "layout.html" %} +{% block body %} +

{{_(title)}}

+ + + + {% if entries[0] %} +
+ {% for entry in entries %} +
+
+ + {{ entry[0].name }} + {{entry.count}} + +
+
+ +

{{entry[0].series[0].name|shortentitle}}

+ +
+
+ {% endfor %} +
+ {% endif %} + + +{% endblock %} +{% block js %} + +{% endblock %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 349feca9..4e529eea 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -17,6 +17,7 @@ {% if g.current_theme == 1 %} + {% endif %} @@ -25,7 +26,7 @@ - +
diff --git a/cps/templates/register.html b/cps/templates/register.html index 0e218ec2..043378c3 100644 --- a/cps/templates/register.html +++ b/cps/templates/register.html @@ -3,10 +3,12 @@

{{_('Register New Account')}}

+ {% if not config.config_register_email %}
+ {% endif %}
diff --git a/cps/templates/search_form.html b/cps/templates/search_form.html index 9faa7877..60cffa1d 100644 --- a/cps/templates/search_form.html +++ b/cps/templates/search_form.html @@ -1,6 +1,6 @@ {% extends "layout.html" %} {% block body %} -
+
diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index 16e6626a..2648dd7c 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -3,6 +3,7 @@

{{title}}

+
{% if new_user or ( g.user and content.nickname != "Guest" and g.user.role_admin() ) %}
@@ -65,6 +66,7 @@
{% endif %} +
{% for element in sidebar %} {% if element['config_show'] %} diff --git a/cps/ub.py b/cps/ub.py index f5d0f28a..c923dce2 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -49,6 +49,7 @@ from . import constants session = None +app_DB_path = None Base = declarative_base() @@ -107,6 +108,11 @@ def get_sidebar_config(kwargs=None): {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived", "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived", "show_text": _('Show archived books'), "config_show": content}) + '''sidebar.append( + {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_list', "id": "list", + "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list", + "show_text": _('Show Books List'), "config_show": content})''' + return sidebar @@ -211,6 +217,7 @@ class User(UserBase, Base): denied_column_value = Column(String, default="") allowed_column_value = Column(String, default="") remote_auth_token = relationship('RemoteAuthToken', backref='user', lazy='dynamic') + series_view = Column(String(10), default="list") if oauth_support: @@ -251,6 +258,7 @@ class Anonymous(AnonymousUserMixin, UserBase): self.allowed_tags = data.allowed_tags self.denied_column_value = data.denied_column_value self.allowed_column_value = data.allowed_column_value + self.series_view = data.series_view def role_admin(self): return False @@ -447,7 +455,7 @@ class RemoteAuthToken(Base): # Migrate database to current version, has to be updated after every database change. Currently migration from -# everywhere to curent should work. Migration is done by checking if relevant coloums are existing, and than adding +# everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding # rows with SQL commands def migrate_Database(session): engine = session.bind @@ -557,6 +565,12 @@ def migrate_Database(session): conn.execute("ALTER TABLE user ADD column `denied_column_value` DEFAULT ''") conn.execute("ALTER TABLE user ADD column `allowed_column_value` DEFAULT ''") session.commit() + try: + session.query(exists().where(User.series_view)).scalar() + except exc.OperationalError: + conn = engine.connect() + conn.execute("ALTER TABLE user ADD column `series_view` VARCHAR(10) DEFAULT 'list'") + if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() \ is None: create_anonymous_user(session) @@ -568,7 +582,7 @@ def migrate_Database(session): # Create new table user_id and copy contents of table user into it conn = engine.connect() conn.execute("CREATE TABLE user_id (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT," - " nickname VARCHAR(64)," + "nickname VARCHAR(64)," "email VARCHAR(120)," "role SMALLINT," "password VARCHAR," @@ -576,10 +590,11 @@ def migrate_Database(session): "locale VARCHAR(2)," "sidebar_view INTEGER," "default_language VARCHAR(3)," + "series_view VARCHAR(10)," "UNIQUE (nickname)," "UNIQUE (email))") conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale," - "sidebar_view, default_language) " + "sidebar_view, default_language, series_view) " "SELECT id, nickname, email, role, password, kindle_mail, locale," "sidebar_view, default_language FROM user") # delete old user table and rename new user_id table to user: @@ -616,8 +631,7 @@ def delete_download(book_id): session.query(Downloads).filter(book_id == Downloads.book_id).delete() session.commit() - -# Generate user Guest (translated text), as anoymous user, no rights +# Generate user Guest (translated text), as anonymous user, no rights def create_anonymous_user(session): user = User() user.nickname = "Guest" @@ -651,7 +665,9 @@ def create_admin_user(session): def init_db(app_db_path): # Open session for database connection global session + global app_DB_path + app_DB_path = app_db_path engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False) Session = sessionmaker() diff --git a/cps/uploader.py b/cps/uploader.py index a21c9fc9..1323e3d0 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -40,7 +40,7 @@ try: from wand.exceptions import PolicyError use_generic_pdf_cover = False except (ImportError, RuntimeError) as e: - log.debug('cannot import Image, generating pdf covers for pdf uploads will not work: %s', e) + log.debug('Cannot import Image, generating pdf covers for pdf uploads will not work: %s', e) use_generic_pdf_cover = True try: @@ -48,21 +48,21 @@ try: from PyPDF2 import __version__ as PyPdfVersion use_pdf_meta = True except ImportError as e: - log.debug('cannot import PyPDF2, extracting pdf metadata will not work: %s', e) + log.debug('Cannot import PyPDF2, extracting pdf metadata will not work: %s', e) use_pdf_meta = False try: from . import epub use_epub_meta = True except ImportError as e: - log.debug('cannot import epub, extracting epub metadata will not work: %s', e) + log.debug('Cannot import epub, extracting epub metadata will not work: %s', e) use_epub_meta = False try: from . import fb2 use_fb2_meta = True except ImportError as e: - log.debug('cannot import fb2, extracting fb2 metadata will not work: %s', e) + log.debug('Cannot import fb2, extracting fb2 metadata will not work: %s', e) use_fb2_meta = False try: @@ -70,20 +70,17 @@ try: from PIL import __version__ as PILversion use_PIL = True except ImportError as e: - log.debug('cannot import Pillow, using png and webp images as cover will not work: %s', e) + log.debug('Cannot import Pillow, using png and webp images as cover will not work: %s', e) use_PIL = False -__author__ = 'lemmsh' - def process(tmp_file_path, original_file_name, original_file_extension, rarExecutable): - """Get the metadata for tmp_file_path.""" meta = None extension_upper = original_file_extension.upper() try: if ".PDF" == extension_upper: meta = pdf_meta(tmp_file_path, original_file_name, original_file_extension) - elif ".EPUB" == extension_upper and use_epub_meta is True: + elif extension_upper in [".KEPUB", ".EPUB"] and use_epub_meta is True: meta = epub.get_epub_info(tmp_file_path, original_file_name, original_file_extension) elif ".FB2" == extension_upper and use_fb2_meta is True: meta = fb2.get_fb2_info(tmp_file_path, original_file_extension) @@ -182,7 +179,7 @@ def get_versions(): else: PILVersion = u'not installed' if comic.use_comic_meta: - ComicVersion = u'installed' + ComicVersion = comic.comic_version or u'installed' else: ComicVersion = u'not installed' return {'Image Magick': IVersion, diff --git a/cps/web.py b/cps/web.py index 5529aea8..1c1098c1 100644 --- a/cps/web.py +++ b/cps/web.py @@ -37,9 +37,9 @@ from flask import Blueprint from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for from flask_babel import gettext as _ from flask_login import login_user, logout_user, login_required, current_user -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError from sqlalchemy.sql.expression import text, func, true, false, not_, and_, or_ -from werkzeug.exceptions import default_exceptions +from werkzeug.exceptions import default_exceptions, InternalServerError from sqlalchemy.sql.functions import coalesce try: from werkzeug.exceptions import FailedDependency @@ -48,13 +48,13 @@ except ImportError: from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash, check_password_hash -from . import constants, logger, isoLanguages, services, worker +from . import constants, logger, isoLanguages, services, worker, cli from . import searched_ids, lm, babel, db, ub, config, get_locale, app +from . import calibre_db from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import common_filters, get_search_results, fill_indexpage, fill_indexpage_with_archived_books, \ - speaking_language, check_valid_domain, order_authors, get_typeahead, render_task_status, json_serial, \ +from .helper import check_valid_domain, render_task_status, json_serial, \ get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ - send_registration_mail, check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password + send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back @@ -124,6 +124,21 @@ if feature_support['ldap']: log.debug('LDAP server not accessible while trying to login to opds feed') return error_http(FailedDependency()) +# @app.errorhandler(InvalidRequestError) +#@app.errorhandler(OperationalError) +#def handle_db_exception(e): +# db.session.rollback() +# log.error('Database request error: %s',e) +# return internal_error(InternalServerError(e)) + +@app.after_request +def add_security_headers(resp): + # resp.headers['Content-Security-Policy']= "script-src 'self' https://www.googleapis.com https://api.douban.com https://comicvine.gamespot.com;" + resp.headers['X-Content-Type-Options'] = 'nosniff' + resp.headers['X-Frame-Options'] = 'SAMEORIGIN' + resp.headers['X-XSS-Protection'] = '1; mode=block' + # resp.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains' + return resp web = Blueprint('web', __name__) log = logger.create() @@ -423,21 +438,24 @@ def toggle_read(book_id): ub.session.commit() else: try: - db.update_title_sort(config) - book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() + calibre_db.update_title_sort(config) + book = calibre_db.get_filtered_book(book_id) read_status = getattr(book, 'custom_column_' + str(config.config_read_column)) if len(read_status): read_status[0].value = not read_status[0].value - db.session.commit() + calibre_db.session.commit() else: cc_class = db.cc_classes[config.config_read_column] new_cc = cc_class(value=1, book=book_id) - db.session.add(new_cc) - db.session.commit() + calibre_db.session.add(new_cc) + calibre_db.session.commit() except KeyError: log.error(u"Custom Column No.%d is not exisiting in calibre database", config.config_read_column) - return "" + except OperationalError as e: + calibre_db.session.rollback() + log.error(u"Read status could not set: %e", e) + return "" @web.route("/ajax/togglearchived/", methods=['POST']) @login_required @@ -455,11 +473,30 @@ def toggle_archived(book_id): return "" +@web.route("/ajax/view", methods=["POST"]) +@login_required +def update_view(): + to_save = request.form.to_dict() + allowed_view = ['grid', 'list'] + if "series_view" in to_save and to_save["series_view"] in allowed_view: + current_user.series_view = to_save["series_view"] + else: + log.error("Invalid request received: %r %r", request, to_save) + return "Invalid request", 400 + + try: + ub.session.commit() + except InvalidRequestError: + log.error("Invalid request received: %r ", request, ) + return "Invalid request", 400 + return "", 200 + + ''' @web.route("/ajax/getcomic///") @login_required def get_comic_book(book_id, book_format, page): - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + book = calibre_db.get_book(book_id) if not book: return "", 204 else: @@ -513,25 +550,25 @@ def get_comic_book(book_id, book_format, page): @web.route("/get_authors_json", methods=['GET']) @login_required_if_no_ano def get_authors_json(): - return get_typeahead(db.Authors, request.args.get('q'), ('|', ',')) + return calibre_db.get_typeahead(db.Authors, request.args.get('q'), ('|', ',')) @web.route("/get_publishers_json", methods=['GET']) @login_required_if_no_ano def get_publishers_json(): - return get_typeahead(db.Publishers, request.args.get('q'), ('|', ',')) + return calibre_db.get_typeahead(db.Publishers, request.args.get('q'), ('|', ',')) @web.route("/get_tags_json", methods=['GET']) @login_required_if_no_ano def get_tags_json(): - return get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters()) + return calibre_db.get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters()) @web.route("/get_series_json", methods=['GET']) @login_required_if_no_ano def get_series_json(): - return get_typeahead(db.Series, request.args.get('q')) + return calibre_db.get_typeahead(db.Series, request.args.get('q')) @web.route("/get_languages_json", methods=['GET']) @@ -552,8 +589,8 @@ def get_languages_json(): @login_required_if_no_ano def get_matching_tags(): tag_dict = {'tags': []} - q = db.session.query(db.Books) - db.session.connection().connection.connection.create_function("lower", 1, lcase) + q = calibre_db.session.query(db.Books) + calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) author_input = request.args.get('author_name') or '' title_input = request.args.get('book_title') or '' include_tag_inputs = request.args.getlist('include_tag') or '' @@ -583,7 +620,7 @@ def get_matching_tags(): @web.route('/page/') @login_required_if_no_ano def index(page): - entries, random, pagination = fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()]) + entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()]) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Recently Added Books"), page="root") @@ -610,15 +647,17 @@ def books_list(data, sort, book_id, page): if data == "rated": if current_user.check_visibility(constants.SIDEBAR_BEST_RATED): - entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.rating > 9), - order) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.ratings.any(db.Ratings.rating > 9), + order) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, title=_(u"Top Rated Books"), page="rated") else: abort(404) elif data == "discover": if current_user.check_visibility(constants.SIDEBAR_RANDOM): - entries, __, pagination = fill_indexpage(page, db.Books, True, [func.randomblob(2)]) + entries, __, pagination = calibre_db.calibre_db.fill_indexpage(page, db.Books, True, [func.randomblob(2)]) pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page) return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id, title=_(u"Discover (Random Books)"), page="discover") @@ -647,7 +686,7 @@ def books_list(data, sort, book_id, page): elif data == "archived": return render_archived_books(page, order) else: - entries, random, pagination = fill_indexpage(page, db.Books, True, order) + entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, order) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Books"), page="newest") @@ -655,7 +694,7 @@ def books_list(data, sort, book_id, page): def render_hot_books(page): if current_user.check_visibility(constants.SIDEBAR_HOT): if current_user.show_detail_random(): - random = db.session.query(db.Books).filter(common_filters()) \ + random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ .order_by(func.random()).limit(config.config_random_books) else: random = false() @@ -665,7 +704,7 @@ def render_hot_books(page): hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - downloadBook = db.session.query(db.Books).filter(common_filters()).filter( + downloadBook = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter( db.Books.id == book.Downloads.book_id).first() if downloadBook: entries.append(downloadBook) @@ -682,15 +721,18 @@ def render_hot_books(page): def render_author_books(page, author_id, order): - entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == author_id), - [order[0], db.Series.name, db.Books.series_index], - db.books_series_link, db.Series) + entries, __, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.authors.any(db.Authors.id == author_id), + [order[0], db.Series.name, db.Books.series_index], + db.books_series_link, + db.Series) if entries is None or not len(entries): flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), category="error") return redirect(url_for("web.index")) - author = db.session.query(db.Authors).get(author_id) + author = calibre_db.session.query(db.Authors).get(author_id) author_name = author.name.replace('|', ',') author_info = None @@ -705,12 +747,14 @@ def render_author_books(page, author_id, order): def render_publisher_books(page, book_id, order): - publisher = db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first() + publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first() if publisher: - entries, random, pagination = fill_indexpage(page, db.Books, - db.Books.publishers.any(db.Publishers.id == book_id), - [db.Series.name, order[0], db.Books.series_index], - db.books_series_link, db.Series) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.publishers.any(db.Publishers.id == book_id), + [db.Series.name, order[0], db.Books.series_index], + db.books_series_link, + db.Series) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher") else: @@ -718,10 +762,12 @@ def render_publisher_books(page, book_id, order): def render_series_books(page, book_id, order): - name = db.session.query(db.Series).filter(db.Series.id == book_id).first() + name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first() if name: - entries, random, pagination = fill_indexpage(page, db.Books, db.Books.series.any(db.Series.id == book_id), - [db.Books.series_index, order[0]]) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.series.any(db.Series.id == book_id), + [db.Books.series_index, order[0]]) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Series: %(serie)s", serie=name.name), page="series") else: @@ -729,9 +775,11 @@ def render_series_books(page, book_id, order): def render_ratings_books(page, book_id, order): - name = db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first() - entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.id == book_id), - [db.Books.timestamp.desc(), order[0]]) + name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first() + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.ratings.any(db.Ratings.id == book_id), + [db.Books.timestamp.desc(), order[0]]) if name and name.rating <= 10: return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings") @@ -740,11 +788,12 @@ def render_ratings_books(page, book_id, order): def render_formats_books(page, book_id, order): - name = db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first() + name = calibre_db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first() if name: - entries, random, pagination = fill_indexpage(page, db.Books, - db.Books.data.any(db.Data.format == book_id.upper()), - [db.Books.timestamp.desc(), order[0]]) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.data.any(db.Data.format == book_id.upper()), + [db.Books.timestamp.desc(), order[0]]) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"File format: %(format)s", format=name.format), page="formats") else: @@ -752,11 +801,13 @@ def render_formats_books(page, book_id, order): def render_category_books(page, book_id, order): - name = db.session.query(db.Tags).filter(db.Tags.id == book_id).first() + name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first() if name: - entries, random, pagination = fill_indexpage(page, db.Books, db.Books.tags.any(db.Tags.id == book_id), - [order[0], db.Series.name, db.Books.series_index], - db.books_series_link, db.Series) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.tags.any(db.Tags.id == book_id), + [order[0], db.Series.name, db.Books.series_index], + db.books_series_link, db.Series) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, title=_(u"Category: %(name)s", name=name.name), page="category") else: @@ -772,21 +823,29 @@ def render_language_books(page, name, order): lang_name = _(isoLanguages.get(part3=name).name) except KeyError: abort(404) - entries, random, pagination = fill_indexpage(page, db.Books, db.Books.languages.any(db.Languages.lang_code == name), - [db.Books.timestamp.desc(), order[0]]) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db.Books.languages.any(db.Languages.lang_code == name), + [db.Books.timestamp.desc(), order[0]]) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, title=_(u"Language: %(name)s", name=lang_name), page="language") +'''@web.route("/table") +@login_required_if_no_ano +def books_table(): + return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, + title=_(u"Language: %(name)s", name=lang_name), page="language")''' + @web.route("/author") @login_required_if_no_ano def author_list(): if current_user.check_visibility(constants.SIDEBAR_AUTHOR): - entries = db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \ - .join(db.books_authors_link).join(db.Books).filter(common_filters()) \ + entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \ + .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).all() - charlist = db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \ - .join(db.books_authors_link).join(db.Books).filter(common_filters()) \ + charlist = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \ + .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all() for entry in entries: entry.Authors.name = entry.Authors.name.replace('|', ',') @@ -800,11 +859,11 @@ def author_list(): @login_required_if_no_ano def publisher_list(): if current_user.check_visibility(constants.SIDEBAR_PUBLISHER): - entries = db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \ - .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \ + entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \ + .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.name).all() - charlist = db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ - .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \ + charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ + .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Publishers"), page="publisherlist", data="publisher") @@ -816,14 +875,25 @@ def publisher_list(): @login_required_if_no_ano def series_list(): if current_user.check_visibility(constants.SIDEBAR_SERIES): - entries = db.session.query(db.Series, func.count('books_series_link.book').label('count')) \ - .join(db.books_series_link).join(db.Books).filter(common_filters()) \ - .group_by(text('books_series_link.series')).order_by(db.Series.sort).all() - charlist = db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \ - .join(db.books_series_link).join(db.Books).filter(common_filters()) \ - .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() - return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, - title=_(u"Series"), page="serieslist", data="series") + if current_user.series_view == 'list': + entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \ + .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ + .group_by(text('books_series_link.series')).order_by(db.Series.sort).all() + charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \ + .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ + .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, + title=_(u"Series"), page="serieslist", data="series") + else: + entries = calibre_db.session.query(db.Books, func.count('books_series_link').label('count')) \ + .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters()) \ + .group_by(text('books_series_link.series')).order_by(db.Series.sort).all() + charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \ + .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ + .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() + + return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, + title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view") else: abort(404) @@ -832,9 +902,9 @@ def series_list(): @login_required_if_no_ano def ratings_list(): if current_user.check_visibility(constants.SIDEBAR_RATING): - entries = db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'), + entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'), (db.Ratings.rating / 2).label('name')) \ - .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \ + .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all() return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(), title=_(u"Ratings list"), page="ratingslist", data="ratings") @@ -846,8 +916,10 @@ def ratings_list(): @login_required_if_no_ano def formats_list(): if current_user.check_visibility(constants.SIDEBAR_FORMAT): - entries = db.session.query(db.Data, func.count('data.book').label('count'), db.Data.format.label('format')) \ - .join(db.Books).filter(common_filters()) \ + entries = calibre_db.session.query(db.Data, + func.count('data.book').label('count'), + db.Data.format.label('format')) \ + .join(db.Books).filter(calibre_db.common_filters()) \ .group_by(db.Data.format).order_by(db.Data.format).all() return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(), title=_(u"File formats list"), page="formatslist", data="formats") @@ -861,20 +933,20 @@ def language_overview(): if current_user.check_visibility(constants.SIDEBAR_LANGUAGE): charlist = list() if current_user.filter_language() == u"all": - languages = speaking_language() + languages = calibre_db.speaking_language() # ToDo: generate first character list for languages else: try: cur_l = LC.parse(current_user.filter_language()) except UnknownLocaleError: cur_l = None - languages = db.session.query(db.Languages).filter( + languages = calibre_db.session.query(db.Languages).filter( db.Languages.lang_code == current_user.filter_language()).all() if cur_l: languages[0].name = cur_l.get_language_name(get_locale()) else: languages[0].name = _(isoLanguages.get(part3=languages[0].lang_code).name) - lang_counter = db.session.query(db.books_languages_link, + lang_counter = calibre_db.session.query(db.books_languages_link, func.count('books_languages_link.book').label('bookcount')).group_by( text('books_languages_link.lang_code')).all() return render_title_template('languages.html', languages=languages, lang_counter=lang_counter, @@ -888,11 +960,11 @@ def language_overview(): @login_required_if_no_ano def category_list(): if current_user.check_visibility(constants.SIDEBAR_CATEGORY): - entries = db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \ - .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(common_filters()) \ + entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \ + .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(calibre_db.common_filters()) \ .group_by(text('books_tags_link.tag')).all() - charlist = db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \ - .join(db.books_tags_link).join(db.Books).filter(common_filters()) \ + charlist = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \ + .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all() return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Categories"), page="catlist", data="category") @@ -912,21 +984,21 @@ def get_tasks_status(): return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") -# ################################### Search functions ################################################################ - @app.route("/reconnect") def reconnect(): - db.reconnect_db(config) + db.reconnect_db(config, ub.app_DB_path) return json.dumps({}) +# ################################### Search functions ################################################################ + + @web.route("/search", methods=["GET"]) @login_required_if_no_ano def search(): term = request.args.get("query") if term: - term.strip().lower() - entries = get_search_results(term) + entries = calibre_db.get_search_results(term) ids = list() for element in entries: ids.append(element.id) @@ -948,9 +1020,9 @@ def search(): @login_required_if_no_ano def advanced_search(): # Build custom columns names - cc = get_cc_columns() - db.session.connection().connection.connection.create_function("lower", 1, lcase) - q = db.session.query(db.Books).filter(common_filters()).order_by(db.Books.sort) + cc = get_cc_columns(filter_config_custom_read=True) + calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) + q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(db.Books.sort) include_tag_inputs = request.args.getlist('include_tag') exclude_tag_inputs = request.args.getlist('exclude_tag') @@ -1003,13 +1075,13 @@ def advanced_search(): format='medium', locale=get_locale())]) except ValueError: pub_start = u"" - tag_names = db.session.query(db.Tags).filter(db.Tags.id.in_(include_tag_inputs)).all() + tag_names = calibre_db.session.query(db.Tags).filter(db.Tags.id.in_(include_tag_inputs)).all() searchterm.extend(tag.name for tag in tag_names) - serie_names = db.session.query(db.Series).filter(db.Series.id.in_(include_series_inputs)).all() + serie_names = calibre_db.session.query(db.Series).filter(db.Series.id.in_(include_series_inputs)).all() searchterm.extend(serie.name for serie in serie_names) - language_names = db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all() + language_names = calibre_db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all() if language_names: - language_names = speaking_language(language_names) + language_names = calibre_db.speaking_language(language_names) searchterm.extend(language.name for language in language_names) if rating_high: searchterm.extend([_(u"Rating <= %(rating)s", rating=rating_high)]) @@ -1064,13 +1136,16 @@ def advanced_search(): # search custom culumns for c in cc: custom_query = request.args.get('custom_column_' + str(c.id)) - if custom_query: + if custom_query != '' and custom_query is not None: if c.datatype == 'bool': q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( db.cc_classes[c.id].value == (custom_query == "True"))) - elif c.datatype == 'int': + elif c.datatype == 'int' or c.datatype == 'float': q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( db.cc_classes[c.id].value == custom_query)) + elif c.datatype == 'rating': + q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( + db.cc_classes[c.id].value == int(custom_query) * 2)) else: q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%"))) @@ -1082,15 +1157,26 @@ def advanced_search(): return render_title_template('search.html', adv_searchterm=searchterm, entries=q, title=_(u"search"), page="search") # prepare data for search-form - tags = db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters()) \ - .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).all() - series = db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters()) \ - .group_by(text('books_series_link.series')).order_by(db.Series.name).filter(common_filters()).all() - extensions = db.session.query(db.Data).join(db.Books).filter(common_filters()) \ - .group_by(db.Data.format).order_by(db.Data.format).all() - + tags = calibre_db.session.query(db.Tags)\ + .join(db.books_tags_link)\ + .join(db.Books)\ + .filter(calibre_db.common_filters()) \ + .group_by(text('books_tags_link.tag'))\ + .order_by(db.Tags.name).all() + series = calibre_db.session.query(db.Series)\ + .join(db.books_series_link)\ + .join(db.Books)\ + .filter(calibre_db.common_filters()) \ + .group_by(text('books_series_link.series'))\ + .order_by(db.Series.name)\ + .filter(calibre_db.common_filters()).all() + extensions = calibre_db.session.query(db.Data)\ + .join(db.Books)\ + .filter(calibre_db.common_filters()) \ + .group_by(db.Data.format)\ + .order_by(db.Data.format).all() if current_user.filter_language() == u"all": - languages = speaking_language() + languages = calibre_db.speaking_language() else: languages = None return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions, @@ -1100,30 +1186,35 @@ def advanced_search(): def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs): order = order or [] if not config.config_read_column: - readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\ - .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all() - readBookIds = [x.book_id for x in readBooks] if are_read: - db_filter = db.Books.id.in_(readBookIds) + db_filter = and_(ub.ReadBook.user_id == int(current_user.id), + ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED) else: - db_filter = ~db.Books.id.in_(readBookIds) - entries, random, pagination = fill_indexpage(page, db.Books, db_filter, order) + db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db_filter, + order, + ub.ReadBook, db.Books.id==ub.ReadBook.book_id) else: try: if are_read: db_filter = db.cc_classes[config.config_read_column].value == True else: db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True - # book_count = db.session.query(func.count(db.Books.id)).filter(common_filters()).filter(db_filter).scalar() - entries, random, pagination = fill_indexpage(page, db.Books, - db_filter, - order, - db.cc_classes[config.config_read_column]) + entries, random, pagination = calibre_db.fill_indexpage(page, + db.Books, + db_filter, + order, + db.cc_classes[config.config_read_column]) except KeyError: log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column) - book_count = 0 - - + if not as_xml: + flash(_("Custom Column No.%(column)d is not existing in calibre database", + column=config.config_read_column), + category="error") + return redirect(url_for("web.index")) + # ToDo: Handle error Case for opds if as_xml: return entries, pagination else: @@ -1149,8 +1240,11 @@ def render_archived_books(page, order): archived_filter = db.Books.id.in_(archived_book_ids) - entries, random, pagination = fill_indexpage_with_archived_books(page, db.Books, archived_filter, order, - allow_show_archived=True) + entries, random, pagination = calibre_db.fill_indexpage_with_archived_books(page, + db.Books, + archived_filter, + order, + allow_show_archived=True) name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" @@ -1172,9 +1266,8 @@ def get_cover(book_id): @viewer_required def serve_book(book_id, book_format, anyname): book_format = book_format.split(".")[0] - book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == book_format.upper()) \ - .first() + book = calibre_db.get_book(book_id) + data = calibre_db.get_book_format(book.id, book_format.upper()) log.info('Serving book: %s', data.name) if config.config_use_google_drive: headers = Headers() @@ -1190,7 +1283,12 @@ def serve_book(book_id, book_format, anyname): @login_required_if_no_ano @download_required def download_link(book_id, book_format, anyname): - return get_download_link(book_id, book_format.lower()) + if "Kobo" in request.headers.get('User-Agent'): + client = "kobo" + else: + client="" + + return get_download_link(book_id, book_format, client) @web.route('/send///') @@ -1231,30 +1329,33 @@ def register(): if request.method == "POST": to_save = request.form.to_dict() - if not to_save["nickname"] or not to_save["email"]: + if config.config_register_email: + nickname = to_save["email"] + else: + nickname = to_save["nickname"] + if not nickname or not to_save["email"]: flash(_(u"Please fill out all fields!"), category="error") return render_title_template('register.html', title=_(u"register"), page="register") - existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"] + + existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == nickname .lower()).first() existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()).first() if not existing_user and not existing_email: content = ub.User() - # content.password = generate_password_hash(to_save["password"]) if check_valid_domain(to_save["email"]): - content.nickname = to_save["nickname"] + content.nickname = nickname content.email = to_save["email"] password = generate_random_password() content.password = generate_password_hash(password) content.role = config.config_default_role content.sidebar_view = config.config_default_show - # content.mature_content = bool(config.config_default_show & constants.MATURE_CONTENT) try: ub.session.add(content) ub.session.commit() if feature_support['oauth']: register_user_with_oauth(content) - send_registration_mail(to_save["email"], to_save["nickname"], password) + send_registration_mail(to_save["email"], nickname, password) except Exception: ub.session.rollback() flash(_(u"An unknown error occurred. Please try again later."), category="error") @@ -1448,7 +1549,7 @@ def token_verified(): @login_required def profile(): downloads = list() - languages = speaking_language() + languages = calibre_db.speaking_language() translations = babel.list_translations() + [LC('en')] kobo_support = feature_support['kobo'] and config.config_kobo_sync if feature_support['oauth']: @@ -1457,9 +1558,9 @@ def profile(): oauth_status = None for book in current_user.downloads: - downloadBook = db.session.query(db.Books).filter(db.Books.id == book.book_id).first() + downloadBook = calibre_db.get_book(book.book_id) if downloadBook: - downloads.append(db.session.query(db.Books).filter(db.Books.id == book.book_id).first()) + downloads.append(downloadBook) else: ub.delete_download(book.book_id) if request.method == "POST": @@ -1538,7 +1639,7 @@ def profile(): @login_required_if_no_ano @viewer_required def read_book(book_id, book_format): - book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() + book = calibre_db.get_filtered_book(book_id) if not book: flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error") log.debug(u"Error opening eBook. File does not exist or file is not accessible:") @@ -1562,7 +1663,7 @@ def read_book(book_id, book_format): else: for fileExt in constants.EXTENSIONS_AUDIO: if book_format.lower() == fileExt: - entries = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() + entries = calibre_db.get_filtered_book(book_id) log.debug(u"Start mp3 listening for %d", book_id) return render_title_template('listenmp3.html', mp3file=book_id, audioformat=book_format.lower(), title=_(u"Read a Book"), entry=entries, bookmark=bookmark) @@ -1588,8 +1689,7 @@ def read_book(book_id, book_format): @web.route("/book/") @login_required_if_no_ano def show_book(book_id): - entries = db.session.query(db.Books).filter(and_(db.Books.id == book_id, - common_filters(allow_show_archived=True))).first() + entries = calibre_db.get_filtered_book(book_id, allow_show_archived=True) if entries: for index in range(0, len(entries.languages)): try: @@ -1598,7 +1698,7 @@ def show_book(book_id): except UnknownLocaleError: entries.languages[index].language_name = _( isoLanguages.get(part3=entries.languages[index].lang_code).name) - cc = get_cc_columns() + cc = get_cc_columns(filter_config_custom_read=True) book_in_shelfs = [] shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() for entry in shelfs: @@ -1629,7 +1729,7 @@ def show_book(book_id): entries.tags = sort(entries.tags, key=lambda tag: tag.name) - entries = order_authors(entries) + entries = calibre_db.order_authors(entries) kindle_list = check_send_to_kindle(entries) reader_list = check_read_formats(entries) diff --git a/cps/worker.py b/cps/worker.py index dabec3e2..bc7dc32b 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -24,6 +24,9 @@ import smtplib import socket import time import threading +import queue +from glob import glob +from shutil import copyfile from datetime import datetime try: @@ -43,9 +46,10 @@ from email.utils import make_msgid from email.generator import Generator from flask_babel import gettext as _ -from . import logger, config, db, gdriveutils +from . import calibre_db, db +from . import logger, config from .subproc_wrapper import process_open - +from . import gdriveutils log = logger.create() @@ -187,6 +191,8 @@ class WorkerThread(threading.Thread): self.UIqueue = list() self.asyncSMTP = None self.id = 0 + self.db_queue = queue.Queue() + calibre_db.add_queue(self.db_queue) self.doLock = threading.Lock() # Main thread loop starting the different tasks @@ -273,7 +279,7 @@ class WorkerThread(threading.Thread): index = self.current self.doLock.release() file_path = self.queue[index]['file_path'] - bookid = self.queue[index]['bookid'] + book_id = self.queue[index]['bookid'] format_old_ext = u'.' + self.queue[index]['settings']['old_book_format'].lower() format_new_ext = u'.' + self.queue[index]['settings']['new_book_format'].lower() @@ -281,95 +287,51 @@ class WorkerThread(threading.Thread): # if it does - mark the conversion task as complete and return a success # this will allow send to kindle workflow to continue to work if os.path.isfile(file_path + format_new_ext): - log.info("Book id %d already converted to %s", bookid, format_new_ext) - cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first() + log.info("Book id %d already converted to %s", book_id, format_new_ext) + cur_book = calibre_db.get_book(book_id) self.queue[index]['path'] = file_path self.queue[index]['title'] = cur_book.title self._handleSuccess() return file_path + format_new_ext else: - log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", bookid, format_new_ext) + log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", + book_id, + format_new_ext) - # check if converter-executable is existing - if not os.path.exists(config.config_converterpath): - # ToDo Text is not translated - self._handleError(u"Convertertool %s not found" % config.config_converterpath) - return - - try: - # check which converter to use kindlegen is "1" - if format_old_ext == '.epub' and format_new_ext == '.mobi': - if config.config_ebookconverter == 1: - command = [config.config_converterpath, file_path + u'.epub'] - quotes = [1] - if config.config_ebookconverter == 2: - # Linux py2.7 encode as list without quotes no empty element for parameters - # linux py3.x no encode and as list without quotes no empty element for parameters - # windows py2.7 encode as string with quotes empty element for parameters is okay - # windows py 3.x no encode and as string with quotes empty element for parameters is okay - # separate handling for windows and linux - quotes = [1,2] - command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] - quotes_index = 3 - if config.config_calibre: - parameters = config.config_calibre.split(" ") - for param in parameters: - command.append(param) - quotes.append(quotes_index) - quotes_index += 1 - p = process_open(command, quotes) - # p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) - except OSError as e: - self._handleError(_(u"Ebook-converter failed: %(error)s", error=e)) - return - - if config.config_ebookconverter == 1: - nextline = p.communicate()[0] - # Format of error message (kindlegen translates its output texts): - # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. - conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) - # If error occoures, store error message for logfile - if conv_error: - error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", - error=conv_error.group(1), message=conv_error.group(2).strip()) - log.debug("convert_kindlegen: %s", nextline) + if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub': + check, error_message = self._convert_kepubify(file_path, + format_old_ext, + format_new_ext, + index) else: - while p.poll() is None: - nextline = p.stdout.readline() - if os.name == 'nt' and sys.version_info < (3, 0): - nextline = nextline.decode('windows-1252') - elif os.name == 'posix' and sys.version_info < (3, 0): - nextline = nextline.decode('utf-8') - log.debug(nextline.strip('\r\n')) - # parse progress string from calibre-converter - progress = re.search(r"(\d+)%\s.*", nextline) - if progress: - self.UIqueue[index]['progress'] = progress.group(1) + ' %' + # check if calibre converter-executable is existing + if not os.path.exists(config.config_converterpath): + # ToDo Text is not translated + self._handleError(_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) + return + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, index) - # process returncode - check = p.returncode - calibre_traceback = p.stderr.readlines() - for ele in calibre_traceback: - if sys.version_info < (3, 0): - ele = ele.decode('utf-8') - log.debug(ele.strip('\n')) - if not ele.startswith('Traceback') and not ele.startswith(' File'): - error_message = "Calibre failed with error: %s" % ele.strip('\n') - - # kindlegen returncodes - # 0 = Info(prcgen):I1036: Mobi file built successfully - # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS! - # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors! - if (check < 2 and config.config_ebookconverter == 1) or \ - (check == 0 and config.config_ebookconverter == 2): - cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first() + if check == 0: + cur_book = calibre_db.get_book(book_id) if os.path.isfile(file_path + format_new_ext): + # self.db_queue.join() new_format = db.Data(name=cur_book.data[0].name, book_format=self.queue[index]['settings']['new_book_format'].upper(), - book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - cur_book.data.append(new_format) - db.session.commit() + book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + task = {'task':'add_format','id': book_id, 'format': new_format} + self.db_queue.put(task) + # To Do how to handle error? + + '''cur_book.data.append(new_format) + try: + # db.session.merge(cur_book) + calibre_db.session.commit() + except OperationalError as e: + calibre_db.session.rollback() + log.error("Database error: %s", e) + self._handleError(_(u"Database error: %(error)s.", error=e)) + return''' + self.queue[index]['path'] = cur_book.path self.queue[index]['title'] = cur_book.title if config.config_use_google_drive: @@ -385,6 +347,87 @@ class WorkerThread(threading.Thread): return + def _convert_calibre(self, file_path, format_old_ext, format_new_ext, index): + try: + # Linux py2.7 encode as list without quotes no empty element for parameters + # linux py3.x no encode and as list without quotes no empty element for parameters + # windows py2.7 encode as string with quotes empty element for parameters is okay + # windows py 3.x no encode and as string with quotes empty element for parameters is okay + # separate handling for windows and linux + quotes = [1, 2] + command = [config.config_converterpath, (file_path + format_old_ext), + (file_path + format_new_ext)] + quotes_index = 3 + if config.config_calibre: + parameters = config.config_calibre.split(" ") + for param in parameters: + command.append(param) + quotes.append(quotes_index) + quotes_index += 1 + + p = process_open(command, quotes) + except OSError as e: + return 1, _(u"Ebook-converter failed: %(error)s", error=e) + + while p.poll() is None: + nextline = p.stdout.readline() + if os.name == 'nt' and sys.version_info < (3, 0): + nextline = nextline.decode('windows-1252') + elif os.name == 'posix' and sys.version_info < (3, 0): + nextline = nextline.decode('utf-8') + log.debug(nextline.strip('\r\n')) + # parse progress string from calibre-converter + progress = re.search(r"(\d+)%\s.*", nextline) + if progress: + self.UIqueue[index]['progress'] = progress.group(1) + ' %' + + # process returncode + check = p.returncode + calibre_traceback = p.stderr.readlines() + error_message = "" + for ele in calibre_traceback: + if sys.version_info < (3, 0): + ele = ele.decode('utf-8') + log.debug(ele.strip('\n')) + if not ele.startswith('Traceback') and not ele.startswith(' File'): + error_message = "Calibre failed with error: %s" % ele.strip('\n') + return check, error_message + + + def _convert_kepubify(self, file_path, format_old_ext, format_new_ext, index): + quotes = [1, 3] + command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)] + try: + p = process_open(command, quotes) + except OSError as e: + return 1, _(u"Kepubify-converter failed: %(error)s", error=e) + self.UIqueue[index]['progress'] = '1 %' + while True: + nextline = p.stdout.readlines() + nextline = [x.strip('\n') for x in nextline if x != '\n'] + if sys.version_info < (3, 0): + nextline = [x.decode('utf-8') for x in nextline] + for line in nextline: + log.debug(line) + if p.poll() is not None: + break + + # ToD Handle + # process returncode + check = p.returncode + + # move file + if check == 0: + converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub")) + if len(converted_file) == 1: + copyfile(converted_file[0], (file_path + format_new_ext)) + os.unlink(converted_file[0]) + else: + return 1, _(u"Converted file not found or more than one file in folder %(folder)s", + folder=os.path.dirname(file_path)) + return check, None + + def add_convert(self, file_path, bookid, user_name, taskMessage, settings, kindle_mail=None): self.doLock.acquire() if self.last >= 20: @@ -533,10 +576,6 @@ class WorkerThread(threading.Thread): self.UIqueue[index]['formRuntime'] = datetime.now() - self.queue[index]['starttime'] -_worker = WorkerThread() -_worker.start() - - def get_taskstatus(): return _worker.get_taskstatus() @@ -551,3 +590,7 @@ def add_upload(user_name, taskMessage): def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None): return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail) + + +_worker = WorkerThread() +_worker.start() diff --git a/optional-requirements.txt b/optional-requirements.txt index d830e700..5cef4a05 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -31,7 +31,7 @@ rarfile>=2.7 # other natsort>=2.2.0,<7.1.0 -git+https://github.com/OzzieIsaacs/comicapi.git@15dff9ce4e1ffed29ba4a2feadfcdb6bed00bcad#egg=comicapi +git+https://github.com/OzzieIsaacs/comicapi.git@3e15b950b72724b1b8ca619c36580b5fbaba9784#egg=comicapi #Kobo integration jsonschema>=3.2.0,<3.3.0 diff --git a/test/Calibre-Web TestSummary.html b/test/Calibre-Web TestSummary.html index 1a9fb744..754a0525 100755 --- a/test/Calibre-Web TestSummary.html +++ b/test/Calibre-Web TestSummary.html @@ -36,17 +36,17 @@
-

Start Time: 2020-05-10 13:45:56

+

Start Time: 2020-05-25 20:05:46

-

Stop Time: 2020-05-10 14:42:12

+

Stop Time: 2020-05-25 21:04:57

-

Duration: 47:37 min

+

Duration: 50:9 min

@@ -100,13 +100,13 @@ test_anonymous.test_anonymous - 12 - 12 + 13 + 13 0 0 0 - Detail + Detail @@ -114,7 +114,7 @@ -
test_guest_about
+
test_check_locale_guest
PASS @@ -123,7 +123,7 @@ -
test_guest_change_visibility_category
+
test_guest_about
PASS @@ -132,7 +132,7 @@ -
test_guest_change_visibility_format
+
test_guest_change_visibility_category
PASS @@ -141,7 +141,7 @@ -
test_guest_change_visibility_hot
+
test_guest_change_visibility_format
PASS @@ -150,7 +150,7 @@ -
test_guest_change_visibility_language
+
test_guest_change_visibility_hot
PASS @@ -159,7 +159,7 @@ -
test_guest_change_visibility_publisher
+
test_guest_change_visibility_language
PASS @@ -168,7 +168,7 @@ -
test_guest_change_visibility_rated
+
test_guest_change_visibility_publisher
PASS @@ -177,7 +177,7 @@ -
test_guest_change_visibility_rating
+
test_guest_change_visibility_rated
PASS @@ -186,7 +186,7 @@ -
test_guest_change_visibility_series
+
test_guest_change_visibility_rating
PASS @@ -195,7 +195,7 @@ -
test_guest_random_books_available
+
test_guest_change_visibility_series
PASS @@ -204,7 +204,7 @@ -
test_guest_restricted_settings_visibility
+
test_guest_random_books_available
PASS @@ -212,6 +212,15 @@ + +
test_guest_restricted_settings_visibility
+ + PASS + + + + +
test_guest_visibility_sidebar
@@ -223,13 +232,13 @@ test_cli.test_cli + 7 6 - 5 0 0 1 - Detail + Detail @@ -246,7 +255,7 @@ -
test_cli_SSL_files
+
test_bind_to_single_interface
PASS @@ -255,7 +264,7 @@ -
test_cli_different_folder
+
test_cli_SSL_files
PASS @@ -263,6 +272,15 @@ + +
test_cli_different_folder
+ + PASS + + + + +
test_cli_different_settings_database
@@ -271,19 +289,19 @@ - +
test_cli_gdrive_location
- SKIP + SKIP
-