From 8e8486497fb702583deaca13861e8f3999fa746e Mon Sep 17 00:00:00 2001 From: tomjmul Date: Tue, 19 Mar 2019 12:57:29 +0000 Subject: [PATCH 001/688] Restirct a user to a set of tags --- .gitignore | 1 + cps/templates/user_edit.html | 5 +++++ cps/translations/de/LC_MESSAGES/messages.po | 5 +++++ cps/translations/es/LC_MESSAGES/messages.po | 4 ++++ cps/translations/fr/LC_MESSAGES/messages.po | 4 ++++ cps/translations/hu/LC_MESSAGES/messages.po | 4 ++++ cps/translations/it/LC_MESSAGES/messages.po | 4 ++++ cps/translations/ja/LC_MESSAGES/messages.po | 4 ++++ cps/translations/km/LC_MESSAGES/messages.po | 4 ++++ cps/translations/nl/LC_MESSAGES/messages.po | 4 ++++ cps/translations/pl/LC_MESSAGES/messages.po | 4 ++++ cps/translations/ru/LC_MESSAGES/messages.po | 4 ++++ cps/translations/sv/LC_MESSAGES/messages.po | 4 ++++ cps/translations/uk/LC_MESSAGES/messages.po | 4 ++++ .../zh_Hans_CN/LC_MESSAGES/messages.po | 4 ++++ cps/ub.py | 1 + cps/web.py | 21 +++++++++++++++++-- 17 files changed, 79 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 09bf3faa..cd822a75 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ gdrive_credentials vendor client_secrets.json +venv/ diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index 79de265f..f25d7710 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -44,6 +44,11 @@ {% endfor %} +
+ + +
+
diff --git a/cps/translations/de/LC_MESSAGES/messages.po b/cps/translations/de/LC_MESSAGES/messages.po index d4ee9a09..72c3cb3f 100644 --- a/cps/translations/de/LC_MESSAGES/messages.po +++ b/cps/translations/de/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "Zeige nur Bücher mit dieser Sprache" msgid "Show all" msgstr "Zeige alle" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Zulässige Tags" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Benutzer löschen" @@ -2005,3 +2009,4 @@ msgstr "Benutzer löschen" msgid "Recent Downloads" msgstr "Letzte Downloads" + diff --git a/cps/translations/es/LC_MESSAGES/messages.po b/cps/translations/es/LC_MESSAGES/messages.po index c3803984..fc00ba33 100644 --- a/cps/translations/es/LC_MESSAGES/messages.po +++ b/cps/translations/es/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "Mostrar libros con idioma" msgid "Show all" msgstr "Mostrar todo" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Etiquetas permitidas" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Borrar este usuario" diff --git a/cps/translations/fr/LC_MESSAGES/messages.po b/cps/translations/fr/LC_MESSAGES/messages.po index 99d32f30..4ca6b4c4 100644 --- a/cps/translations/fr/LC_MESSAGES/messages.po +++ b/cps/translations/fr/LC_MESSAGES/messages.po @@ -2010,6 +2010,10 @@ msgstr "Montrer les livres dans la langue" msgid "Show all" msgstr "Montrer tout" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Tags autorisés" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Supprimer cet utilisateur" diff --git a/cps/translations/hu/LC_MESSAGES/messages.po b/cps/translations/hu/LC_MESSAGES/messages.po index 6365bc0c..04e0b066 100644 --- a/cps/translations/hu/LC_MESSAGES/messages.po +++ b/cps/translations/hu/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "Mutasd a könyveket a következő nyelvvel" msgid "Show all" msgstr "Mindent mutass" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Engedélyezett címkék" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "A felhasználó törlése" diff --git a/cps/translations/it/LC_MESSAGES/messages.po b/cps/translations/it/LC_MESSAGES/messages.po index f1c07611..1c48cecc 100644 --- a/cps/translations/it/LC_MESSAGES/messages.po +++ b/cps/translations/it/LC_MESSAGES/messages.po @@ -1996,6 +1996,10 @@ msgstr "Mostra libri per lingua" msgid "Show all" msgstr "Mostra tutto" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Tag consentiti" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Elimina questo utente" diff --git a/cps/translations/ja/LC_MESSAGES/messages.po b/cps/translations/ja/LC_MESSAGES/messages.po index 02d209d5..44352129 100644 --- a/cps/translations/ja/LC_MESSAGES/messages.po +++ b/cps/translations/ja/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "言語で本を表示する" msgid "Show all" msgstr "全て表示" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "許可されたタグ" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "このユーザを削除する" diff --git a/cps/translations/km/LC_MESSAGES/messages.po b/cps/translations/km/LC_MESSAGES/messages.po index 72317e95..f69209bf 100644 --- a/cps/translations/km/LC_MESSAGES/messages.po +++ b/cps/translations/km/LC_MESSAGES/messages.po @@ -1998,6 +1998,10 @@ msgstr "បង្ហាញសៀវភៅដែលមានភាសា" msgid "Show all" msgstr "បង្ហាញទាំងអស់" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "ស្លាកដែលបានអនុញ្ញាត" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "លុបអ្នកប្រើប្រាស់នេះ" diff --git a/cps/translations/nl/LC_MESSAGES/messages.po b/cps/translations/nl/LC_MESSAGES/messages.po index b62e0cd8..80d877c1 100644 --- a/cps/translations/nl/LC_MESSAGES/messages.po +++ b/cps/translations/nl/LC_MESSAGES/messages.po @@ -1998,6 +1998,10 @@ msgstr "Toon boeken met taal" msgid "Show all" msgstr "Toon alles" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Toegestane tags" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Wis deze gebruiker" diff --git a/cps/translations/pl/LC_MESSAGES/messages.po b/cps/translations/pl/LC_MESSAGES/messages.po index b905f11a..6a3c8e20 100644 --- a/cps/translations/pl/LC_MESSAGES/messages.po +++ b/cps/translations/pl/LC_MESSAGES/messages.po @@ -2000,6 +2000,10 @@ msgstr "Pokaż książki w języku" msgid "Show all" msgstr "Pokaż wszystko" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Dozwolone tagi" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Usuń tego użytkownika" diff --git a/cps/translations/ru/LC_MESSAGES/messages.po b/cps/translations/ru/LC_MESSAGES/messages.po index 50fe62ff..33ffb087 100644 --- a/cps/translations/ru/LC_MESSAGES/messages.po +++ b/cps/translations/ru/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "Показать книги на языках" msgid "Show all" msgstr "Показать все" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Разрешенные теги" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Удалить этого пользователя" diff --git a/cps/translations/sv/LC_MESSAGES/messages.po b/cps/translations/sv/LC_MESSAGES/messages.po index 39fb8c45..6631d115 100644 --- a/cps/translations/sv/LC_MESSAGES/messages.po +++ b/cps/translations/sv/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "Visa böcker med språk" msgid "Show all" msgstr "Visa alla" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Tillåten etiketter" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Ta bort den här användaren" diff --git a/cps/translations/uk/LC_MESSAGES/messages.po b/cps/translations/uk/LC_MESSAGES/messages.po index 76a6234b..de5d8d00 100644 --- a/cps/translations/uk/LC_MESSAGES/messages.po +++ b/cps/translations/uk/LC_MESSAGES/messages.po @@ -1995,6 +1995,10 @@ msgstr "Показувати книги на мовах" msgid "Show all" msgstr "Показати всі" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "Дозволені теги" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "Видалити цього користувача" diff --git a/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po b/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po index 1af2fc34..2a5b881a 100644 --- a/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po +++ b/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po @@ -1997,6 +1997,10 @@ msgstr "按语言显示书籍" msgid "Show all" msgstr "显示全部" +#: cps/templates/user_edit.html:48 +msgid "Allowed Tags" +msgstr "允许的标签" + #: cps/templates/user_edit.html:141 msgid "Delete this user" msgstr "删除此用户" diff --git a/cps/ub.py b/cps/ub.py index 4b69a457..58042220 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -192,6 +192,7 @@ class User(UserBase, Base): sidebar_view = Column(Integer, default=1) default_language = Column(String(3), default="all") mature_content = Column(Boolean, default=True) + allowed_tags = Column(String) # Class for anonymous user is derived from User base and completly overrides methods and properties for the diff --git a/cps/web.py b/cps/web.py index d59b6ba8..6a7a4321 100644 --- a/cps/web.py +++ b/cps/web.py @@ -497,6 +497,12 @@ def edit_required(f): return inner +def allowed_tags(tags): + if sys.version_info > (3, 0): # Python3 str, Python2 unicode + lstrip = str.lstrip + else: + lstrip = unicode.lstrip + return list(map(lstrip, tags.split(","))) # Language and content filters for displaying in the UI def common_filters(): @@ -504,9 +510,14 @@ def common_filters(): lang_filter = db.Books.languages.any(db.Languages.lang_code == current_user.filter_language()) else: lang_filter = true() + if current_user.allowed_tags is not None and current_user.allowed_tags != '': + tags_filter = db.Books.tags.any(db.Tags.name.in_(allowed_tags(current_user.allowed_tags))) + else: + tags_filter = true() content_rating_filter = false() if current_user.mature_content else \ db.Books.tags.any(db.Tags.name.in_(config.mature_content_tags())) - return and_(lang_filter, ~content_rating_filter) + + return and_(tags_filter, lang_filter, ~content_rating_filter) # Creates for all stored languages a translated speaking name in the array for the UI @@ -1085,7 +1096,8 @@ def get_comic_book(book_id, book_format, page): def get_authors_json(): if request.method == "GET": query = request.args.get('q') - entries = db.session.query(db.Authors).filter(db.Authors.name.ilike("%" + query + "%")).all() + entries = db.session.query(db.Authors).join(db.books_authors_link).join(db.Books).filter(common_filters())\ + .filter(db.Authors.name.ilike("%" + query + "%")).all() json_dumps = json.dumps([dict(name=r.name.replace('|',',')) for r in entries]) return json_dumps @@ -2710,6 +2722,8 @@ def profile(): content.password = generate_password_hash(to_save["password"]) if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: content.kindle_mail = to_save["kindle_mail"] + if "allowed_tags" in to_save and to_save["allowed_tags"] != content.allowed_tags: + content.allowed_tags = to_save["allowed_tags"].strip() if to_save["email"] and to_save["email"] != content.email: if config.config_public_reg and not check_valid_domain(to_save["email"]): flash(_(u"E-mail is not from valid domain"), category="error") @@ -2872,6 +2886,7 @@ def view_configuration(): content.config_default_show = content.config_default_show + ub.SIDEBAR_SORTED if "show_mature_content" in to_save: content.config_default_show = content.config_default_show + ub.MATURE_CONTENT + ub.session.commit() flash(_(u"Calibre-Web configuration updated"), category="success") config.loadSettings() @@ -3336,6 +3351,8 @@ def edit_user(user_id): content.email = to_save["email"] if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: content.kindle_mail = to_save["kindle_mail"] + if "allowed_tags" in to_save and to_save["allowed_tags"] != content.allowed_tags: + content.allowed_tags = to_save["allowed_tags"] try: ub.session.commit() flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success") From 147947662ca606f26a3ae0a05e61e5e5de4fcf11 Mon Sep 17 00:00:00 2001 From: Krakinou Date: Mon, 17 Jun 2019 23:46:38 +0200 Subject: [PATCH 002/688] Base64 --- cps/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cps/admin.py b/cps/admin.py index f6fd838f..0f553035 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -25,6 +25,7 @@ from __future__ import division, print_function, unicode_literals import os import json import time +import base64 from datetime import datetime, timedelta try: from imp import reload From 3d0beba26172489e68c9cdd9a53ca1d3c6000b0e Mon Sep 17 00:00:00 2001 From: Krakinou Date: Mon, 17 Jun 2019 23:47:35 +0200 Subject: [PATCH 003/688] Base64 --- cps/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/admin.py b/cps/admin.py index 0f553035..c641f3a0 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -25,7 +25,7 @@ from __future__ import division, print_function, unicode_literals import os import json import time -import base64 +import base64 from datetime import datetime, timedelta try: from imp import reload From e5b9da5201d7ed1e597e2a8884fbd5f59b0ffb4b Mon Sep 17 00:00:00 2001 From: Krakinou Date: Mon, 1 Jul 2019 21:44:58 +0200 Subject: [PATCH 004/688] Error management --- cps/web.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/cps/web.py b/cps/web.py index 3f8964d4..1ae1aa86 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1098,19 +1098,15 @@ def login(): flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.nickname), category="success") return redirect_back(url_for("web.index")) - except ldap.ldap.INVALID_CREDENTIALS as e: - log.error('Login Error: ' + str(e)) - ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) - log.info('LDAP Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) - flash(_(u"Wrong Username or Password"), category="error") - except ldap.ldap.SERVER_DOWN: - log.info('LDAP Login failed, LDAP Server down') - flash(_(u"Could not login. LDAP server down, please contact your administrator"), category="error") - '''except LDAPException as exception: - app.logger.error('Login Error: ' + str(exception)) - ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) - app.logger.info('LDAP Login failed for user "' + form['username'] + ', IP-address :' + ipAdress) - flash(_(u"Wrong Username or Password"), category="error")''' + except Exception as exception: + app.logger.info('Login Error: ' + str(exception)) + if str(exception) == 'Invalid credentials': + ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) + app.logger.info('LDAP Login failed for user "' + form['username'] + ', IP-address :' + ipAdress) + flash(_(u"Wrong Username or Password"), category="error") + if str(exception) == 'Server down': + log.info('LDAP Login failed, LDAP Server down') + flash(_(u"Could not login. LDAP server down, please contact your administrator"), category="error") else: if user and check_password_hash(user.password, form['password']) and user.nickname is not "Guest": login_user(user, remember=True) From 00a29f3d8840e285cc17e6d929d7616715612110 Mon Sep 17 00:00:00 2001 From: Krakinou Date: Mon, 1 Jul 2019 21:45:35 +0200 Subject: [PATCH 005/688] Check for change before encoding --- cps/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index c641f3a0..9c08d2d4 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -413,12 +413,13 @@ def configuration_helper(origin): goodreads=goodreads_support, title=_(u"Basic Configuration"), page="config") else: - content.config_use_ldap = 1 + content.config_login_type = 1 content.config_ldap_provider_url = to_save["config_ldap_provider_url"] content.config_ldap_port = to_save["config_ldap_port"] content.config_ldap_schema = to_save["config_ldap_schema"] content.config_ldap_serv_username = to_save["config_ldap_serv_username"] - content.config_ldap_serv_password = base64.b64encode(to_save["config_ldap_serv_password"]) + if content.config_ldap_serv_password != to_save["config_ldap_serv_password"]: + content.config_ldap_serv_password = base64.b64encode(to_save["config_ldap_serv_password"]) content.config_ldap_dn = to_save["config_ldap_dn"] content.config_ldap_user_object = to_save["config_ldap_user_object"] reboot_required = True From 38a255e0696ecf985241e6774237f1ec47365e72 Mon Sep 17 00:00:00 2001 From: Vincent Kriek Date: Sun, 18 Aug 2019 21:44:19 +0200 Subject: [PATCH 006/688] Add automatic epub to kepub conversion using kepubify --- cps/admin.py | 4 ++++ cps/config_sql.py | 4 ++++ cps/helper.py | 18 +++++++++++++++--- cps/templates/config_edit.html | 12 ++++++++++++ cps/web.py | 6 +++++- 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index b6dcf66a..4dcd3eaa 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -303,6 +303,10 @@ def _configuration_update_helper(): _config_string("config_calibre") _config_string("config_converterpath") + _config_checkbox_int("config_automatic_kepub") + _config_string("config_kepubify_path") + _config_string("config_kepub_cache_dir") + if _config_int("config_login_type"): reboot_required |= config.config_login_type != constants.LOGIN_STANDARD diff --git a/cps/config_sql.py b/cps/config_sql.py index 809e97d8..eaefb131 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -104,6 +104,10 @@ class _Settings(_Base): config_calibre = Column(String) config_rarfile_location = Column(String) + config_automatic_kepub = Column(Boolean, default=False) + config_kepubify_path = Column(String) + config_kepub_cache_dir = Column(String) + config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) def __repr__(self): diff --git a/cps/helper.py b/cps/helper.py index e909086e..606a2c7d 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -523,7 +523,7 @@ def save_cover(img, book_path): -def do_download_file(book, book_format, data, headers): +def do_download_file(book, book_format, client, data, headers): if config.config_use_google_drive: startTime = time.time() df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) @@ -537,7 +537,18 @@ def do_download_file(book, book_format, data, headers): if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): # ToDo: improve error handling log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + + if client == "kobo" and book_format == "epub": + filename = config.config_kepub_cache_dir + os.system('{0} "{1}" -o {2}'.format( + config.config_kepubify_path, + os.path.join(filename, data.name + "." + book_format), + filename)) + book_format = "kepub.epub" + headers["Content-Disposition"] = headers["Content-Disposition"].replace(".epub", ".kepub.epub") + response = make_response(send_from_directory(filename, data.name + "." + book_format)) + response.headers = headers return response @@ -756,7 +767,7 @@ def get_cc_columns(): cc = tmpcc return cc -def get_download_link(book_id, book_format): +def get_download_link(book_id, book_format, client): book_format = book_format.split(".")[0] book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() if book: @@ -776,7 +787,8 @@ def get_download_link(book_id, book_format): headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (quote(file_name.encode('utf-8')), book_format) - return do_download_file(book, book_format, data, headers) + + return do_download_file(book, book_format, client, data, headers) else: abort(404) diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 4a5b154d..56d7e996 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -309,6 +309,18 @@
{% endif %} +
+ + +
+
+ + +
+
+ + +
diff --git a/cps/web.py b/cps/web.py index 15f0f976..3b33449d 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1010,7 +1010,11 @@ def serve_book(book_id, book_format): @login_required_if_no_ano @download_required def download_link(book_id, book_format, anyname): - return get_download_link(book_id, book_format) + if (config.config_automatic_kepub and + "Kobo" in request.headers.get('User-Agent')): + client = "kobo" + + return get_download_link(book_id, book_format, client) @web.route('/send///') From d1afdb4aacdc614fd519fe20ced9a1741de484e7 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Thu, 31 Oct 2019 15:44:36 +0100 Subject: [PATCH 007/688] Fix #1074, #1071 --- cps/server.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 cps/server.py diff --git a/cps/server.py b/cps/server.py old mode 100644 new mode 100755 index e5fe78e4..43792ecd --- a/cps/server.py +++ b/cps/server.py @@ -146,6 +146,9 @@ class WebServer(object): self.unix_socket_file = None def _start_tornado(self): + if os.name == 'nt': + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port)) # Max Buffersize set to 200MB ) From 222797e6312a67f1a7eb65b5644010e72c214fb6 Mon Sep 17 00:00:00 2001 From: zhiyue Date: Fri, 22 Nov 2019 00:12:05 +0800 Subject: [PATCH 008/688] support douban book search using apikey --- cps/static/js/get_meta.js | 69 +++++++++++++++++++++++++++--------- cps/templates/book_edit.html | 5 +-- cps/templates/search.html | 3 ++ 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/cps/static/js/get_meta.js b/cps/static/js/get_meta.js index cf079ba7..a453b092 100644 --- a/cps/static/js/get_meta.js +++ b/cps/static/js/get_meta.js @@ -19,15 +19,15 @@ * Google Books api document: https://developers.google.com/books/docs/v1/using * Douban Books api document: https://developers.douban.com/wiki/?title=book_v2 (Chinese Only) */ -/* global _, i18nMsg, tinymce */ -// var dbResults = []; +/* global _, i18nMsg, tinymce */ +var dbResults = []; var ggResults = []; $(function () { var msg = i18nMsg; - /*var douban = "https://api.douban.com"; - var dbSearch = "/v2/book/search";*/ - // var dbDone = true; + var douban = "https://api.douban.com"; + var dbSearch = "/v2/book/search"; + var dbDone = true; var google = "https://www.googleapis.com"; var ggSearch = "/books/v1/volumes"; @@ -43,12 +43,22 @@ $(function () { function populateForm (book) { tinymce.get("description").setContent(book.description); + var uniqueTags = []; + $.each(book.tags, function(i, el) { + if ($.inArray(el, uniqueTags) === -1) uniqueTags.push(el); + }); + $("#bookAuthor").val(book.authors); $("#book_title").val(book.title); - $("#tags").val(book.tags.join(",")); + $("#tags").val(uniqueTags.join(",")); $("#rating").data("rating").setValue(Math.round(book.rating)); $(".cover img").attr("src", book.cover); $("#cover_url").val(book.cover); + $("#pubdate").val(book.publishedDate); + $("#publisher").val(book.publisher) + if (book.series != undefined) { + $("#series").val(book.series) + } } function showResult () { @@ -56,10 +66,24 @@ $(function () { if (showFlag === 1) { $("#meta-info").html("
    "); } - if (!ggDone) { + if (!ggDone && !dbDone) { $("#meta-info").html("

    " + msg.no_result + "

    "); return; } + function formatDate (date) { + var d = new Date(date), + month = '' + (d.getMonth() + 1), + day = '' + d.getDate(), + year = d.getFullYear(); + + if (month.length < 2) + month = '0' + month; + if (day.length < 2) + day = '0' + day; + + return [year, month, day].join('-'); + } + if (ggDone && ggResults.length > 0) { ggResults.forEach(function(result) { var book = { @@ -72,8 +96,7 @@ $(function () { tags: result.volumeInfo.categories || [], rating: result.volumeInfo.averageRating || 0, cover: result.volumeInfo.imageLinks ? - result.volumeInfo.imageLinks.thumbnail : - "/static/generic_cover.jpg", + result.volumeInfo.imageLinks.thumbnail : "/static/generic_cover.jpg", url: "https://books.google.com/books?id=" + result.id, source: { id: "google", @@ -91,19 +114,30 @@ $(function () { }); ggDone = false; } - /*if (dbDone && dbResults.length > 0) { + if (dbDone && dbResults.length > 0) { dbResults.forEach(function(result) { + if (result.series){ + var series_title = result.series.title + } + var date_fomers = result.pubdate.split("-") + var publishedYear = parseInt(date_fomers[0]) + var publishedMonth = parseInt(date_fomers[1]) + var publishedDate = new Date(publishedYear, publishedMonth-1, 1) + + publishedDate = formatDate(publishedDate) + var book = { id: result.id, title: result.title, authors: result.author || [], description: result.summary, publisher: result.publisher || "", - publishedDate: result.pubdate || "", + publishedDate: publishedDate || "", tags: result.tags.map(function(tag) { - return tag.title; + return tag.title.toLowerCase().replace(/,/g, "_"); }), rating: result.rating.average || 0, + series: series_title || "", cover: result.image, url: "https://book.douban.com/subject/" + result.id, source: { @@ -125,7 +159,7 @@ $(function () { $("#book-list").append($book); }); dbDone = false; - }*/ + } } function ggSearchBook (title) { @@ -148,9 +182,10 @@ $(function () { }); } - /*function dbSearchBook (title) { + function dbSearchBook (title) { + apikey="0df993c66c0c636e29ecbb5344252a4a" $.ajax({ - url: douban + dbSearch + "?q=" + title + "&fields=all&count=10", + url: douban + dbSearch + "?apikey=" + apikey + "&q=" + title + "&fields=all&count=10", type: "GET", dataType: "jsonp", jsonp: "callback", @@ -166,13 +201,13 @@ $(function () { $("#show-douban").trigger("change"); } }); - }*/ + } function doSearch (keyword) { showFlag = 0; $("#meta-info").text(msg.loading); if (keyword) { - // dbSearchBook(keyword); + dbSearchBook(keyword); ggSearchBook(keyword); } } diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 60421ea6..e734698a 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -219,8 +219,8 @@
    - + + @@ -288,6 +288,7 @@ {% endblock %} {% block header %} + {% endblock %} diff --git a/cps/templates/search.html b/cps/templates/search.html index dfa42799..c44fbda7 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,4 +1,7 @@ {% extends "layout.html" %} +{% block header %} + +{% endblock %} {% block body %}
    {% if entries|length < 1 %} From 53578671033d57c62835fb863b80d8414a66a771 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Tue, 5 Nov 2019 23:18:52 -0500 Subject: [PATCH 009/688] Add initial support for Kobo device Sync endpoint. - Supports /v1/library/sync call to get list of books - Supports /v1/library/metadata call to get metadata for a given book + Assumes books are stored on Backblaze for metadata call - Changes to helper.py so that we can return no cover instead of a blank image. --- .gitignore | 1 + cps.py | 3 + cps/admin.py | 2 + cps/config_sql.py | 1 + cps/db.py | 30 +- cps/helper.py | 26 +- cps/kobo.py | 582 +++++++++++++++++++++++++++++++++ cps/templates/config_edit.html | 4 + requirements.txt | 2 + 9 files changed, 638 insertions(+), 13 deletions(-) create mode 100644 cps/kobo.py diff --git a/.gitignore b/.gitignore index 0ce14757..5cf14e51 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ gdrive_credentials vendor client_secrets.json +b2_secrets.json diff --git a/cps.py b/cps.py index ca7d7230..412604d2 100755 --- a/cps.py +++ b/cps.py @@ -41,6 +41,8 @@ from cps.shelf import shelf from cps.admin import admi from cps.gdrive import gdrive from cps.editbooks import editbook +from cps.kobo import kobo + try: from cps.oauth_bb import oauth oauth_available = True @@ -58,6 +60,7 @@ def main(): app.register_blueprint(admi) app.register_blueprint(gdrive) app.register_blueprint(editbook) + app.register_blueprint(kobo) if oauth_available: app.register_blueprint(oauth) success = web_server.start() diff --git a/cps/admin.py b/cps/admin.py index ccb07d84..0e30109c 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -294,6 +294,8 @@ def _configuration_update_helper(): reboot_required |= _config_string("config_certfile") if config.config_certfile and not os.path.isfile(config.config_certfile): return _configuration_result('Certfile location is not valid, please enter correct path', gdriveError) + + _config_string("config_server_url") _config_checkbox_int("config_uploading") _config_checkbox_int("config_anonbrowse") diff --git a/cps/config_sql.py b/cps/config_sql.py index 809e97d8..8ea8b978 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -49,6 +49,7 @@ class _Settings(_Base): config_port = Column(Integer, default=constants.DEFAULT_PORT) config_certfile = Column(String) config_keyfile = Column(String) + config_server_url = Column(String, default='') config_calibre_web_title = Column(String, default=u'Calibre-Web') config_books_per_page = Column(Integer, default=60) diff --git a/cps/db.py b/cps/db.py index b9853896..dc280ad8 100755 --- a/cps/db.py +++ b/cps/db.py @@ -25,13 +25,13 @@ import ast from sqlalchemy import create_engine from sqlalchemy import Table, Column, ForeignKey -from sqlalchemy import String, Integer, Boolean +from sqlalchemy import String, Integer, Boolean, TIMESTAMP from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base session = None -cc_exceptions = ['datetime', 'comments', 'float', 'composite', 'series'] +cc_exceptions = ['comments', 'float', 'composite', 'series'] cc_classes = {} @@ -251,10 +251,10 @@ class Books(Base): title = Column(String) sort = Column(String) author_sort = Column(String) - timestamp = Column(String) + timestamp = Column(TIMESTAMP) pubdate = Column(String) series_index = Column(String) - last_modified = Column(String) + last_modified = Column(TIMESTAMP) path = Column(String) has_cover = Column(Integer) uuid = Column(String) @@ -353,7 +353,7 @@ def setup_db(config): # conn.connection.create_function('upper', 1, ucase) if not cc_classes: - cc = conn.execute("SELECT id, datatype FROM custom_columns") + cc = conn.execute("SELECT id, datatype, normalized FROM custom_columns") cc_ids = [] books_custom_column_links = {} @@ -366,7 +366,7 @@ def setup_db(config): ForeignKey('custom_column_' + str(row.id) + '.id'), primary_key=True) ) - cc_ids.append([row.id, row.datatype]) + cc_ids.append([row.id, row.datatype, row.normalized]) if row.datatype == 'bool': ccdict = {'__tablename__': 'custom_column_' + str(row.id), 'id': Column(Integer, primary_key=True), @@ -377,6 +377,11 @@ def setup_db(config): 'id': Column(Integer, primary_key=True), 'book': Column(Integer, ForeignKey('books.id')), 'value': Column(Integer)} + elif not row.normalized: + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(String)} else: ccdict = {'__tablename__': 'custom_column_' + str(row.id), 'id': Column(Integer, primary_key=True), @@ -384,7 +389,8 @@ def setup_db(config): cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict) for cc_id in cc_ids: - if (cc_id[1] == 'bool') or (cc_id[1] == 'int'): + normalized = cc_id[2] + if (not normalized): setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], primaryjoin=( Books.id == cc_classes[cc_id[0]].book), @@ -393,6 +399,16 @@ def setup_db(config): setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], secondary=books_custom_column_links[cc_id[0]], backref='books')) + #for cc_id in cc_ids: + # if (cc_id[1] == 'bool') or (cc_id[1] == 'int'): + # setattr(Books, 'custom_column_' + str(cc_id[2]), relationship(cc_classes[cc_id[0]], + # primaryjoin=( + # Books.id == cc_classes[cc_id[0]].book), + # backref='books')) + # else: + # setattr(Books, 'custom_column_' + str(cc_id[2]), relationship(cc_classes[cc_id[0]], + # secondary=books_custom_column_links[cc_id[0]], + # backref='books')) global session diff --git a/cps/helper.py b/cps/helper.py index e909086e..a68aad7b 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -428,32 +428,46 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) +def get_cover_on_failure(use_generic_cover): + if use_generic_cover: + return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + else: + return None + def get_book_cover(book_id): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - if book.has_cover: + return get_book_cover_internal(book, False) +def get_book_cover_with_uuid(book_uuid, + use_generic_cover_on_failure=True): + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + return get_book_cover_internal(book, use_generic_cover_on_failure) + +def get_book_cover_internal(book, + use_generic_cover_on_failure): + if book.has_cover: if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): - return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) path=gd.get_cover_via_gdrive(book.path) if path: return redirect(path) else: log.error('%s/cover.jpg not found on Google Drive', book.path) - return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) except Exception as e: log.exception(e) # traceback.print_exc() - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): return send_from_directory(cover_file_path, "cover.jpg") else: - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) else: - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) # saves book cover from url diff --git a/cps/kobo.py b/cps/kobo.py new file mode 100644 index 00000000..7abfe4d0 --- /dev/null +++ b/cps/kobo.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from flask import Blueprint, request, flash, redirect, url_for +from . import logger, ub, searched_ids, db, helper +from . import config + +from flask import make_response +from flask import jsonify +from flask import json +from flask import send_file +from time import gmtime, strftime +import uuid +from uuid import uuid4, uuid3 +from collections import defaultdict +from b2sdk.account_info.in_memory import InMemoryAccountInfo +from b2sdk.api import B2Api +import os +import subprocess +from datetime import datetime, tzinfo, timedelta +from .constants import CONFIG_DIR as _CONFIG_DIR +import copy +import jsonschema +from sqlalchemy import func + +B2_SECRETS = os.path.join(_CONFIG_DIR, "b2_secrets.json") + +kobo = Blueprint("kobo", __name__) +log = logger.create() + +import base64 + + +def b64encode(data): + return base64.b64encode(data) + + +def b64encode_json(json_data): + return b64encode(json.dumps(json_data)) + + +# Python3 has a timestamp() method we could be calling, however it's not avaiable in python2. +def to_epoch_timestamp(datetime_object): + return (datetime_object - datetime(1970, 1, 1)).total_seconds() + + +class SyncToken: + """ The SyncToken is used to persist state accross requests. + When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service. + As an example use-case, the SyncToken is used to detect books that have been added to the library since the last time the device synced to the server. + + Attributes: + books_last_created: Datetime representing the newest book that the device knows about. + books_last_modified: Datetime representing the last modified book that the device knows about. + """ + + SYNC_TOKEN_HEADER = "x-kobo-synctoken" + VERSION = "1-0-0" + MIN_VERSION = "1-0-0" + + token_schema = { + "type": "object", + "properties": {"version": {"type": "string"}, "data": {"type": "object"},}, + } + # This Schema doesn't contain enough information to detect and propagate book deletions from Calibre to the device. + # A potential solution might be to keep a list of all known book uuids in the token, and look for any missing from the db. + data_schema_v1 = { + "type": "object", + "properties": { + "raw_kobo_store_token": {"type": "string"}, + "books_last_modified": {"type": "string"}, + "books_last_created": {"type": "string"}, + }, + } + + def __init__( + self, + raw_kobo_store_token="", + books_last_created=datetime.min, + books_last_modified=datetime.min, + ): + self.raw_kobo_store_token = raw_kobo_store_token + self.books_last_created = books_last_created + self.books_last_modified = books_last_modified + + @staticmethod + def from_headers(headers): + sync_token_header = headers.get(SyncToken.SYNC_TOKEN_HEADER, "") + if sync_token_header == "": + return SyncToken() + + # On the first sync from a Kobo device, we may receive the SyncToken + # from the official Kobo store. Without digging too deep into it, that + # token is of the form [b64encoded blob].[b64encoded blob 2] + if "." in sync_token_header: + return SyncToken(raw_kobo_store_token=sync_token_header) + + sync_token_json = json.loads( + base64.b64decode(sync_token_header + "=" * (-len(sync_token_header) % 4)) + ) + try: + jsonschema.validate(sync_token_json, SyncToken.token_schema) + if sync_token_json["version"] < SyncToken.MIN_VERSION: + raise ValueError + + data_json = sync_token_json["data"] + jsonschema.validate(sync_token_json, SyncToken.data_schema_v1) + except (jsonschema.exceptions.ValidationError, ValueError) as e: + log.error("Sync token contents do not follow the expected json schema.") + return SyncToken() + + raw_kobo_store_token = data_json["raw_kobo_store_token"] + try: + books_last_modified = datetime.utcfromtimestamp( + data_json["books_last_modified"] + ) + books_last_created = datetime.utcfromtimestamp( + data_json["books_last_created"] + ) + except TypeError: + log.error("SyncToken timestamps don't parse to a datetime.") + return SyncToken(raw_kobo_store_token=raw_kobo_store_token) + + return SyncToken( + raw_kobo_store_token=raw_kobo_store_token, + books_last_created=books_last_created, + books_last_modified=books_last_modified, + ) + + def to_headers(self, headers): + headers[SyncToken.SYNC_TOKEN_HEADER] = self.build_sync_token() + + def build_sync_token(self): + token = { + "version": SyncToken.VERSION, + "data": { + "raw_kobo_store_token": self.raw_kobo_store_token, + "books_last_modified": to_epoch_timestamp(self.books_last_modified), + "books_last_created": to_epoch_timestamp(self.books_last_created), + }, + } + return b64encode_json(token) + + +@kobo.route("/v1/library/sync") +def HandleSyncRequest(): + sync_token = SyncToken.from_headers(request.headers) + log.info("Kobo library sync request received.") + + # TODO: Limit the number of books return per sync call, and rely on the sync-continuatation header + # instead so that the device triggers another sync. + + new_books_last_modified = sync_token.books_last_modified + new_books_last_created = sync_token.books_last_created + entitlements = [] + + # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast. + # It looks like it's treating the db.Books.last_modified field as a string and may fail + # the comparison because of the +00:00 suffix. + changed_entries = ( + db.session.query(db.Books) + .filter(func.datetime(db.Books.last_modified) > sync_token.books_last_modified) + .all() + ) + for book in changed_entries: + entitlement = CreateEntitlement(book) + if book.timestamp > sync_token.books_last_created: + entitlements.append({"NewEntitlement": entitlement}) + else: + entitlements.append({"ChangedEntitlement": entitlement}) + + new_books_last_modified = max( + book.last_modified, sync_token.books_last_modified + ) + new_books_last_created = max(book.timestamp, sync_token.books_last_modified) + + sync_token.books_last_created = new_books_last_created + sync_token.books_last_modified = new_books_last_modified + + # Missing feature: Detect server-side book deletions. + + # Missing feature: Join the response with results from the official Kobo store so that users can still buy and access books from the device store (particularly while on-the-road). + + response = make_response(jsonify(entitlements)) + + sync_token.to_headers(response.headers) + response.headers["x-kobo-sync-mode"] = "delta" + response.headers["x-kobo-apitoken"] = "e30=" + return response + + +@kobo.route("/v1/library//metadata") +def get_metadata__v1(book_uuid): + log.info("Kobo library metadata request received for book %s" % book_uuid) + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + if not book: + log.info(u"Book %s not found in database", book_uuid) + return make_response("Book not found in database.", 404) + + download_url = get_download_url_for_book(book) + if not download_url: + return make_response("Could not get a download url for book.", 500) + + metadata = create_metadata(book) + metadata["DownloadUrls"] = [ + { + "DrmType": "SignedNoDrm", + "Format": "KEPUB", + "Platform": "Android", + # TODO: Set the file size. + # "Size": file_info["contentLength"], + "Url": download_url, + } + ] + return jsonify([metadata]) + + +def get_download_url_for_book(book): + # TODO: Research what formats Kobo will support over the sync protocol. + # For now let's just assume all books are converted to KEPUB. + data = ( + db.session.query(db.Data) + .filter(db.Data.book == book.id) + .filter(db.Data.format == "KEPUB") + .first() + ) + + if not data: + log.info(u"Book %s does have a kepub format", book_uuid) + return None + + file_name = data.name + ".kepub" + file_path = os.path.join(book.path, file_name) + + if not os.path.isfile(B2_SECRETS): + log.error(u"b2 secret file not found") + return None + with open(B2_SECRETS, "r") as filedata: + secrets = json.load(filedata) + + info = InMemoryAccountInfo() + b2_api = B2Api(info) + b2_api.authorize_account( + "production", secrets["application_key_id"], secrets["application_key"] + ) + bucket = b2_api.get_bucket_by_name(secrets["bucket_name"]) + if not bucket: + log.error(u"b2 bucket not found") + return None + + download_url = b2_api.get_download_url_for_file_name( + secrets["bucket_name"], file_path + ) + download_authorization = bucket.get_download_authorization( + file_path, valid_duration_in_seconds=600 + ) + return download_url + "?Authorization=" + download_authorization + + +def CreateBookEntitlement(book): + book_uuid = book.uuid + return { + "Accessibility": "Full", + "ActivePeriod": {"From": current_time(),}, + "Created": book.timestamp, + "CrossRevisionId": book_uuid, + "Id": book_uuid, + "IsHiddenFromArchive": False, + "IsLocked": False, + # Setting this to true removes from the device. + "IsRemoved": False, + "LastModified": book.last_modified, + "OriginCategory": "Imported", + "RevisionId": book_uuid, + "Status": "Active", + } + + +def CreateEntitlement(book): + return { + "BookEntitlement": CreateBookEntitlement(book), + "BookMetadata": create_metadata(book), + "ReadingState": reading_state(book), + } + + +def current_time(): + return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) + + +def get_description(book): + if not book.comments: + return None + return book.comments[0].text + + +# TODO handle multiple authors +def get_author(book): + if not book.authors: + return None + return book.authors[0].name + + +def get_publisher(book): + if not book.publishers: + return None + return book.publishers[0].name + + +def get_series(book): + if not book.series: + return None + return book.series[0].name + + +def create_metadata(book): + book_uuid = book.uuid + metadata = { + "Categories": ["00000000-0000-0000-0000-000000000001",], + "Contributors": get_author(book), + "CoverImageId": book_uuid, + "CrossRevisionId": book_uuid, + "CurrentDisplayPrice": {"CurrencyCode": "USD", "TotalAmount": 0}, + "CurrentLoveDisplayPrice": {"TotalAmount": 0}, + "Description": get_description(book), + "DownloadUrls": [ + # Looks like we need to pass at least one url in the + # v1/library/sync call. The new entitlement is ignored + # otherwise. + # May want to experiment more with this. + { + "DrmType": "None", + "Format": "KEPUB", + "Platform": "Android", + "Size": 1024775, + "Url": "https://google.com", + }, + ], + "EntitlementId": book_uuid, + "ExternalIds": [], + "Genre": "00000000-0000-0000-0000-000000000001", + "IsEligibleForKoboLove": False, + "IsInternetArchive": False, + "IsPreOrder": False, + "IsSocialEnabled": True, + "Language": "en", + "PhoneticPronunciations": {}, + "PublicationDate": "2019-02-03T00:25:03.0000000Z", # current_time(), + "Publisher": {"Imprint": "", "Name": get_publisher(book),}, + "RevisionId": book_uuid, + "Title": book.title, + "WorkId": book_uuid, + } + + if get_series(book): + metadata["Series"] = { + "Name": get_series(book), + "Number": book.series_index, + "NumberFloat": float(book.series_index), + # Get a deterministic id based on the series name. + "Id": uuid3(uuid.NAMESPACE_DNS, get_series(book).encode("utf-8")), + } + + return metadata + + +def get_single_cc_value(book, custom_column_name): + custom_column_values = get_custom_column_values(book, custom_column_name) + if custom_column_values: + return custom_column_values[0].value + return None + + +def get_custom_column_values(book, custom_column_name): + custom_column = ( + db.session.query(db.Custom_Columns) + .filter(db.Custom_Columns.label == custom_column_name) + .one() + ) + cc_string = "custom_column_" + str(custom_column.id) + + return getattr(book, cc_string) + + +def reading_state(book): + # TODO: Make the state custom columns configurable. + # Possibly use calibre-web User db instead of the Calibre metadata.db? + reading_state = { + "StatusInfo": { + "LastModified": get_single_cc_value(book, "lastreadtimestamp"), + "Status": get_single_cc_value(book, "reading_status"), + } + # TODO: CurrentBookmark, Location + } + return reading_state + + +# def get_shelves(book): +# shelves = get_custom_column_values(book, "myshelves") +# return shelves + + +@kobo.route( + "//////image.jpg" +) +def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monochrome): + book_cover = helper.get_book_cover_with_uuid( + book_uuid, use_generic_cover_on_failure=False + ) + if not book_cover: + return make_response() + return book_cover + + +@kobo.route("/v1/user/profile") +@kobo.route("/v1/user/loyalty/benefits") +@kobo.route("/v1/analytics/gettests/", methods=["GET", "POST"]) +@kobo.route("/v1/user/wishlist") +@kobo.route("/v1/user/") +@kobo.route("/v1/user/recommendations") +@kobo.route("/v1/products/") +@kobo.route("/v1/products//nextread") +@kobo.route("/v1/products/featured/") +@kobo.route("/v1/products/featured/") +@kobo.route("/v1/library/", methods=["DELETE", "GET"]) # TODO: implement +def HandleDummyRequest(dummy=None): + return make_response(jsonify({})) + + +@kobo.route("/v1/auth/device", methods=["POST"]) +def HandleAuthRequest(): + # Missing feature: Authentication :) + response = make_response( + jsonify( + { + "AccessToken": "abcde", + "RefreshToken": "abcde", + "TokenType": "Bearer", + "TrackingId": "abcde", + "UserKey": "abcdefgeh", + } + ) + ) + return response + + +@kobo.route("/v1/initialization") +def HandleInitRequest(): + resources = NATIVE_KOBO_RESOURCES(calibre_web_url=config.config_server_url) + response = make_response(jsonify({"Resources": resources})) + response.headers["x-kobo-apitoken"] = "e30=" + return response + + +def NATIVE_KOBO_RESOURCES(calibre_web_url): + return { + "account_page": "https://secure.kobobooks.com/profile", + "account_page_rakuten": "https://my.rakuten.co.jp/", + "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}", + "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate", + "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion", + "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations", + "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete", + "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"}, + "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}", + "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}", + "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}", + "book_landing_page": "https://store.kobobooks.com/ebooks", + "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions", + "categories": "https://storeapi.kobo.com/v1/categories", + "categories_page": "https://store.kobobooks.com/ebooks/categories", + "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}", + "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured", + "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products", + "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow", + "configuration_data": "https://storeapi.kobo.com/v1/configuration", + "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access", + "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO", + "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal", + "deals": "https://storeapi.kobo.com/v1/deals", + "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}", + "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete", + "device_auth": "https://storeapi.kobo.com/v1/auth/device", + "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh", + "dictionary_host": "https://kbdownload1-a.akamaihd.net", + "discovery_host": "https://discovery.kobobooks.com", + "eula_page": "https://www.kobo.com/termsofuse?style=onestore", + "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange", + "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}", + "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/", + "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}", + "featured_lists": "https://storeapi.kobo.com/v1/products/featured", + "free_books_page": { + "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks", + "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits", + "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti", + "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", + "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis", + }, + "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback", + "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests", + "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader", + "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem", + "help_page": "http://www.kobo.com/help", + "image_host": calibre_web_url, + "image_url_quality_template": calibre_web_url + + "/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg", + "image_url_template": calibre_web_url + + "/{ImageId}/{Width}/{Height}/false/image.jpg", + "kobo_audiobooks_enabled": "False", + "kobo_audiobooks_orange_deal_enabled": "False", + "kobo_audiobooks_subscriptions_enabled": "False", + "kobo_nativeborrow_enabled": "True", + "kobo_onestorelibrary_enabled": "False", + "kobo_redeem_enabled": "True", + "kobo_shelfie_enabled": "False", + "kobo_subscriptions_enabled": "False", + "kobo_superpoints_enabled": "False", + "kobo_wishlist_enabled": "True", + "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}", + "library_items": "https://storeapi.kobo.com/v1/user/library", + "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata", + "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices", + "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}", + "library_sync": "https://storeapi.kobo.com/v1/library/sync", + "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints", + "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}", + "magazine_landing_page": "https://store.kobobooks.com/emagazines", + "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration", + "oauth_host": "https://oauth.kobo.com", + "overdrive_account": "https://auth.overdrive.com/account", + "overdrive_library": "https://{libraryKey}.auth.overdrive.com/library", + "overdrive_library_finder_host": "https://libraryfinder.api.overdrive.com", + "overdrive_thunder_host": "https://thunder.api.overdrive.com", + "password_retrieval_page": "https://www.kobobooks.com/passwordretrieval.html", + "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event", + "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore", + "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread", + "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices", + "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations", + "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews", + "products": "https://storeapi.kobo.com/v1/products", + "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/", + "purchase_buy": "https://www.kobo.com/checkout/createpurchase/", + "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}", + "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout", + "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase", + "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}", + "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state", + "redeem_interstitial_page": "https://store.kobobooks.com", + "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://store.kobobooks.com/", + "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related", + "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}", + "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}", + "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}", + "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie", + "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://store.kobobooks.com/", + "social_authorization_host": "https://social.kobobooks.com:8443", + "social_host": "https://social.kobobooks.com", + "stacks_host_productId": "https://store.kobobooks.com/collections/byproductid/", + "store_home": "www.kobo.com/{region}/{language}", + "store_host": "store.kobobooks.com", + "store_newreleases": "https://store.kobobooks.com/{culture}/List/new-releases/961XUjtsU0qxkFItWOutGA", + "store_search": "https://store.kobobooks.com/{culture}/Search?Query={query}", + "store_top50": "https://store.kobobooks.com/{culture}/ebooks/Top", + "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items", + "tags": "https://storeapi.kobo.com/v1/library/tags", + "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile", + "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview", + "use_one_store": "False", + "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits", + "user_platform": "https://storeapi.kobo.com/v1/user/platform", + "user_profile": "https://storeapi.kobo.com/v1/user/profile", + "user_ratings": "https://storeapi.kobo.com/v1/user/ratings", + "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations", + "user_reviews": "https://storeapi.kobo.com/v1/user/reviews", + "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist", + "userguide_host": "https://kbdownload1-a.akamaihd.net", + "wishlist_page": "https://store.kobobooks.com/{region}/{language}/account/wishlist", + } diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index dd406d39..b73ecd21 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -104,6 +104,10 @@
    +
    + + +
    diff --git a/requirements.txt b/requirements.txt index daf2538d..d0c541a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ SQLAlchemy>=1.1.0 tornado>=4.1 Wand>=0.4.4 unidecode>=0.04.19 +b2sdk>=1.0.2,<2.0.0 +jsonschema>=3.2.0 From 55b54de6a013b61e74ac09705799cdc6554bdfd0 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Thu, 5 Dec 2019 19:06:39 -0500 Subject: [PATCH 010/688] Add simple get_download_url implementation to replace the backblaze-backed implementation --- cps/kobo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cps/kobo.py b/cps/kobo.py index 7abfe4d0..c41b6a4c 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -214,8 +214,10 @@ def get_metadata__v1(book_uuid): ] return jsonify([metadata]) - def get_download_url_for_book(book): + return "{url_base}/download/{book_id}/kepub".format(url_base=config.config_server_url, book_id=book.id) + +def get_download_url_for_book_b2(book): # TODO: Research what formats Kobo will support over the sync protocol. # For now let's just assume all books are converted to KEPUB. data = ( From c61463447fcf208d71d2501fa9b03dc49d8a5c8e Mon Sep 17 00:00:00 2001 From: Christian Keil Date: Fri, 6 Dec 2019 15:00:01 +0100 Subject: [PATCH 011/688] Merge metadata of uploaded book versions. --- cps/editbooks.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index f0156f71..492333a8 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -360,6 +360,9 @@ def upload_single_file(request, book, book_id): worker.add_upload(current_user.nickname, "" + uploadText + "") + return uploader.process( + saved_filename, *os.path.splitext(requested_file.filename)) + def upload_cover(request, book): if 'btn-upload-cover' in request.files: @@ -393,11 +396,12 @@ def edit_book(book_id): flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error") return redirect(url_for("web.index")) - upload_single_file(request, book, book_id) + meta = upload_single_file(request, book, book_id) if upload_cover(request, book) is True: book.has_cover = 1 try: to_save = request.form.to_dict() + merge_metadata(to_save, meta) # Update book edited_books_id = None #handle book title @@ -531,6 +535,20 @@ def edit_book(book_id): return redirect(url_for('web.show_book', book_id=book.id)) +def merge_metadata(to_save, meta): + if to_save['author_name'].lower() == _(u'unknown'): + to_save['author_name'] = '' + if to_save['book_title'].lower() == _(u'unknown'): + to_save['book_title'] = '' + for s_field, m_field in [ + ('tags', 'tags'), ('author_name', 'author'), ('series', 'series'), + ('series_index', 'series_id'), ('languages', 'languages'), + ('book_title', 'title')]: + to_save[s_field] = to_save[s_field] or getattr(meta, m_field, '') + to_save["description"] = to_save["description"] or Markup( + getattr(meta, 'description', '')).unescape() + + @editbook.route("/upload", methods=["GET", "POST"]) @login_required_if_no_ano @upload_required From 9ede01f130269694efad9fa813626e642db60b8a Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Sat, 7 Dec 2019 19:21:08 -0500 Subject: [PATCH 012/688] * Add a UserKeyToken to the User table for Kobo authorization. * Add proper authorization checks on the new Kobo endpoints. Important Note: As a side-effect, all CalibreWeb API calls can be authorized using this token (i.e without a username&password). --- cps/admin.py | 13 ++++++- cps/kobo.py | 13 ++++--- cps/kobo_auth.py | 67 ++++++++++++++++++++++++++++++++++++ cps/templates/user_edit.html | 4 +++ cps/ub.py | 13 +++++-- cps/web.py | 4 ++- 6 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 cps/kobo_auth.py diff --git a/cps/admin.py b/cps/admin.py index 0e30109c..0292eee3 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -565,7 +565,6 @@ def edit_user(user_id): else: if "password" in to_save and to_save["password"]: content.password = generate_password_hash(to_save["password"]) - anonymous = content.is_anonymous content.role = constants.selected_roles(to_save) if anonymous: @@ -593,6 +592,18 @@ def edit_user(user_id): content.default_language = to_save["default_language"] if "locale" in to_save and to_save["locale"]: content.locale = to_save["locale"] + + if "kobo_user_key" in to_save and to_save["kobo_user_key"]: + kobo_user_key_hash = generate_password_hash(to_save["kobo_user_key"]) + if kobo_user_key_hash != content.kobo_user_key_hash: + existing_kobo_user_key = ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash == kobo_user_key_hash).first() + if not existing_kobo_user_key: + content.kobo_user_key_hash = kobo_user_key_hash + else: + flash(_(u"Found an existing account for this Kobo UserKey."), category="error") + return render_title_template("user_edit.html", translations=translations, languages=languages, + new_user=0, content=content, downloads=downloads, registered_oauth=oauth_check, + title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") if to_save["email"] and to_save["email"] != content.email: existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ .first() diff --git a/cps/kobo.py b/cps/kobo.py index c41b6a4c..d78e6f19 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- from flask import Blueprint, request, flash, redirect, url_for -from . import logger, ub, searched_ids, db, helper -from . import config - +from . import config, logger, kobo_auth, ub, db, helper +from .web import download_required from flask import make_response from flask import jsonify from flask import json @@ -22,15 +21,17 @@ from .constants import CONFIG_DIR as _CONFIG_DIR import copy import jsonschema from sqlalchemy import func +from flask_login import login_required B2_SECRETS = os.path.join(_CONFIG_DIR, "b2_secrets.json") kobo = Blueprint("kobo", __name__) +kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) + log = logger.create() import base64 - def b64encode(data): return base64.b64encode(data) @@ -143,6 +144,8 @@ class SyncToken: @kobo.route("/v1/library/sync") +@login_required +@download_required def HandleSyncRequest(): sync_token = SyncToken.from_headers(request.headers) log.info("Kobo library sync request received.") @@ -190,6 +193,8 @@ def HandleSyncRequest(): @kobo.route("/v1/library//metadata") +@login_required +@download_required def get_metadata__v1(book_uuid): log.info("Kobo library metadata request received for book %s" % book_uuid) book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py new file mode 100644 index 00000000..c1f45f08 --- /dev/null +++ b/cps/kobo_auth.py @@ -0,0 +1,67 @@ +"""This module is used to control authentication/authorization of Kobo sync requests. +This module also includes research notes into the auth protocol used by Kobo devices. + +Log-in: +When first booting a Kobo device the user must sign into a Kobo (or affiliate) account. +Upon successful sign-in, the user is redirected to + https://auth.kobobooks.com/CrossDomainSignIn?id= +which serves the following response: + . +And triggers the insertion of a userKey into the device's User table. + +IMPORTANT SECURITY CAUTION: +Together, the device's DeviceId and UserKey act as an *irrevocable* authentication +token to most (if not all) Kobo APIs. In fact, in most cases only the UserKey is +required to authorize the API call. + +Changing Kobo password *does not* invalidate user keys! This is apparently a known +issue for a few years now https://www.mobileread.com/forums/showpost.php?p=3476851&postcount=13 +(although this poster hypothesised that Kobo could blacklist a DeviceId, many endpoints +will still grant access given the userkey.) + +Api authorization: +* For most of the endpoints we care about (sync, metadata, tags, etc), the userKey is +passed in the x-kobo-userkey header, and is sufficient to authorize the API call. +* Some endpoints (e.g: AnnotationService) instead make use of Bearer tokens. To get a +BearerToken, the device makes a POST request to the v1/auth/device endpoint with the +secret UserKey and the device's DeviceId. + +Our implementation: +For now, we rely on the official Kobo store's UserKey for authentication. Because of the +irrevocable power granted by the key, we only ever store and compare a hash of the key. +To obtain their UserKey, a user can query the user table from the +.kobo/KoboReader.sqlite database found on their device. +This isn't exactly user friendly however. + +Some possible alternatives that require more research: + * Instead of having users query the device database to find out their UserKey, we could + provide a list of recent Kobo sync attempts in the calibre-web UI for users to + authenticate sync attempts (e.g: 'this was me' button). + * We may be able to craft a sign-in flow with a redirect back to the CalibreWeb + server containing the KoboStore's UserKey. + * Can we create our own UserKey instead of relying on the real store's userkey? + (Maybe using something like location.href=kobo://UserAuthenticated?userId=...?) +""" + +from functools import wraps +from flask import request, make_response +from werkzeug.security import check_password_hash +from . import logger, ub, lm + +USER_KEY_HEADER = "x-kobo-userkey" +log = logger.create() + +def disable_failed_auth_redirect_for_blueprint(bp): + lm.blueprint_login_views[bp.name] = None + +@lm.request_loader +def load_user_from_kobo_request(request): + user_key = request.headers.get(USER_KEY_HEADER) + if user_key: + for user in ( + ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash != "").all() + ): + if check_password_hash(str(user.kobo_user_key_hash), user_key): + return user + log.info("Received Kobo request without a recognizable UserKey.") + return None \ No newline at end of file diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index e22a9415..1e7fa9b9 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -28,6 +28,10 @@
    +
    + + +
    {% if feature_support['ldap'] %} -
    +
    From efcee0a7b7ecf33e6527a98b05b1acdd82c1d2f6 Mon Sep 17 00:00:00 2001 From: Andrew Roberts Date: Thu, 12 Dec 2019 21:31:21 -0500 Subject: [PATCH 028/688] added reverse proxy configuration form and handler --- cps/admin.py | 4 ++++ cps/templates/config_edit.html | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/cps/admin.py b/cps/admin.py index 1862dda8..6cb5bfdb 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -347,6 +347,10 @@ def _configuration_update_helper(): _config_int("config_updatechannel") + # Reverse proxy login configuration + _config_checkbox("config_allow_reverse_proxy_header_login") + _config_string("config_reverse_proxy_login_header_name") + # GitHub OAuth configuration if config.config_login_type == constants.LOGIN_OAUTH: active_oauths = 0 diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index b46d07f5..0d28b8ea 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -271,6 +271,16 @@
    {% endif %} {% endif %} +
    + + +
    +
    +
    + + +
    +
    From 3dc372c5737b50e7bfc59bbf4cbd9241bbbdd1fe Mon Sep 17 00:00:00 2001 From: Andrew Roberts Date: Thu, 12 Dec 2019 21:38:45 -0500 Subject: [PATCH 029/688] fixed typo --- cps/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/web.py b/cps/web.py index e88d423d..c6a3e1c4 100644 --- a/cps/web.py +++ b/cps/web.py @@ -135,7 +135,7 @@ def load_user_from_request(request): if config.config_allow_reverse_proxy_header_login: rp_header_name = config.config_reverse_proxy_login_header_name if rp_header_name: - rp_header = request.headers.get(rp_header_name) + rp_header_username = request.headers.get(rp_header_name) if rp_header_username: user = _fetch_user_by_name(rp_header_username) if user: From 86fe970651e4ae15c818d7fc4e21e2cf9d762537 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sat, 14 Dec 2019 22:22:27 +0100 Subject: [PATCH 030/688] More fixes for googledrive --- cps/admin.py | 7 ++++++- cps/gdrive.py | 6 +++++- cps/gdriveutils.py | 4 ++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 1862dda8..3c64b01b 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -170,6 +170,9 @@ def update_view_configuration(): _config_int("config_books_per_page") _config_int("config_authors_max") + if config.config_google_drive_watch_changes_response: + config.config_google_drive_watch_changes_response = json.dumps(config.config_google_drive_watch_changes_response) + config.config_default_role = constants.selected_roles(to_save) config.config_default_role &= ~constants.ROLE_ANONYMOUS @@ -258,6 +261,7 @@ def _configuration_update_helper(): db_change = False to_save = request.form.to_dict() + # _config_dict = lambda x: config.set_from_dictionary(to_save, x, lambda y: y['id']) _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) _config_int = lambda x: config.set_from_dictionary(to_save, x, int) _config_checkbox = lambda x: config.set_from_dictionary(to_save, x, lambda y: y == "on", False) @@ -415,7 +419,8 @@ def _configuration_result(error_flash=None, gdriveError=None): if gdriveError: gdriveError = _(gdriveError) else: - if config.config_use_google_drive and not gdrive_authenticate: + # if config.config_use_google_drive and\ + if not gdrive_authenticate: gdrivefolders = gdriveutils.listRootFolders() show_back_button = current_user.is_authenticated diff --git a/cps/gdrive.py b/cps/gdrive.py index 263c829b..a95060b0 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -23,6 +23,7 @@ from __future__ import division, print_function, unicode_literals import os +import sys import hashlib import json import tempfile @@ -141,7 +142,10 @@ def on_received_watch_confirmation(): response = gdriveutils.getChangeById(gdriveutils.Gdrive.Instance().drive, j['id']) log.debug('%r', response) if response: - dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + if sys.version_info < (3, 0): + dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + else: + dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != hashlib.md5(dbpath): tmpDir = tempfile.gettempdir() log.info('Database file updated') diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index d950f738..f70747a6 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -47,6 +47,10 @@ CREDENTIALS = os.path.join(_CONFIG_DIR, 'gdrive_credentials') CLIENT_SECRETS = os.path.join(_CONFIG_DIR, 'client_secrets.json') log = logger.create() +if gdrive_support: + logger.get('googleapiclient.discovery_cache').setLevel(logger.logging.ERROR) + if not logger.is_debug_enabled(): + logger.get('googleapiclient.discovery').setLevel(logger.logging.ERROR) class Singleton: From 2215bf3d7f7feba7a904b57316f800bb92421846 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 15 Dec 2019 11:35:07 +0100 Subject: [PATCH 031/688] Implemented #1083 (Advanced search for extensions) --- cps/templates/search_form.html | 22 +++++++++++++++++++++- cps/web.py | 23 +++++++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/cps/templates/search_form.html b/cps/templates/search_form.html index 4cb20a02..6a64085d 100644 --- a/cps/templates/search_form.html +++ b/cps/templates/search_form.html @@ -31,7 +31,7 @@ -
    +
    {% for tag in tags %}
    {% endif%} + +
    +
    + {% for extension in extensions %} + + {% endfor %} +
    +
    + +
    +
    + {% for extension in extensions %} + + {% endfor %} +
    +
    diff --git a/cps/web.py b/cps/web.py index 7aa921e4..7823be39 100644 --- a/cps/web.py +++ b/cps/web.py @@ -431,6 +431,8 @@ def get_matching_tags(): title_input = request.args.get('book_title') include_tag_inputs = request.args.getlist('include_tag') exclude_tag_inputs = request.args.getlist('exclude_tag') + include_extension_inputs = request.args.getlist('include_extension') + exclude_extension_inputs = request.args.getlist('exclude_extension') q = q.filter(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + author_input + "%")), func.lower(db.Books.title).ilike("%" + title_input + "%")) if len(include_tag_inputs) > 0: @@ -439,6 +441,12 @@ def get_matching_tags(): if len(exclude_tag_inputs) > 0: for tag in exclude_tag_inputs: q = q.filter(not_(db.Books.tags.any(db.Tags.id == tag))) + '''if len(include_extension_inputs) > 0: + for tag in exclude_tag_inputs: + q = q.filter(not_(db.Books.tags.any(db.Tags.id == tag))) + if len(exclude_extension_inputs) > 0: + for tag in exclude_tag_inputs: + q = q.filter(not_(db.Books.tags.any(db.Tags.id == tag)))''' for book in q: for tag in book.tags: if tag.id not in tag_dict['tags']: @@ -818,6 +826,8 @@ def advanced_search(): exclude_series_inputs = request.args.getlist('exclude_serie') include_languages_inputs = request.args.getlist('include_language') exclude_languages_inputs = request.args.getlist('exclude_language') + include_extension_inputs = request.args.getlist('include_extension') + exclude_extension_inputs = request.args.getlist('exclude_extension') author_name = request.args.get("author_name") book_title = request.args.get("book_title") @@ -843,7 +853,8 @@ def advanced_search(): if include_tag_inputs or exclude_tag_inputs or include_series_inputs or exclude_series_inputs or \ include_languages_inputs or exclude_languages_inputs or author_name or book_title or \ - publisher or pub_start or pub_end or rating_low or rating_high or description or cc_present: + publisher or pub_start or pub_end or rating_low or rating_high or description or cc_present or \ + include_extension_inputs or exclude_extension_inputs: searchterm = [] searchterm.extend((author_name.replace('|', ','), book_title, publisher)) if pub_start: @@ -872,6 +883,8 @@ def advanced_search(): searchterm.extend([_(u"Rating <= %(rating)s", rating=rating_high)]) if rating_low: searchterm.extend([_(u"Rating >= %(rating)s", rating=rating_low)]) + searchterm.extend(ext for ext in include_extension_inputs) + searchterm.extend(ext for ext in exclude_extension_inputs) # handle custom columns for c in cc: if request.args.get('custom_column_' + str(c.id)): @@ -896,6 +909,10 @@ def advanced_search(): q = q.filter(db.Books.series.any(db.Series.id == serie)) for serie in exclude_series_inputs: q = q.filter(not_(db.Books.series.any(db.Series.id == serie))) + for extension in include_extension_inputs: + q = q.filter(db.Books.data.any(db.Data.format == extension)) + for extension in exclude_extension_inputs: + q = q.filter(not_(db.Books.data.any(db.Data.format == extension))) if current_user.filter_language() != "all": q = q.filter(db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())) else: @@ -936,11 +953,13 @@ def advanced_search(): # tags = db.session.query(db.Tags).order_by(db.Tags.name).all() tags = db.session.query(db.Tags).filter(tags_filters()).order_by(db.Tags.name).all() series = db.session.query(db.Series).order_by(db.Series.name).all() + extensions = db.session.query(db.Data) \ + .group_by(db.Data.format).order_by(db.Data.format).all() if current_user.filter_language() == u"all": languages = speaking_language() else: languages = None - return render_title_template('search_form.html', tags=tags, languages=languages, + return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions, series=series, title=_(u"search"), cc=cc, page="advsearch") From c33623efeea9ec5be5afa8c81bceed5dea10e332 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 15 Dec 2019 13:32:34 +0100 Subject: [PATCH 032/688] Unified wording for recently added books, series, categories, etc in opds and web UI ( #1045) Added file formats and languages to opds feed --- cps/opds.py | 68 +++++++++++++++++++++++++++++++++++++++-- cps/templates/feed.xml | 4 ++- cps/templates/index.xml | 20 ++++++++++-- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/cps/opds.py b/cps/opds.py index 9198554b..f5cc4673 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -31,11 +31,13 @@ from flask_login import current_user from sqlalchemy.sql.expression import func, text, or_, and_ from werkzeug.security import check_password_hash -from . import constants, logger, config, db, ub, services -from .helper import fill_indexpage, get_download_link, get_book_cover +from . import constants, logger, config, db, ub, services, get_locale, isoLanguages +from .helper import fill_indexpage, get_download_link, get_book_cover, speaking_language from .pagination import Pagination from .web import common_filters, get_search_results, render_read_books, download_required - +from flask_babel import gettext as _ +from babel import Locale as LC +from babel.core import UnknownLocaleError opds = Blueprint('opds', __name__) @@ -213,6 +215,66 @@ def feed_series(book_id): db.Books, db.Books.series.any(db.Series.id == book_id), [db.Books.series_index]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) +@opds.route("/opds/formats") +@requires_basic_auth_if_no_ano +def feed_formatindex(): + off = request.args.get("offset") or 0 + entries = db.session.query(db.Data).join(db.Books).filter(common_filters()) \ + .group_by(db.Data.format).order_by(db.Data.format).all() + pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, + len(entries)) + for entry in entries: + entry.name = entry.format + entry.id = entry.format + return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_format', pagination=pagination) + + +@opds.route("/opds/formats/") +@requires_basic_auth_if_no_ano +def feed_format(book_id): + off = request.args.get("offset") or 0 + entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [db.Books.timestamp.desc()]) + return render_xml_template('feed.xml', entries=entries, pagination=pagination) + +@opds.route("/opds/language") +@opds.route("/opds/language/") +@requires_basic_auth_if_no_ano +def feed_languagesindex(): + off = request.args.get("offset") or 0 + if current_user.filter_language() == u"all": + languages = speaking_language() + else: + try: + cur_l = LC.parse(current_user.filter_language()) + except UnknownLocaleError: + cur_l = None + languages = db.session.query(db.Languages).filter( + db.Languages.lang_code == current_user.filter_language()).all() + if cur_l: + languages[0].name = cur_l.get_language_name(get_locale()) + else: + languages[0].name = _(isoLanguages.get(part3=languages[0].lang_code).name) + pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, + len(languages)) + return render_xml_template('feed.xml', listelements=languages, folder='opds.feed_languages', pagination=pagination) + + +@opds.route("/opds/language/") +@requires_basic_auth_if_no_ano +def feed_languages(book_id): + off = request.args.get("offset") or 0 + entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + db.Books, db.Books.languages.any(db.Languages.id == book_id), [db.Books.timestamp.desc()]) + '''for entry in entries: + for index in range(0, len(entry.languages)): + try: + entry.languages[index].language_name = LC.parse(entry.languages[index].lang_code).get_language_name( + get_locale()) + except UnknownLocaleError: + entry.languages[index].language_name = _( + isoLanguages.get(part3=entry.languages[index].lang_code).name)''' + return render_xml_template('feed.xml', entries=entries, pagination=pagination) @opds.route("/opds/shelfindex/", defaults={'public': 0}) @opds.route("/opds/shelfindex/") diff --git a/cps/templates/feed.xml b/cps/templates/feed.xml index 16548a91..37b7765e 100644 --- a/cps/templates/feed.xml +++ b/cps/templates/feed.xml @@ -53,7 +53,9 @@ {{entry.publishers[0].name}} {% endif %} - {{entry.language}} + {% for lang in entry.languages %} + {{lang.lang_code}} + {% endfor %} {% for tag in entry.tags %} {{_('Popular publications from this catalog based on Rating.')}} - {{_('New Books')}} + {{_('Recently added Books')}} {{url_for('opds.feed_new')}} {{ current_time }} @@ -72,19 +72,33 @@ {{_('Books ordered by publisher')}} - {{_('Category list')}} + {{_('Categories')}} {{url_for('opds.feed_categoryindex')}} {{ current_time }} {{_('Books ordered by category')}} - {{_('Series list')}} + {{_('Series')}} {{url_for('opds.feed_seriesindex')}} {{ current_time }} {{_('Books ordered by series')}} + + {{_('Languages')}} + + {{url_for('opds.feed_languagesindex')}} + {{ current_time }} + {{_('Books ordered by Languages')}} + + + {{_('File formats')}} + + {{url_for('opds.feed_formatindex')}} + {{ current_time }} + {{_('Books ordered by file formats')}} + {{_('Public Shelves')}} From b6d7207ec3a53b3c83b17f275bde3f90e1f3f4e3 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 15 Dec 2019 13:33:38 +0100 Subject: [PATCH 033/688] Added platform information for better debugging --- cps/about.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cps/about.py b/cps/about.py index 630243f4..aa1e866e 100644 --- a/cps/about.py +++ b/cps/about.py @@ -22,6 +22,7 @@ from __future__ import division, print_function, unicode_literals import sys +import platform import sqlite3 from collections import OrderedDict @@ -48,6 +49,7 @@ about = flask.Blueprint('about', __name__) _VERSIONS = OrderedDict( + Platform = ' '.join(platform.uname()), Python=sys.version, WebServer=server.VERSION, Flask=flask.__version__, @@ -65,6 +67,7 @@ _VERSIONS = OrderedDict( Unidecode = unidecode_version, Flask_SimpleLDAP = u'installed' if bool(services.ldap) else u'not installed', Goodreads = u'installed' if bool(services.goodreads_support) else u'not installed', + ) _VERSIONS.update(uploader.get_versions()) From eabc6e23be014ae921989ba31515ef28691af422 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 15 Dec 2019 17:08:17 +0100 Subject: [PATCH 034/688] Test Email now send to user's email address (#834) Added forgot/reset password routine (#1098, #1063) --- cps/admin.py | 24 +++++++++--------------- cps/helper.py | 14 ++++++++++++++ cps/templates/login.html | 3 +++ cps/web.py | 37 ++++++++++++++++++++++++++----------- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 3c64b01b..57796080 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -36,11 +36,10 @@ from flask_babel import gettext as _ from sqlalchemy import and_ from sqlalchemy.exc import IntegrityError from sqlalchemy.sql.expression import func -from werkzeug.security import generate_password_hash from . import constants, logger, helper, services from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils -from .helper import speaking_language, check_valid_domain, send_test_mail, generate_random_password, send_registration_mail +from .helper import speaking_language, check_valid_domain, send_test_mail, reset_password, generate_password_hash from .gdriveutils import is_gdrive_ready, gdrive_support from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano @@ -524,15 +523,15 @@ def update_mailsettings(): config.save() if to_save.get("test"): - if current_user.kindle_mail: - result = send_test_mail(current_user.kindle_mail, current_user.nickname) + if current_user.email: + result = send_test_mail(current_user.email, current_user.nickname) if result is None: - flash(_(u"Test e-mail successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail), + flash(_(u"Test e-mail successfully send to %(kindlemail)s", kindlemail=current_user.email), category="success") else: flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error") else: - flash(_(u"Please configure your kindle e-mail address first..."), category="error") + flash(_(u"Please configure your e-mail address first..."), category="error") else: flash(_(u"E-mail server settings updated"), category="success") @@ -644,15 +643,10 @@ def reset_password(user_id): if not config.config_public_reg: abort(404) if current_user is not None and current_user.is_authenticated: - existing_user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() - password = generate_random_password() - existing_user.password = generate_password_hash(password) - try: - ub.session.commit() - send_registration_mail(existing_user.email, existing_user.nickname, password, True) - flash(_(u"Password for user %(user)s reset", user=existing_user.nickname), category="success") - except Exception: - ub.session.rollback() + ret, message = reset_password(user_id) + if ret == 1: + flash(_(u"Password for user %(user)s reset", user=message), category="success") + else: flash(_(u"An unknown error occurred. Please try again later."), category="error") return redirect(url_for('admin.admin')) diff --git a/cps/helper.py b/cps/helper.py index 9771d57a..025fc4d1 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -41,6 +41,7 @@ from flask_babel import gettext as _ from flask_login import current_user from sqlalchemy.sql.expression import true, false, and_, or_, text, func from werkzeug.datastructures import Headers +from werkzeug.security import generate_password_hash try: from urllib.parse import quote @@ -407,6 +408,19 @@ def delete_book_gdrive(book, book_format): return error +def reset_password(user_id): + existing_user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() + password = generate_random_password() + existing_user.password = generate_password_hash(password) + try: + ub.session.commit() + send_registration_mail(existing_user.email, existing_user.nickname, password, True) + return (1, existing_user.nickname) + except Exception: + ub.session.rollback() + return (0, None) + + def generate_random_password(): s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?" passlen = 8 diff --git a/cps/templates/login.html b/cps/templates/login.html index 4f01157b..4a259287 100644 --- a/cps/templates/login.html +++ b/cps/templates/login.html @@ -18,6 +18,9 @@
    + {% if config.config_login_type == 0 and mail%} + + {% endif %} {% if config.config_remote_login %} {{_('Log in with magic link')}} {% endif %} diff --git a/cps/web.py b/cps/web.py index 7823be39..572ac969 100644 --- a/cps/web.py +++ b/cps/web.py @@ -38,8 +38,7 @@ from flask import render_template, request, redirect, send_from_directory, make_ from flask_babel import gettext as _ from flask_login import login_user, logout_user, login_required, current_user from sqlalchemy.exc import IntegrityError -from sqlalchemy.sql.expression import text, func, true, false, not_, and_, \ - exists +from sqlalchemy.sql.expression import text, func, true, false, not_, and_, exists from werkzeug.exceptions import default_exceptions from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash, check_password_hash @@ -50,7 +49,7 @@ from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \ order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \ get_book_cover, get_download_link, send_mail, generate_random_password, send_registration_mail, \ - check_send_to_kindle, check_read_formats, lcase, tags_filters + check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back @@ -1043,7 +1042,7 @@ def download_link(book_id, book_format): @download_required def send_to_kindle(book_id, book_format, convert): settings = config.get_mail_settings() - if settings.get("mail_server", "mail.example.com") == "mail.example.com": + if settings.get("mail_server", "mail.example.org") == "mail.example.org": flash(_(u"Please configure the SMTP mail settings first..."), category="error") elif current_user.kindle_mail: result = send_mail(book_id, book_format, convert, current_user.kindle_mail, config.config_calibre_dir, @@ -1140,17 +1139,33 @@ def login(): log.info('LDAP Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) flash(_(u"Wrong Username or Password"), category="error") else: - if user and check_password_hash(str(user.password), form['password']) and user.nickname != "Guest": - login_user(user, remember=True) - flash(_(u"You are now logged in as: '%(nickname)s'", nickname=user.nickname), category="success") - return redirect_back(url_for("web.index")) ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) - log.info('Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) - flash(_(u"Wrong Username or Password"), category="error") + if 'forgot' in form and form['forgot'] == 'forgot': + if user != None and user.nickname != "Guest": + ret, __ = reset_password(user.id) + if ret == 1: + flash(_(u"New Password was send to your email address"), category="info") + log.info('Password reset for user "%s" IP-adress: %s', form['username'], ipAdress) + else: + flash(_(u"An unknown error occurred. Please try again later."), category="error") + else: + flash(_(u"Please enter valid username to reset password"), category="error") + log.info('Username missing for password reset IP-adress: %s', ipAdress) + else: + if user and check_password_hash(str(user.password), form['password']) and user.nickname != "Guest": + login_user(user, remember=True) + flash(_(u"You are now logged in as: '%(nickname)s'", nickname=user.nickname), category="success") + return redirect_back(url_for("web.index")) + else: + log.info('Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) + flash(_(u"Wrong Username or Password"), category="error") + settings = config.get_mail_settings() + mail_configured = bool(settings.get("mail_server", "mail.example.org") != "mail.example.org") next_url = url_for('web.index') - return render_title_template('login.html', title=_(u"login"), next_url=next_url, config=config, page="login") + return render_title_template('login.html', title=_(u"login"), next_url=next_url, config=config, + mail = mail_configured, page="login") @web.route('/logout') From 7098d08888a2f0e340737ba8f97202f81d3712cf Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 15 Dec 2019 18:44:02 +0100 Subject: [PATCH 035/688] Added option to convert AZW3 to mobi for sending to kindle --- cps/editbooks.py | 4 ++-- cps/helper.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 51ed9e48..91134db6 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -729,7 +729,7 @@ def convert_bookformat(book_id): if (book_format_from is None) or (book_format_to is None): flash(_(u"Source or destination format for conversion missing"), category="error") - return redirect(request.environ["HTTP_REFERER"]) + return redirect(url_for('editbook.edit_book', book_id=book_id)) log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to) rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(), @@ -741,4 +741,4 @@ def convert_bookformat(book_id): category="success") else: flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error") - return redirect(request.environ["HTTP_REFERER"]) + return redirect(url_for('editbook.edit_book', book_id=book_id)) diff --git a/cps/helper.py b/cps/helper.py index 025fc4d1..2b92ef75 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -167,10 +167,10 @@ def check_send_to_kindle(entry): if 'EPUB' in formats and not 'MOBI' in formats: bookformats.append({'format': 'Mobi','convert':1, 'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Mobi')}) - '''if config.config_ebookconverter == 2: - if 'EPUB' in formats and not 'AZW3' in formats: - bookformats.append({'format': 'Azw3','convert':1, - 'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Azw3')})''' + if config.config_ebookconverter == 2: + if 'AZW3' in formats and not 'MOBI' in formats: + bookformats.append({'format': 'Mobi','convert':2, + 'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Azw3',format='Mobi')}) return bookformats else: log.error(u'Cannot find book entry %d', entry.id) @@ -197,9 +197,13 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): """Send email with attachments""" book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - if convert: + if convert == 1: # returns None if success, otherwise errormessage return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail) + if convert == 2: + # returns None if success, otherwise errormessage + return convert_book_format(book_id, calibrepath, u'azw3', book_format.lower(), user_id, kindle_mail) + for entry in iter(book.data): if entry.format.upper() == book_format.upper(): From f705889c2353b82f3da28e41c58e6e77c3162d23 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Tue, 17 Dec 2019 18:00:35 +0100 Subject: [PATCH 036/688] Inital Kobo --- cps.py | 3 + cps/admin.py | 14 + cps/config_sql.py | 1 + cps/db.py | 6 +- cps/helper.py | 26 +- cps/kobo.py | 523 +++++++++++++++++++++++++++++++++ cps/kobo_auth.py | 102 +++++++ cps/templates/config_edit.html | 4 + cps/templates/user_edit.html | 4 + cps/ub.py | 13 +- cps/web.py | 4 +- requirements.txt | 1 + 12 files changed, 688 insertions(+), 13 deletions(-) create mode 100644 cps/kobo.py create mode 100644 cps/kobo_auth.py diff --git a/cps.py b/cps.py index ca7d7230..412604d2 100755 --- a/cps.py +++ b/cps.py @@ -41,6 +41,8 @@ from cps.shelf import shelf from cps.admin import admi from cps.gdrive import gdrive from cps.editbooks import editbook +from cps.kobo import kobo + try: from cps.oauth_bb import oauth oauth_available = True @@ -58,6 +60,7 @@ def main(): app.register_blueprint(admi) app.register_blueprint(gdrive) app.register_blueprint(editbook) + app.register_blueprint(kobo) if oauth_available: app.register_blueprint(oauth) success = web_server.start() diff --git a/cps/admin.py b/cps/admin.py index 57796080..50e0589d 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -299,6 +299,8 @@ def _configuration_update_helper(): reboot_required |= _config_string("config_certfile") if config.config_certfile and not os.path.isfile(config.config_certfile): return _configuration_result('Certfile location is not valid, please enter correct path', gdriveError) + + _config_string("config_server_url") _config_checkbox_int("config_uploading") _config_checkbox_int("config_anonbrowse") @@ -597,6 +599,18 @@ def edit_user(user_id): content.default_language = to_save["default_language"] if "locale" in to_save and to_save["locale"]: content.locale = to_save["locale"] + + if "kobo_user_key" in to_save and to_save["kobo_user_key"]: + kobo_user_key_hash = generate_password_hash(to_save["kobo_user_key"]) + if kobo_user_key_hash != content.kobo_user_key_hash: + existing_kobo_user_key = ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash == kobo_user_key_hash).first() + if not existing_kobo_user_key: + content.kobo_user_key_hash = kobo_user_key_hash + else: + flash(_(u"Found an existing account for this Kobo UserKey."), category="error") + return render_title_template("user_edit.html", translations=translations, languages=languages, + new_user=0, content=content, downloads=downloads, registered_oauth=oauth_check, + title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") if to_save["email"] and to_save["email"] != content.email: existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ .first() diff --git a/cps/config_sql.py b/cps/config_sql.py index 809e97d8..8ea8b978 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -49,6 +49,7 @@ class _Settings(_Base): config_port = Column(Integer, default=constants.DEFAULT_PORT) config_certfile = Column(String) config_keyfile = Column(String) + config_server_url = Column(String, default='') config_calibre_web_title = Column(String, default=u'Calibre-Web') config_books_per_page = Column(Integer, default=60) diff --git a/cps/db.py b/cps/db.py index b9853896..5765bf68 100755 --- a/cps/db.py +++ b/cps/db.py @@ -25,7 +25,7 @@ import ast from sqlalchemy import create_engine from sqlalchemy import Table, Column, ForeignKey -from sqlalchemy import String, Integer, Boolean +from sqlalchemy import String, Integer, Boolean, TIMESTAMP from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base @@ -251,10 +251,10 @@ class Books(Base): title = Column(String) sort = Column(String) author_sort = Column(String) - timestamp = Column(String) + timestamp = Column(TIMESTAMP) pubdate = Column(String) series_index = Column(String) - last_modified = Column(String) + last_modified = Column(TIMESTAMP) path = Column(String) has_cover = Column(Integer) uuid = Column(String) diff --git a/cps/helper.py b/cps/helper.py index 2b92ef75..e5e616d5 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -446,32 +446,46 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) +def get_cover_on_failure(use_generic_cover): + if use_generic_cover: + return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + else: + return None + def get_book_cover(book_id): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - if book.has_cover: + return get_book_cover_internal(book, use_generic_cover_on_failure=True) +def get_book_cover_with_uuid(book_uuid, + use_generic_cover_on_failure=True): + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + return get_book_cover_internal(book, use_generic_cover_on_failure) + +def get_book_cover_internal(book, + use_generic_cover_on_failure): + if book and book.has_cover: if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): - return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) path=gd.get_cover_via_gdrive(book.path) if path: return redirect(path) else: log.error('%s/cover.jpg not found on Google Drive', book.path) - return send_from_directory(_STATIC_DIR, "generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) except Exception as e: log.exception(e) # traceback.print_exc() - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): return send_from_directory(cover_file_path, "cover.jpg") else: - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) else: - return send_from_directory(_STATIC_DIR,"generic_cover.jpg") + return get_cover_on_failure(use_generic_cover_on_failure) # saves book cover from url diff --git a/cps/kobo.py b/cps/kobo.py new file mode 100644 index 00000000..270f5c33 --- /dev/null +++ b/cps/kobo.py @@ -0,0 +1,523 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 shavitmichael, OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +import uuid +from base64 import b64decode, b64encode +from datetime import datetime +from time import gmtime, strftime + +from jsonschema import validate, exceptions +from flask import Blueprint, request, make_response, jsonify, json +from flask_login import login_required +from sqlalchemy import func + +from . import config, logger, kobo_auth, db, helper +from .web import download_required + +kobo = Blueprint("kobo", __name__) +kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) + +log = logger.create() + + +def b64encode_json(json_data): + if sys.version_info < (3, 0): + return b64encode(json.dumps(json_data)) + else: + return b64encode(json.dumps(json_data).encode()) + + +# Python3 has a timestamp() method we could be calling, however it's not avaiable in python2. +def to_epoch_timestamp(datetime_object): + return (datetime_object - datetime(1970, 1, 1)).total_seconds() + + +class SyncToken: + """ The SyncToken is used to persist state accross requests. + When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service. + As an example use-case, the SyncToken is used to detect books that have been added to the library since the last time the device synced to the server. + + Attributes: + books_last_created: Datetime representing the newest book that the device knows about. + books_last_modified: Datetime representing the last modified book that the device knows about. + """ + + SYNC_TOKEN_HEADER = "x-kobo-synctoken" + VERSION = "1-0-0" + MIN_VERSION = "1-0-0" + + token_schema = { + "type": "object", + "properties": {"version": {"type": "string"}, "data": {"type": "object"},}, + } + # This Schema doesn't contain enough information to detect and propagate book deletions from Calibre to the device. + # A potential solution might be to keep a list of all known book uuids in the token, and look for any missing from the db. + data_schema_v1 = { + "type": "object", + "properties": { + "raw_kobo_store_token": {"type": "string"}, + "books_last_modified": {"type": "string"}, + "books_last_created": {"type": "string"}, + }, + } + + def __init__( + self, + raw_kobo_store_token="", + books_last_created=datetime.min, + books_last_modified=datetime.min, + ): + self.raw_kobo_store_token = raw_kobo_store_token + self.books_last_created = books_last_created + self.books_last_modified = books_last_modified + + @staticmethod + def from_headers(headers): + sync_token_header = headers.get(SyncToken.SYNC_TOKEN_HEADER, "") + if sync_token_header == "": + return SyncToken() + + # On the first sync from a Kobo device, we may receive the SyncToken + # from the official Kobo store. Without digging too deep into it, that + # token is of the form [b64encoded blob].[b64encoded blob 2] + if "." in sync_token_header: + return SyncToken(raw_kobo_store_token=sync_token_header) + + try: + sync_token_json = json.loads( + b64decode(sync_token_header + "=" * (-len(sync_token_header) % 4)) + ) + validate(sync_token_json, SyncToken.token_schema) + if sync_token_json["version"] < SyncToken.MIN_VERSION: + raise ValueError + + data_json = sync_token_json["data"] + validate(sync_token_json, SyncToken.data_schema_v1) + except (exceptions.ValidationError, ValueError) as e: + log.error("Sync token contents do not follow the expected json schema.") + return SyncToken() + + raw_kobo_store_token = data_json["raw_kobo_store_token"] + try: + books_last_modified = datetime.utcfromtimestamp( + data_json["books_last_modified"] + ) + books_last_created = datetime.utcfromtimestamp( + data_json["books_last_created"] + ) + except TypeError: + log.error("SyncToken timestamps don't parse to a datetime.") + return SyncToken(raw_kobo_store_token=raw_kobo_store_token) + + return SyncToken( + raw_kobo_store_token=raw_kobo_store_token, + books_last_created=books_last_created, + books_last_modified=books_last_modified, + ) + + def to_headers(self, headers): + headers[SyncToken.SYNC_TOKEN_HEADER] = self.build_sync_token() + + def build_sync_token(self): + token = { + "version": SyncToken.VERSION, + "data": { + "raw_kobo_store_token": self.raw_kobo_store_token, + "books_last_modified": to_epoch_timestamp(self.books_last_modified), + "books_last_created": to_epoch_timestamp(self.books_last_created), + }, + } + return b64encode_json(token) + + +@kobo.route("/v1/library/sync") +@login_required +@download_required +def HandleSyncRequest(): + sync_token = SyncToken.from_headers(request.headers) + log.info("Kobo library sync request received.") + + # TODO: Limit the number of books return per sync call, and rely on the sync-continuatation header + # instead so that the device triggers another sync. + + new_books_last_modified = sync_token.books_last_modified + new_books_last_created = sync_token.books_last_created + entitlements = [] + + # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast. + # It looks like it's treating the db.Books.last_modified field as a string and may fail + # the comparison because of the +00:00 suffix. + changed_entries = ( + db.session.query(db.Books) + .filter(func.datetime(db.Books.last_modified) != sync_token.books_last_modified) + .all() + ) + for book in changed_entries: + entitlement = { + "BookEntitlement": create_book_entitlement(book), + "BookMetadata": get_metadata(book), + "ReadingState": reading_state(book), + } + if book.timestamp > sync_token.books_last_created: + entitlements.append({"NewEntitlement": entitlement}) + else: + entitlements.append({"ChangedEntitlement": entitlement}) + + new_books_last_modified = max( + book.last_modified, sync_token.books_last_modified + ) + new_books_last_created = max(book.timestamp, sync_token.books_last_modified) + + sync_token.books_last_created = new_books_last_created + sync_token.books_last_modified = new_books_last_modified + + # Missing feature: Detect server-side book deletions. + + # Missing feature: Join the response with results from the official Kobo store so that users can still buy and access books from the device store (particularly while on-the-road). + + response = make_response(jsonify(entitlements)) + + sync_token.to_headers(response.headers) + response.headers["x-kobo-sync-mode"] = "delta" + response.headers["x-kobo-apitoken"] = "e30=" + return response + + +@kobo.route("/v1/library//metadata") +@login_required +@download_required +def HandleMetadataRequest(book_uuid): + log.info("Kobo library metadata request received for book %s" % book_uuid) + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + if not book or not book.data: + log.info(u"Book %s not found in database", book_uuid) + return make_response("Book not found in database.", 404) + + metadata = get_metadata(book) + return jsonify([metadata]) + + +def get_download_url_for_book(book, book_format): + return "{url_base}/download/{book_id}/{book_format}".format( + url_base=request.environ['werkzeug.request'].base_url, + book_id=book.id, + book_format=book_format.lower(), + ) + + +def create_book_entitlement(book): + book_uuid = book.uuid + return { + "Accessibility": "Full", + "ActivePeriod": {"From": current_time(),}, + "Created": book.timestamp, + "CrossRevisionId": book_uuid, + "Id": book_uuid, + "IsHiddenFromArchive": False, + "IsLocked": False, + # Setting this to true removes from the device. + "IsRemoved": False, + "LastModified": book.last_modified, + "OriginCategory": "Imported", + "RevisionId": book_uuid, + "Status": "Active", + } + + +def current_time(): + return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) + + +def get_description(book): + if not book.comments: + return None + return book.comments[0].text + + +# TODO handle multiple authors +def get_author(book): + if not book.authors: + return None + return book.authors[0].name + + +def get_publisher(book): + if not book.publishers: + return None + return book.publishers[0].name + + +def get_series(book): + if not book.series: + return None + return book.series[0].name + + +def get_metadata(book): + ALLOWED_FORMATS = {"KEPUB"} + download_urls = [] + + for book_data in book.data: + if book_data.format in ALLOWED_FORMATS: + download_urls.append( + { + "Format": book_data.format, + "Size": book_data.uncompressed_size, + "Url": get_download_url_for_book(book, book_data.format), + # "DrmType": "None", # Not required + "Platform": "Android", # Required field. + } + ) + + book_uuid = book.uuid + metadata = { + "Categories": ["00000000-0000-0000-0000-000000000001",], + "Contributors": get_author(book), + "CoverImageId": book_uuid, + "CrossRevisionId": book_uuid, + "CurrentDisplayPrice": {"CurrencyCode": "USD", "TotalAmount": 0}, + "CurrentLoveDisplayPrice": {"TotalAmount": 0}, + "Description": get_description(book), + "DownloadUrls": download_urls, + "EntitlementId": book_uuid, + "ExternalIds": [], + "Genre": "00000000-0000-0000-0000-000000000001", + "IsEligibleForKoboLove": False, + "IsInternetArchive": False, + "IsPreOrder": False, + "IsSocialEnabled": True, + "Language": "en", + "PhoneticPronunciations": {}, + "PublicationDate": "2019-02-03T00:25:03.0000000Z", # current_time(), + "Publisher": {"Imprint": "", "Name": get_publisher(book),}, + "RevisionId": book_uuid, + "Title": book.title, + "WorkId": book_uuid, + } + + if get_series(book): + if sys.version_info < (3, 0): + name = get_series(book).encode("utf-8") + else: + name = get_series(book) + metadata["Series"] = { + "Name": get_series(book), + "Number": book.series_index, + "NumberFloat": float(book.series_index), + # Get a deterministic id based on the series name. + "Id": uuid.uuid3(uuid.NAMESPACE_DNS, name), + } + + return metadata + + +def reading_state(book): + # TODO: Implement + reading_state = { + # "StatusInfo": { + # "LastModified": get_single_cc_value(book, "lastreadtimestamp"), + # "Status": get_single_cc_value(book, "reading_status"), + # } + # TODO: CurrentBookmark, Location + } + return reading_state + + +@kobo.route( + "//////image.jpg" +) +def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monochrome): + book_cover = helper.get_book_cover_with_uuid( + book_uuid, use_generic_cover_on_failure=False + ) + if not book_cover: + return make_response() + return book_cover + + +@kobo.route("/v1/user/profile") +@kobo.route("/v1/user/loyalty/benefits") +@kobo.route("/v1/analytics/gettests", methods=["GET", "POST"]) +@kobo.route("/v1/user/wishlist") +@kobo.route("/v1/user/") +@kobo.route("/v1/user/recommendations") +@kobo.route("/v1/products/") +@kobo.route("/v1/products//nextread") +@kobo.route("/v1/products/featured/") +@kobo.route("/v1/products/featured/") +@kobo.route("/v1/library/", methods=["DELETE", "GET"]) # TODO: implement +def HandleDummyRequest(dummy=None): + return make_response(jsonify({})) + + +@kobo.route("/v1/auth/device", methods=["POST"]) +def HandleAuthRequest(): + # This AuthRequest isn't used for most of our usecases. + response = make_response( + jsonify( + { + "AccessToken": "abcde", + "RefreshToken": "abcde", + "TokenType": "Bearer", + "TrackingId": "abcde", + "UserKey": "abcdefgeh", + } + ) + ) + return response + + +@kobo.route("/v1/initialization") +def HandleInitRequest(): + resources = NATIVE_KOBO_RESOURCES(calibre_web_url=config.config_server_url) + response = make_response(jsonify({"Resources": resources})) + response.headers["x-kobo-apitoken"] = "e30=" + return response + + +def NATIVE_KOBO_RESOURCES(calibre_web_url): + return { + "account_page": "https://secure.kobobooks.com/profile", + "account_page_rakuten": "https://my.rakuten.co.jp/", + "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}", + "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate", + "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion", + "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations", + "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete", + "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"}, + "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}", + "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}", + "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}", + "book_landing_page": "https://store.kobobooks.com/ebooks", + "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions", + "categories": "https://storeapi.kobo.com/v1/categories", + "categories_page": "https://store.kobobooks.com/ebooks/categories", + "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}", + "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured", + "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products", + "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow", + "configuration_data": "https://storeapi.kobo.com/v1/configuration", + "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access", + "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO", + "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal", + "deals": "https://storeapi.kobo.com/v1/deals", + "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}", + "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete", + "device_auth": "https://storeapi.kobo.com/v1/auth/device", + "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh", + "dictionary_host": "https://kbdownload1-a.akamaihd.net", + "discovery_host": "https://discovery.kobobooks.com", + "eula_page": "https://www.kobo.com/termsofuse?style=onestore", + "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange", + "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}", + "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/", + "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}", + "featured_lists": "https://storeapi.kobo.com/v1/products/featured", + "free_books_page": { + "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks", + "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits", + "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti", + "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", + "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis", + }, + "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback", + "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests", + "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader", + "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem", + "help_page": "http://www.kobo.com/help", + "image_host": calibre_web_url, + "image_url_quality_template": calibre_web_url + + "/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg", + "image_url_template": calibre_web_url + + "/{ImageId}/{Width}/{Height}/false/image.jpg", + "kobo_audiobooks_enabled": "False", + "kobo_audiobooks_orange_deal_enabled": "False", + "kobo_audiobooks_subscriptions_enabled": "False", + "kobo_nativeborrow_enabled": "True", + "kobo_onestorelibrary_enabled": "False", + "kobo_redeem_enabled": "True", + "kobo_shelfie_enabled": "False", + "kobo_subscriptions_enabled": "False", + "kobo_superpoints_enabled": "False", + "kobo_wishlist_enabled": "True", + "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}", + "library_items": "https://storeapi.kobo.com/v1/user/library", + "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata", + "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices", + "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}", + "library_sync": "https://storeapi.kobo.com/v1/library/sync", + "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints", + "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}", + "magazine_landing_page": "https://store.kobobooks.com/emagazines", + "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration", + "oauth_host": "https://oauth.kobo.com", + "overdrive_account": "https://auth.overdrive.com/account", + "overdrive_library": "https://{libraryKey}.auth.overdrive.com/library", + "overdrive_library_finder_host": "https://libraryfinder.api.overdrive.com", + "overdrive_thunder_host": "https://thunder.api.overdrive.com", + "password_retrieval_page": "https://www.kobobooks.com/passwordretrieval.html", + "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event", + "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore", + "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread", + "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices", + "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations", + "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews", + "products": "https://storeapi.kobo.com/v1/products", + "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/", + "purchase_buy": "https://www.kobo.com/checkout/createpurchase/", + "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}", + "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout", + "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase", + "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}", + "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state", + "redeem_interstitial_page": "https://store.kobobooks.com", + "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://store.kobobooks.com/", + "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related", + "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}", + "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}", + "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}", + "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie", + "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://store.kobobooks.com/", + "social_authorization_host": "https://social.kobobooks.com:8443", + "social_host": "https://social.kobobooks.com", + "stacks_host_productId": "https://store.kobobooks.com/collections/byproductid/", + "store_home": "www.kobo.com/{region}/{language}", + "store_host": "store.kobobooks.com", + "store_newreleases": "https://store.kobobooks.com/{culture}/List/new-releases/961XUjtsU0qxkFItWOutGA", + "store_search": "https://store.kobobooks.com/{culture}/Search?Query={query}", + "store_top50": "https://store.kobobooks.com/{culture}/ebooks/Top", + "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items", + "tags": "https://storeapi.kobo.com/v1/library/tags", + "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile", + "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview", + "use_one_store": "False", + "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits", + "user_platform": "https://storeapi.kobo.com/v1/user/platform", + "user_profile": "https://storeapi.kobo.com/v1/user/profile", + "user_ratings": "https://storeapi.kobo.com/v1/user/ratings", + "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations", + "user_reviews": "https://storeapi.kobo.com/v1/user/reviews", + "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist", + "userguide_host": "https://kbdownload1-a.akamaihd.net", + "wishlist_page": "https://store.kobobooks.com/{region}/{language}/account/wishlist", + } diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py new file mode 100644 index 00000000..1504c25b --- /dev/null +++ b/cps/kobo_auth.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 shavitmichael, OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +"""This module is used to control authentication/authorization of Kobo sync requests. +This module also includes research notes into the auth protocol used by Kobo devices. + +Log-in: +When first booting a Kobo device the user must sign into a Kobo (or affiliate) account. +Upon successful sign-in, the user is redirected to + https://auth.kobobooks.com/CrossDomainSignIn?id= +which serves the following response: + . +And triggers the insertion of a userKey into the device's User table. + +IMPORTANT SECURITY CAUTION: +Together, the device's DeviceId and UserKey act as an *irrevocable* authentication +token to most (if not all) Kobo APIs. In fact, in most cases only the UserKey is +required to authorize the API call. + +Changing Kobo password *does not* invalidate user keys! This is apparently a known +issue for a few years now https://www.mobileread.com/forums/showpost.php?p=3476851&postcount=13 +(although this poster hypothesised that Kobo could blacklist a DeviceId, many endpoints +will still grant access given the userkey.) + +Official Kobo Store Api authorization: +* For most of the endpoints we care about (sync, metadata, tags, etc), the userKey is +passed in the x-kobo-userkey header, and is sufficient to authorize the API call. +* Some endpoints (e.g: AnnotationService) instead make use of Bearer tokens pass through +an authorization header. To get a BearerToken, the device makes a POST request to the +v1/auth/device endpoint with the secret UserKey and the device's DeviceId. +* The book download endpoint passes an auth token as a URL param instead of a header. + +Our implementation: +For now, we rely on the official Kobo store's UserKey for authentication. +Once authenticated, we set the login cookie on the response that will be sent back for +the duration of the session to authorize subsequent API calls. +Ideally we'd only perform UserKey-based authentication for the v1/initialization or the +v1/device/auth call, however sessions don't always start with those calls. + +Because of the irrevocable power granted by the key, we only ever store and compare a +hash of the key. To obtain their UserKey, a user can query the user table from the +.kobo/KoboReader.sqlite database found on their device. +This isn't exactly user friendly however. + +Some possible alternatives that require more research: + * Instead of having users query the device database to find out their UserKey, we could + provide a list of recent Kobo sync attempts in the calibre-web UI for users to + authenticate sync attempts (e.g: 'this was me' button). + * We may be able to craft a sign-in flow with a redirect back to the CalibreWeb + server containing the KoboStore's UserKey (if the same as the devices?). + * Can we create our own UserKey instead of relying on the real store's userkey? + (Maybe using something like location.href=kobo://UserAuthenticated?userId=...?) +""" + +from functools import wraps +from flask import request, make_response +from flask_login import login_user +from werkzeug.security import check_password_hash + +from . import logger, ub, lm + +USER_KEY_HEADER = "x-kobo-userkey" +USER_KEY_URL_PARAM = "kobo_userkey" + +log = logger.create() + + +def disable_failed_auth_redirect_for_blueprint(bp): + lm.blueprint_login_views[bp.name] = None + + +@lm.request_loader +def load_user_from_kobo_request(request): + user_key = request.headers.get(USER_KEY_HEADER) + if user_key: + for user in ( + ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash != "").all() + ): + if check_password_hash(str(user.kobo_user_key_hash), user_key): + # The Kobo device won't preserve the cookie accross sessions, even if we + # were to set remember_me=true. + login_user(user) + return user + log.info("Received Kobo request without a recognizable UserKey.") + return None diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 85b9598e..9556eef4 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -104,6 +104,10 @@
    +
    + + +
    diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index e4e36c64..5ace1eab 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -28,6 +28,10 @@
    +
    + + +
    -
    - - -
    -
    - - -
    From 2118d920f544035d0dd99899d218a7cf471264d8 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Fri, 20 Dec 2019 01:04:12 -0500 Subject: [PATCH 039/688] Formatter. --- cps/kobo.py | 15 ++++++++++++--- cps/kobo_auth.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index db16fe9a..2acecb6c 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -33,7 +33,7 @@ from sqlalchemy import func from . import config, logger, kobo_auth, db, helper from .web import download_required -kobo = Blueprint("kobo", __name__, url_prefix='/kobo/') +kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) kobo_auth.register_url_value_preprocessor(kobo) @@ -218,7 +218,12 @@ def HandleMetadataRequest(book_uuid): def get_download_url_for_book(book, book_format): - return url_for("web.download_link", book_id=book.id, book_format=book_format.lower(), _external = True) + return url_for( + "web.download_link", + book_id=book.id, + book_format=book_format.lower(), + _external=True, + ) def create_book_entitlement(book): @@ -350,10 +355,12 @@ def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monoc return make_response() return book_cover + @kobo.route("") def TopLevelEndpoint(): return make_response(jsonify({})) + @kobo.route("/v1/user/profile") @kobo.route("/v1/user/loyalty/benefits") @kobo.route("/v1/analytics/gettests/", methods=["GET", "POST"]) @@ -388,7 +395,9 @@ def HandleAuthRequest(): @kobo.route("/v1/initialization") def HandleInitRequest(): - resources = NATIVE_KOBO_RESOURCES(calibre_web_url=url_for("web.index", _external=True).strip("/")) + resources = NATIVE_KOBO_RESOURCES( + calibre_web_url=url_for("web.index", _external=True).strip("/") + ) response = make_response(jsonify({"Resources": resources})) response.headers["x-kobo-apitoken"] = "e30=" return response diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index fef92599..0b9eba6e 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -70,10 +70,11 @@ from .web import render_title_template log = logger.create() + def register_url_value_preprocessor(kobo): @kobo.url_value_preprocessor def pop_auth_token(endpoint, values): - g.auth_token = values.pop('auth_token') + g.auth_token = values.pop("auth_token") def disable_failed_auth_redirect_for_blueprint(bp): @@ -82,31 +83,44 @@ def disable_failed_auth_redirect_for_blueprint(bp): @lm.request_loader def load_user_from_kobo_request(request): - if 'auth_token' in g: - auth_token = g.get('auth_token') - user = ub.session.query(ub.User).join(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == auth_token).first() + if "auth_token" in g: + auth_token = g.get("auth_token") + user = ( + ub.session.query(ub.User) + .join(ub.RemoteAuthToken) + .filter(ub.RemoteAuthToken.auth_token == auth_token) + .first() + ) if user is not None: login_user(user) return user log.info("Received Kobo request without a recognizable auth token.") return None -kobo_auth = Blueprint("kobo_auth", __name__, url_prefix='/kobo_auth') -@kobo_auth.route('/generate_auth_token') +kobo_auth = Blueprint("kobo_auth", __name__, url_prefix="/kobo_auth") + + +@kobo_auth.route("/generate_auth_token") @login_required def generate_auth_token(): # Invalidate any prevously generated Kobo Auth token for this user. - ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == current_user.id).delete() + ub.session.query(ub.RemoteAuthToken).filter( + ub.RemoteAuthToken.user_id == current_user.id + ).delete() auth_token = ub.RemoteAuthToken() auth_token.user_id = current_user.id auth_token.expiration = datetime.max - auth_token.auth_token = (hexlify(urandom(16))).decode('utf-8') + auth_token.auth_token = (hexlify(urandom(16))).decode("utf-8") ub.session.add(auth_token) ub.session.commit() - - return render_title_template('generate_kobo_auth_url.html', title=_(u"Kobo Set-up"), - kobo_auth_url=url_for("kobo.TopLevelEndpoint", auth_token=auth_token.auth_token, _external=True)) \ No newline at end of file + return render_title_template( + "generate_kobo_auth_url.html", + title=_(u"Kobo Set-up"), + kobo_auth_url=url_for( + "kobo.TopLevelEndpoint", auth_token=auth_token.auth_token, _external=True + ), + ) From f84274f1c534a5d589fd844239c1a61a6d2f0fc7 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Fri, 20 Dec 2019 01:08:15 -0500 Subject: [PATCH 040/688] git add missing generate_kobo_auth_url.html --- cps/templates/generate_kobo_auth_url.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cps/templates/generate_kobo_auth_url.html diff --git a/cps/templates/generate_kobo_auth_url.html b/cps/templates/generate_kobo_auth_url.html new file mode 100644 index 00000000..28b098cf --- /dev/null +++ b/cps/templates/generate_kobo_auth_url.html @@ -0,0 +1,15 @@ +{% extends "layout.html" %} +{% block body %} +
    +

    {{_('Generate Kobo Auth URL')}}

    +

    + {{_('Open the .kobo/Kobo eReader.conf file in a text editor and add (or edit):')}}. +

    +

    + {{_('api_endpoint=')}}{{kobo_auth_url}} +

    +

    + {{_('Please note that every visit to this current page invalidates any previously generated Authentication url for this user.')}}. +

    +
    +{% endblock %} \ No newline at end of file From d6a974682406edd366cd972b0823002526ec06dc Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Fri, 20 Dec 2019 01:28:53 -0500 Subject: [PATCH 041/688] Add a filter to the Sync request endpoint to ignore books that don't have any formats supported by the device. --- cps/kobo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 2acecb6c..c44915c4 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -33,6 +33,9 @@ from sqlalchemy import func from . import config, logger, kobo_auth, db, helper from .web import download_required +#TODO: Test more formats :) . +KOBO_SUPPORTED_FORMATS = {"KEPUB"} + kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) kobo_auth.register_url_value_preprocessor(kobo) @@ -169,7 +172,9 @@ def HandleSyncRequest(): # the comparison because of the +00:00 suffix. changed_entries = ( db.session.query(db.Books) + .join(db.Data) .filter(func.datetime(db.Books.last_modified) > sync_token.books_last_modified) + .filter(db.Data.format.in_(KOBO_SUPPORTED_FORMATS)) .all() ) for book in changed_entries: @@ -275,11 +280,10 @@ def get_series(book): def get_metadata(book): - ALLOWED_FORMATS = {"KEPUB"} download_urls = [] for book_data in book.data: - if book_data.format in ALLOWED_FORMATS: + if book_data.format in KOBO_SUPPORTED_FORMATS: download_urls.append( { "Format": book_data.format, From f2c07d8f81322dfbc07e93f8aeb347abe793a1c1 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Fri, 20 Dec 2019 19:17:08 +0100 Subject: [PATCH 042/688] Update Kobo sync --- cps/kobo.py | 18 +++++++++++------- cps/web.py | 4 +++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 270f5c33..26ec734a 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -27,7 +27,7 @@ from time import gmtime, strftime from jsonschema import validate, exceptions from flask import Blueprint, request, make_response, jsonify, json from flask_login import login_required -from sqlalchemy import func +from sqlalchemy import func, or_ from . import config, logger, kobo_auth, db, helper from .web import download_required @@ -166,8 +166,9 @@ def HandleSyncRequest(): # It looks like it's treating the db.Books.last_modified field as a string and may fail # the comparison because of the +00:00 suffix. changed_entries = ( - db.session.query(db.Books) + db.session.query(db.Books).join(db.Data) .filter(func.datetime(db.Books.last_modified) != sync_token.books_last_modified) + .filter(or_(db.Data.format == 'KEPUB', db.Data.format == 'EPUB')) .all() ) for book in changed_entries: @@ -217,9 +218,9 @@ def HandleMetadataRequest(book_uuid): def get_download_url_for_book(book, book_format): return "{url_base}/download/{book_id}/{book_format}".format( - url_base=request.environ['werkzeug.request'].base_url, + url_base=get_base_url(), # request.environ['werkzeug.request'].base_url, book_id=book.id, - book_format=book_format.lower(), + book_format="kepub", ) @@ -272,14 +273,14 @@ def get_series(book): def get_metadata(book): - ALLOWED_FORMATS = {"KEPUB"} + ALLOWED_FORMATS = {"KEPUB", "EPUB"} download_urls = [] for book_data in book.data: if book_data.format in ALLOWED_FORMATS: download_urls.append( { - "Format": book_data.format, + "Format": "KEPUB", "Size": book_data.uncompressed_size, "Url": get_download_url_for_book(book, book_data.format), # "DrmType": "None", # Not required @@ -385,9 +386,12 @@ def HandleAuthRequest(): return response +def get_base_url(): + return "{root}:{port}".format(root=request.url_root[:-1], port=str(config.config_port)) + @kobo.route("/v1/initialization") def HandleInitRequest(): - resources = NATIVE_KOBO_RESOURCES(calibre_web_url=config.config_server_url) + resources = NATIVE_KOBO_RESOURCES(calibre_web_url=get_base_url()) response = make_response(jsonify({"Resources": resources})) response.headers["x-kobo-apitoken"] = "e30=" return response diff --git a/cps/web.py b/cps/web.py index f6d2ab6c..995ed13c 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1029,11 +1029,13 @@ def serve_book(book_id, book_format, anyname): return send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format) -# @web.route("/download//", defaults={'anyname': 'None'}) @web.route("/download//") @login_required_if_no_ano @download_required def download_link(book_id, book_format): + if book_format.lower() == "kepub": + book_format= "epub" + log.info("Book %s in format %s downloaded", str(book_id), book_format) return get_download_link(book_id, book_format) From b586a32843bf9bf5f4528a053d177fc0b967af8c Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 22 Dec 2019 15:24:22 +0100 Subject: [PATCH 043/688] Fix #1115 (comic reader not working under iOS, maybe invalid issue) Improvement for #925 (Next/Prev buttons are bigger) --- cps/static/css/kthoom.css | 33 ----------------------- cps/static/css/listen.css | 24 +---------------- cps/static/css/main.css | 22 +++++++-------- cps/static/js/archive/rarvm.js | 12 +++++++-- cps/static/js/reading/epub.js | 4 ++- cps/templates/readtxt.html | 49 +++++++--------------------------- 6 files changed, 33 insertions(+), 111 deletions(-) diff --git a/cps/static/css/kthoom.css b/cps/static/css/kthoom.css index 6dfb9967..d7668156 100644 --- a/cps/static/css/kthoom.css +++ b/cps/static/css/kthoom.css @@ -152,39 +152,6 @@ body { max-width: 70%; } -#left { - left: 40px; -} - -#right { - right: 40px; -} - -.arrow { - position: absolute; - top: 50%; - margin-top: -32px; - font-size: 64px; - color: #E2E2E2; - font-family: arial, sans-serif; - font-weight: bold; - cursor: pointer; - -webkit-user-select: none; - -khtml-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; -} - -.arrow:hover { - color: #777; -} - -.arrow:active, -.arrow.active { - color: #000; -} - th, td { padding: 5px; } diff --git a/cps/static/css/listen.css b/cps/static/css/listen.css index b08cc33c..0e34d163 100644 --- a/cps/static/css/listen.css +++ b/cps/static/css/listen.css @@ -65,28 +65,6 @@ right: 40px; } - .arrow { - position: absolute; - top: 50%; - margin-top: -32px; - font-size: 64px; - color: #E2E2E2; - font-family: arial, sans-serif; - font-weight: bold; - cursor: pointer; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; - } - - .arrow:hover { - color: #777; - } - - .arrow:active { - color: #000; - } - xmp, pre, plaintext { @@ -111,4 +89,4 @@ -moz-column-gap: 20px; -webkit-column-gap: 20px; position: relative; - } \ No newline at end of file + } diff --git a/cps/static/css/main.css b/cps/static/css/main.css index 4afb28f7..60470f3b 100644 --- a/cps/static/css/main.css +++ b/cps/static/css/main.css @@ -15,16 +15,10 @@ body { } #main { - /* height: 500px; */ position: absolute; width: 100%; height: 100%; right: 0; - /* left: 40px; */ -/* -webkit-transform: translate(40px, 0); - -moz-transform: translate(40px, 0); */ - - /* border-radius: 5px 0px 0px 5px; */ border-radius: 5px; background: #fff; overflow: hidden; @@ -114,18 +108,20 @@ body { border: none; } -#prev { +#left,#prev { left: 40px; + padding-right:80px; } -#next { +#right,#next { right: 40px; + padding-left:80px; } .arrow { position: absolute; top: 50%; - margin-top: -32px; + margin-top: -192px; font-size: 64px; color: #E2E2E2; font-family: arial, sans-serif; @@ -136,6 +132,8 @@ body { -moz-user-select: none; -ms-user-select: none; user-select: none; + padding-top: 160px; + padding-bottom: 160px; } .arrow:hover { @@ -753,9 +751,9 @@ input:-ms-placeholder { } }*/ -@media only screen -and (min-device-width : 768px) -and (max-device-width : 1024px) +@media only screen +and (min-device-width : 768px) +and (max-device-width : 1024px) and (orientation : landscape) /*and (-webkit-min-device-pixel-ratio: 2)*/ { #viewer{ diff --git a/cps/static/js/archive/rarvm.js b/cps/static/js/archive/rarvm.js index 44e09330..8858fa05 100644 --- a/cps/static/js/archive/rarvm.js +++ b/cps/static/js/archive/rarvm.js @@ -9,10 +9,18 @@ /** * CRC Implementation. */ -/* global Uint8Array, Uint32Array, bitjs, DataView */ +/* global Uint8Array, Uint32Array, bitjs, DataView, mem */ /* exported MAXWINMASK, UnpackFilter */ -var CRCTab = new Array(256).fill(0); +function emptyArr(n, v) { + var arr = []; + for (var i = 0; i < n; i += 1) { + arr[i] = v; + } + return arr; +} + +var CRCTab = emptyArr(256, 0); function initCRC() { for (var i = 0; i < 256; ++i) { diff --git a/cps/static/js/reading/epub.js b/cps/static/js/reading/epub.js index 169c207f..8b7fb075 100644 --- a/cps/static/js/reading/epub.js +++ b/cps/static/js/reading/epub.js @@ -1,12 +1,14 @@ /* global $, calibre, EPUBJS, ePubReader */ +var reader; + (function() { "use strict"; EPUBJS.filePath = calibre.filePath; EPUBJS.cssPath = calibre.cssPath; - var reader = ePubReader(calibre.bookUrl, { + reader = ePubReader(calibre.bookUrl, { restore: true, bookmarks: calibre.bookmark ? [calibre.bookmark] : [] }); diff --git a/cps/templates/readtxt.html b/cps/templates/readtxt.html index 316878f9..46a16d8f 100644 --- a/cps/templates/readtxt.html +++ b/cps/templates/readtxt.html @@ -7,7 +7,7 @@ - + @@ -16,10 +16,10 @@