From 8e1641dac9c9211ef324d5aeb8cfd399cc496bc0 Mon Sep 17 00:00:00 2001 From: Michael Shavit Date: Sat, 15 Feb 2020 17:27:07 -0500 Subject: [PATCH] Add support for syncing Kobo reading state. --- cps/kobo.py | 216 ++++++++++++++++++++++++++++++++------ cps/services/SyncToken.py | 25 +++-- cps/ub.py | 74 +++++++++++-- cps/web.py | 9 +- 4 files changed, 278 insertions(+), 46 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 15a17022..192d10fe 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import datetime import sys import base64 import os import uuid -from datetime import datetime from time import gmtime, strftime try: from urllib import unquote @@ -38,12 +38,13 @@ from flask import ( redirect, abort ) -from flask_login import login_required +from flask_login import current_user, login_required from werkzeug.datastructures import Headers from sqlalchemy import func +from sqlalchemy.sql.expression import and_ import requests -from . import config, logger, kobo_auth, db, helper +from . import config, logger, kobo_auth, db, helper, ub from .services import SyncToken as SyncToken from .web import download_required from .kobo_auth import requires_kobo_auth @@ -116,6 +117,9 @@ def redirect_or_proxy_request(): return make_response(jsonify({})) +def convert_to_kobo_timestamp_string(timestamp): + return timestamp.strftime("%Y-%m-%dT%H:%M:%SZ") + @kobo.route("/v1/library/sync") @requires_kobo_auth @download_required @@ -130,7 +134,8 @@ def HandleSyncRequest(): new_books_last_modified = sync_token.books_last_modified new_books_last_created = sync_token.books_last_created - entitlements = [] + new_reading_state_last_modified = sync_token.reading_state_last_modified + sync_results = [] # We reload the book database so that the user get's a fresh view of the library # in case of external changes (e.g: adding a book through Calibre). @@ -147,41 +152,63 @@ def HandleSyncRequest(): .all() ) + reading_states_in_new_entitlements = [] for book in changed_entries: + kobo_reading_state = get_or_create_reading_state(book.id) entitlement = { "BookEntitlement": create_book_entitlement(book), "BookMetadata": get_metadata(book), - "ReadingState": reading_state(book), } + + if kobo_reading_state.last_modified > sync_token.reading_state_last_modified: + entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state) + new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified) + reading_states_in_new_entitlements.append(book.id) + if book.timestamp > sync_token.books_last_created: - entitlements.append({"NewEntitlement": entitlement}) + sync_results.append({"NewEntitlement": entitlement}) else: - entitlements.append({"ChangedEntitlement": entitlement}) + sync_results.append({"ChangedEntitlement": entitlement}) new_books_last_modified = max( book.last_modified, sync_token.books_last_modified ) new_books_last_created = max(book.timestamp, sync_token.books_last_created) + changed_reading_states = ( + ub.session.query(ub.KoboReadingState) + .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified, + ub.KoboReadingState.user_id == current_user.id, + ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements)))) + for kobo_reading_state in changed_reading_states.all(): + book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one() + sync_results.append({ + "ChangedReadingState": { + "ReadingState": get_kobo_reading_state_response(book, kobo_reading_state) + } + }) + new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified) + sync_token.books_last_created = new_books_last_created sync_token.books_last_modified = new_books_last_modified + sync_token.reading_state_last_modified = new_reading_state_last_modified if config.config_kobo_proxy: - return generate_sync_response(request, sync_token, entitlements) + return generate_sync_response(request, sync_token, sync_results) - return make_response(jsonify(entitlements)) + return make_response(jsonify(sync_results)) # Missing feature: Detect server-side book deletions. -def generate_sync_response(request, sync_token, entitlements): +def generate_sync_response(request, sync_token, sync_results): extra_headers = {} if config.config_kobo_proxy: # Merge in sync results from the official Kobo store. try: store_response = make_request_to_kobo_store(sync_token) - store_entitlements = store_response.json() - entitlements += store_entitlements + store_sync_results = store_response.json() + sync_results += store_sync_results sync_token.merge_from_store_response(store_response) extra_headers["x-kobo-sync"] = store_response.headers.get("x-kobo-sync") extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode") @@ -191,7 +218,7 @@ def generate_sync_response(request, sync_token, entitlements): log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e)) sync_token.to_headers(extra_headers) - response = make_response(jsonify(entitlements), extra_headers) + response = make_response(jsonify(sync_results), extra_headers) return response @@ -243,25 +270,21 @@ def create_book_entitlement(book): book_uuid = book.uuid return { "Accessibility": "Full", - "ActivePeriod": {"From": current_time(),}, - "Created": book.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"), + "ActivePeriod": {"From": convert_to_kobo_timestamp_string(datetime.datetime.now())}, + "Created": convert_to_kobo_timestamp_string(book.timestamp), "CrossRevisionId": book_uuid, "Id": book_uuid, "IsHiddenFromArchive": False, "IsLocked": False, # Setting this to true removes from the device. "IsRemoved": False, - "LastModified": book.last_modified.strftime("%Y-%m-%dT%H:%M:%SZ"), + "LastModified": convert_to_kobo_timestamp_string(book.last_modified), "OriginCategory": "Imported", "RevisionId": book_uuid, "Status": "Active", } -def current_time(): - return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) - - def get_description(book): if not book.comments: return None @@ -324,6 +347,8 @@ def get_metadata(book): "IsSocialEnabled": True, "Language": "en", "PhoneticPronunciations": {}, + # TODO: Fix book.pubdate to return a datetime object so that we can easily + # convert it to the format Kobo devices expect. "PublicationDate": book.pubdate, "Publisher": {"Imprint": "", "Name": get_publisher(book),}, "RevisionId": book_uuid, @@ -347,16 +372,148 @@ def get_metadata(book): return metadata -def reading_state(book): - # TODO: Implement - reading_state = { - # "StatusInfo": { - # "LastModified": get_single_cc_value(book, "lastreadtimestamp"), - # "Status": get_single_cc_value(book, "reading_status"), - # } - # TODO: CurrentBookmark, Location +@kobo.route("/v1/library//state", methods=["GET", "PUT"]) +@login_required +def HandleStateRequest(book_uuid): + book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() + if not book or not book.data: + log.info(u"Book %s not found in database", book_uuid) + return redirect_or_proxy_request() + + kobo_reading_state = get_or_create_reading_state(book.id) + + if request.method == "GET": + return jsonify([get_kobo_reading_state_response(book, kobo_reading_state)]) + else: + update_results_response = {"EntitlementId": book_uuid} + + request_data = request.json + if "ReadingStates" not in request_data: + abort(400, description="Malformed request data is missing 'ReadingStates' key") + request_reading_state = request_data["ReadingStates"][0] + + request_bookmark = request_reading_state.get("CurrentBookmark") + if request_bookmark: + current_bookmark = kobo_reading_state.current_bookmark + current_bookmark.progress_percent = request_bookmark.get("ProgressPercent") + current_bookmark.content_source_progress_percent = request_bookmark.get("ContentSourceProgressPercent") + location = request_bookmark.get("Location") + if location: + current_bookmark.location_value = location.get("Value") + current_bookmark.location_type = location.get("Type") + current_bookmark.location_source = location.get("Source") + update_results_response["CurrentBookmarkResult"] = {"Result": "Success"} + + request_statistics = request_reading_state.get("Statistics") + if request_statistics: + statistics = kobo_reading_state.statistics + statistics.spent_reading_minutes = request_statistics.get("SpentReadingMinutes") + statistics.remaining_time_minutes = request_statistics.get("RemainingTimeMinutes") + update_results_response["StatisticsResult"] = {"Result": "Success"} + + request_status_info = request_reading_state.get("StatusInfo") + if request_status_info: + book_read = kobo_reading_state.book_read_link + new_book_read_status = get_ub_read_status(request_status_info.get("Status")) + if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS and new_book_read_status != book_read.read_status: + book_read.times_started_reading += 1 + book_read.last_time_started_reading = datetime.datetime.utcnow() + book_read.read_status = new_book_read_status + update_results_response["StatusInfoResult"] = {"Result": "Success"} + + ub.session.merge(kobo_reading_state) + ub.session.commit() + return jsonify({ + "RequestResult": "Success", + "UpdateResults": [update_results_response], + }) + + +def get_read_status_for_kobo(ub_book_read): + enum_to_string_map = { + None: "ReadyToRead", + ub.ReadBook.STATUS_UNREAD: "ReadyToRead", + ub.ReadBook.STATUS_FINISHED: "Finished", + ub.ReadBook.STATUS_IN_PROGRESS: "Reading", } - return reading_state + return enum_to_string_map[ub_book_read.read_status] + + +def get_ub_read_status(kobo_read_status): + string_to_enum_map = { + None: None, + "ReadyToRead": ub.ReadBook.STATUS_UNREAD, + "Finished": ub.ReadBook.STATUS_FINISHED, + "Reading": ub.ReadBook.STATUS_IN_PROGRESS, + } + return string_to_enum_map[kobo_read_status] + + +def get_or_create_reading_state(book_id): + book_read = ub.session.query(ub.ReadBook).filter(and_(ub.ReadBook.book_id == book_id, + ub.ReadBook.user_id == current_user.id)).one_or_none() + if not book_read: + book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id) + if not book_read.kobo_reading_state: + kobo_reading_state = ub.KoboReadingState(user_id=book_read.user_id, book_id=book_id) + kobo_reading_state.current_bookmark = ub.KoboBookmark() + kobo_reading_state.statistics = ub.KoboStatistics() + book_read.kobo_reading_state = kobo_reading_state + ub.session.add(book_read) + ub.session.commit() + return book_read.kobo_reading_state + + +def get_kobo_reading_state_response(book, kobo_reading_state): + return { + "EntitlementId": book.uuid, + "Created": convert_to_kobo_timestamp_string(book.timestamp), + "LastModified": convert_to_kobo_timestamp_string(kobo_reading_state.last_modified), + # AFAICT PriorityTimestamp is always equal to LastModified. + "PriorityTimestamp": convert_to_kobo_timestamp_string(kobo_reading_state.priority_timestamp), + "StatusInfo": get_status_info_response(kobo_reading_state.book_read_link), + "Statistics": get_statistics_response(kobo_reading_state.statistics), + "CurrentBookmark": get_current_bookmark_response(kobo_reading_state.current_bookmark), + } + + +def get_status_info_response(book_read): + resp = { + "LastModified": convert_to_kobo_timestamp_string(book_read.last_modified), + "Status": get_read_status_for_kobo(book_read), + "TimesStartedReading": book_read.times_started_reading, + } + if book_read.last_time_started_reading: + resp["LastTimeStartedReading"] = convert_to_kobo_timestamp_string(book_read.last_time_started_reading) + return resp + + +def get_statistics_response(statistics): + resp = { + "LastModified": convert_to_kobo_timestamp_string(statistics.last_modified), + } + if statistics.spent_reading_minutes: + resp["SpentReadingMinutes"] = statistics.spent_reading_minutes + if statistics.remaining_time_minutes: + resp["RemainingTimeMinutes"] = statistics.remaining_time_minutes + return resp + + +def get_current_bookmark_response(current_bookmark): + resp = { + "LastModified": convert_to_kobo_timestamp_string(current_bookmark.last_modified), + } + if current_bookmark.progress_percent: + resp["ProgressPercent"] = current_bookmark.progress_percent + if current_bookmark.content_source_progress_percent: + resp["ContentSourceProgressPercent"] = current_bookmark.content_source_progress_percent + if current_bookmark.location_value: + resp["Location"] = { + "Value": current_bookmark.location_value, + "Type": current_bookmark.location_type, + "Source": current_bookmark.location_source, + } + return resp @kobo.route("//image.jpg") @@ -381,7 +538,6 @@ def TopLevelEndpoint(): # TODO: Implement the following routes @kobo.route("/v1/library/", methods=["DELETE", "GET"]) -@kobo.route("/v1/library//state", methods=["PUT"]) @kobo.route("/v1/library/tags", methods=["POST"]) @kobo.route("/v1/library/tags/", methods=["POST"]) @kobo.route("/v1/library/tags/", methods=["DELETE"]) diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index 63d82ac0..133942b3 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -42,6 +42,13 @@ def to_epoch_timestamp(datetime_object): return (datetime_object - datetime(1970, 1, 1)).total_seconds() +def get_datetime_from_json(json_object, field_name): + try: + return datetime.utcfromtimestamp(json_object[field_name]) + except KeyError: + return datetime.min + + class SyncToken(): """ The SyncToken is used to persist state accross requests. When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service. @@ -53,7 +60,8 @@ class SyncToken(): """ SYNC_TOKEN_HEADER = "x-kobo-synctoken" - VERSION = "1-0-0" + VERSION = "1-1-0" + LAST_MODIFIED_ADDED_VERSION = "1-1-0" MIN_VERSION = "1-0-0" token_schema = { @@ -68,6 +76,7 @@ class SyncToken(): "raw_kobo_store_token": {"type": "string"}, "books_last_modified": {"type": "string"}, "books_last_created": {"type": "string"}, + "reading_state_last_modified": {"type": "string"}, }, } @@ -76,10 +85,13 @@ class SyncToken(): raw_kobo_store_token="", books_last_created=datetime.min, books_last_modified=datetime.min, + reading_state_last_modified=datetime.min, ): self.raw_kobo_store_token = raw_kobo_store_token self.books_last_created = books_last_created self.books_last_modified = books_last_modified + self.reading_state_last_modified = reading_state_last_modified + @staticmethod def from_headers(headers): @@ -109,12 +121,9 @@ class SyncToken(): raw_kobo_store_token = data_json["raw_kobo_store_token"] try: - books_last_modified = datetime.utcfromtimestamp( - data_json["books_last_modified"] - ) - books_last_created = datetime.utcfromtimestamp( - data_json["books_last_created"] - ) + books_last_modified = get_datetime_from_json(data_json, "books_last_modified") + books_last_created = get_datetime_from_json(data_json, "books_last_created") + reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified") except TypeError: log.error("SyncToken timestamps don't parse to a datetime.") return SyncToken(raw_kobo_store_token=raw_kobo_store_token) @@ -123,6 +132,7 @@ class SyncToken(): raw_kobo_store_token=raw_kobo_store_token, books_last_created=books_last_created, books_last_modified=books_last_modified, + reading_state_last_modified=reading_state_last_modified ) def set_kobo_store_header(self, store_headers): @@ -143,6 +153,7 @@ class SyncToken(): "raw_kobo_store_token": self.raw_kobo_store_token, "books_last_modified": to_epoch_timestamp(self.books_last_modified), "books_last_created": to_epoch_timestamp(self.books_last_created), + "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified) }, } return b64encode_json(token) diff --git a/cps/ub.py b/cps/ub.py index 1e39af7e..f68ae8ab 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -20,6 +20,7 @@ from __future__ import division, print_function, unicode_literals import os import datetime +import itertools from binascii import hexlify from flask import g @@ -31,10 +32,10 @@ try: oauth_support = True except ImportError: oauth_support = False -from sqlalchemy import create_engine, exc, exists +from sqlalchemy import create_engine, exc, exists, event from sqlalchemy import Column, ForeignKey -from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime -from sqlalchemy.orm import foreign, relationship, remote, sessionmaker +from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float +from sqlalchemy.orm import backref, foreign, relationship, remote, sessionmaker, Session from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql.expression import and_ from werkzeug.security import generate_password_hash @@ -292,8 +293,12 @@ class ReadBook(Base): id = Column(Integer, primary_key=True) book_id = Column(Integer, unique=False) user_id = Column(Integer, ForeignKey('user.id'), unique=False) - is_read = Column(Boolean, unique=False) - read_status = Column(Integer, unique=False, default=STATUS_UNREAD) + read_status = Column(Integer, unique=False, default=STATUS_UNREAD, nullable=False) + kobo_reading_state = relationship("KoboReadingState", uselist=False, primaryjoin="and_(ReadBook.user_id == foreign(KoboReadingState.user_id), " + "ReadBook.book_id == foreign(KoboReadingState.book_id))", cascade="all", backref=backref("book_read_link", uselist=False)) + last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) + last_time_started_reading = Column(DateTime, nullable=True) + times_started_reading = Column(Integer, default=0, nullable=False) class Bookmark(Base): @@ -306,6 +311,54 @@ class Bookmark(Base): bookmark_key = Column(String) +# The Kobo ReadingState API keeps track of 4 timestamped entities: +# ReadingState, StatusInfo, Statistics, CurrentBookmark +# Which we map to the following 4 tables: +# KoboReadingState, ReadBook, KoboStatistics and KoboBookmark +class KoboReadingState(Base): + __tablename__ = 'kobo_reading_state' + + id = Column(Integer, primary_key=True, autoincrement=True) + user_id = Column(Integer, ForeignKey('user.id')) + book_id = Column(Integer) + last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) + priority_timestamp = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) + current_bookmark = relationship("KoboBookmark", uselist=False, backref="kobo_reading_state", cascade="all") + statistics = relationship("KoboStatistics", uselist=False, backref="kobo_reading_state", cascade="all") + + +class KoboBookmark(Base): + __tablename__ = 'kobo_bookmark' + + id = Column(Integer, primary_key=True) + kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id')) + last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) + location_source = Column(String) + location_type = Column(String) + location_value = Column(String) + progress_percent = Column(Float) + content_source_progress_percent = Column(Float) + + +class KoboStatistics(Base): + __tablename__ = 'kobo_statistics' + + id = Column(Integer, primary_key=True) + kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id')) + last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) + remaining_time_minutes = Column(Integer) + spent_reading_minutes = Column(Integer) + + +# Updates the last_modified timestamp in the KoboReadingState table if any of its children tables are modified. +@event.listens_for(Session, 'before_flush') +def receive_before_flush(session, flush_context, instances): + for change in itertools.chain(session.new, session.dirty): + if isinstance(change, (ReadBook, KoboStatistics, KoboBookmark)): + if change.kobo_reading_state: + change.kobo_reading_state.last_modified = datetime.datetime.utcnow() + + # Baseclass representing Downloads from calibre-web in app.db class Downloads(Base): __tablename__ = 'downloads' @@ -358,6 +411,12 @@ def migrate_Database(session): ReadBook.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "bookmark"): Bookmark.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "kobo_reading_state"): + KoboReadingState.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "kobo_bookmark"): + KoboBookmark.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "kobo_statistics"): + KoboStatistics.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): ReadBook.__table__.create(bind=engine) conn = engine.connect() @@ -381,10 +440,13 @@ def migrate_Database(session): session.commit() try: session.query(exists().where(ReadBook.read_status)).scalar() - except exc.OperationalError: # Database is not compatible, some columns are missing + except exc.OperationalError: conn = engine.connect() conn.execute("ALTER TABLE book_read_link ADD column 'read_status' INTEGER DEFAULT 0") conn.execute("UPDATE book_read_link SET 'read_status' = 1 WHERE is_read") + conn.execute("ALTER TABLE book_read_link ADD column 'last_modified' DATETIME") + conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME") + conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0") session.commit() # Handle table exists, but no content diff --git a/cps/web.py b/cps/web.py index a2aa047f..4230d9dd 100644 --- a/cps/web.py +++ b/cps/web.py @@ -319,11 +319,14 @@ def toggle_read(book_id): else: book.read_status = ub.ReadBook.STATUS_FINISHED else: - readBook = ub.ReadBook() - readBook.user_id = int(current_user.id) - readBook.book_id = book_id + readBook = ub.ReadBook(user_id=current_user.id, book_id = book_id) readBook.read_status = ub.ReadBook.STATUS_FINISHED book = readBook + if not book.kobo_reading_state: + kobo_reading_state = ub.KoboReadingState(user_id=current_user.id, book_id=book_id) + kobo_reading_state.current_bookmark = ub.KoboBookmark() + kobo_reading_state.statistics = ub.KoboStatistics() + book.kobo_reading_state = kobo_reading_state ub.session.merge(book) ub.session.commit() else: