NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for syncing Kobo reading state.

Browse Source
This commit is contained in:
Michael Shavit 2020-02-15 17:27:07 -05:00
parent 57d37ffba8
commit 8e1641dac9
4 changed files with 278 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
cps/kobo.py
View File

@ -17,11 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import sys
import base64
import os
import uuid
from datetime import datetime
from time import gmtime, strftime
try:
from urllib import unquote
@ -38,12 +38,13 @@ from flask import (
redirect,
abort
)
from flask_login import login_required
from flask_login import current_user, login_required
from werkzeug.datastructures import Headers
from sqlalchemy import func
from sqlalchemy.sql.expression import and_
import requests
from . import config, logger, kobo_auth, db, helper
from . import config, logger, kobo_auth, db, helper, ub
from .services import SyncToken as SyncToken
from .web import download_required
from .kobo_auth import requires_kobo_auth
@ -116,6 +117,9 @@ def redirect_or_proxy_request():
return make_response(jsonify({}))
def convert_to_kobo_timestamp_string(timestamp):
return timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
@kobo.route("/v1/library/sync")
@requires_kobo_auth
@download_required
@ -130,7 +134,8 @@ def HandleSyncRequest():
new_books_last_modified = sync_token.books_last_modified
new_books_last_created = sync_token.books_last_created
entitlements = []
new_reading_state_last_modified = sync_token.reading_state_last_modified
sync_results = []
# We reload the book database so that the user get's a fresh view of the library
# in case of external changes (e.g: adding a book through Calibre).
@ -147,41 +152,63 @@ def HandleSyncRequest():
.all()
)
reading_states_in_new_entitlements = []
for book in changed_entries:
kobo_reading_state = get_or_create_reading_state(book.id)
entitlement = {
"BookEntitlement": create_book_entitlement(book),
"BookMetadata": get_metadata(book),
"ReadingState": reading_state(book),
}
if kobo_reading_state.last_modified > sync_token.reading_state_last_modified:
entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state)
new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
reading_states_in_new_entitlements.append(book.id)
if book.timestamp > sync_token.books_last_created:
entitlements.append({"NewEntitlement": entitlement})
sync_results.append({"NewEntitlement": entitlement})
else:
entitlements.append({"ChangedEntitlement": entitlement})
sync_results.append({"ChangedEntitlement": entitlement})
new_books_last_modified = max(
book.last_modified, sync_token.books_last_modified
)
new_books_last_created = max(book.timestamp, sync_token.books_last_created)
changed_reading_states = (
ub.session.query(ub.KoboReadingState)
.filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
ub.KoboReadingState.user_id == current_user.id,
ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements))))
for kobo_reading_state in changed_reading_states.all():
book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one()
sync_results.append({
"ChangedReadingState": {
"ReadingState": get_kobo_reading_state_response(book, kobo_reading_state)
}
})
new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
sync_token.books_last_created = new_books_last_created
sync_token.books_last_modified = new_books_last_modified
sync_token.reading_state_last_modified = new_reading_state_last_modified
if config.config_kobo_proxy:
return generate_sync_response(request, sync_token, entitlements)
return generate_sync_response(request, sync_token, sync_results)
return make_response(jsonify(entitlements))
return make_response(jsonify(sync_results))
# Missing feature: Detect server-side book deletions.
def generate_sync_response(request, sync_token, entitlements):
def generate_sync_response(request, sync_token, sync_results):
extra_headers = {}
if config.config_kobo_proxy:
# Merge in sync results from the official Kobo store.
try:
store_response = make_request_to_kobo_store(sync_token)
store_entitlements = store_response.json()
entitlements += store_entitlements
store_sync_results = store_response.json()
sync_results += store_sync_results
sync_token.merge_from_store_response(store_response)
extra_headers["x-kobo-sync"] = store_response.headers.get("x-kobo-sync")
extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode")
@ -191,7 +218,7 @@ def generate_sync_response(request, sync_token, entitlements):
log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e))
sync_token.to_headers(extra_headers)
response = make_response(jsonify(entitlements), extra_headers)
response = make_response(jsonify(sync_results), extra_headers)
return response
@ -243,25 +270,21 @@ def create_book_entitlement(book):
book_uuid = book.uuid
return {
"Accessibility": "Full",
"ActivePeriod": {"From": current_time(),},
"Created": book.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
"ActivePeriod": {"From": convert_to_kobo_timestamp_string(datetime.datetime.now())},
"Created": convert_to_kobo_timestamp_string(book.timestamp),
"CrossRevisionId": book_uuid,
"Id": book_uuid,
"IsHiddenFromArchive": False,
"IsLocked": False,
# Setting this to true removes from the device.
"IsRemoved": False,
"LastModified": book.last_modified.strftime("%Y-%m-%dT%H:%M:%SZ"),
"LastModified": convert_to_kobo_timestamp_string(book.last_modified),
"OriginCategory": "Imported",
"RevisionId": book_uuid,
"Status": "Active",
}
def current_time():
return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
def get_description(book):
if not book.comments:
return None
@ -324,6 +347,8 @@ def get_metadata(book):
"IsSocialEnabled": True,
"Language": "en",
"PhoneticPronunciations": {},
# TODO: Fix book.pubdate to return a datetime object so that we can easily
# convert it to the format Kobo devices expect.
"PublicationDate": book.pubdate,
"Publisher": {"Imprint": "", "Name": get_publisher(book),},
"RevisionId": book_uuid,
@ -347,16 +372,148 @@ def get_metadata(book):
return metadata
def reading_state(book):
# TODO: Implement
reading_state = {
# "StatusInfo": {
# "LastModified": get_single_cc_value(book, "lastreadtimestamp"),
# "Status": get_single_cc_value(book, "reading_status"),
# }
# TODO: CurrentBookmark, Location
@kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
@login_required
def HandleStateRequest(book_uuid):
book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
if not book or not book.data:
log.info(u"Book %s not found in database", book_uuid)
return redirect_or_proxy_request()
kobo_reading_state = get_or_create_reading_state(book.id)
if request.method == "GET":
return jsonify([get_kobo_reading_state_response(book, kobo_reading_state)])
else:
update_results_response = {"EntitlementId": book_uuid}
request_data = request.json
if "ReadingStates" not in request_data:
abort(400, description="Malformed request data is missing 'ReadingStates' key")
request_reading_state = request_data["ReadingStates"][0]
request_bookmark = request_reading_state.get("CurrentBookmark")
if request_bookmark:
current_bookmark = kobo_reading_state.current_bookmark
current_bookmark.progress_percent = request_bookmark.get("ProgressPercent")
current_bookmark.content_source_progress_percent = request_bookmark.get("ContentSourceProgressPercent")
location = request_bookmark.get("Location")
if location:
current_bookmark.location_value = location.get("Value")
current_bookmark.location_type = location.get("Type")
current_bookmark.location_source = location.get("Source")
update_results_response["CurrentBookmarkResult"] = {"Result": "Success"}
request_statistics = request_reading_state.get("Statistics")
if request_statistics:
statistics = kobo_reading_state.statistics
statistics.spent_reading_minutes = request_statistics.get("SpentReadingMinutes")
statistics.remaining_time_minutes = request_statistics.get("RemainingTimeMinutes")
update_results_response["StatisticsResult"] = {"Result": "Success"}
request_status_info = request_reading_state.get("StatusInfo")
if request_status_info:
book_read = kobo_reading_state.book_read_link
new_book_read_status = get_ub_read_status(request_status_info.get("Status"))
if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS and new_book_read_status != book_read.read_status:
book_read.times_started_reading += 1
book_read.last_time_started_reading = datetime.datetime.utcnow()
book_read.read_status = new_book_read_status
update_results_response["StatusInfoResult"] = {"Result": "Success"}
ub.session.merge(kobo_reading_state)
ub.session.commit()
return jsonify({
"RequestResult": "Success",
"UpdateResults": [update_results_response],
})
def get_read_status_for_kobo(ub_book_read):
enum_to_string_map = {
None: "ReadyToRead",
ub.ReadBook.STATUS_UNREAD: "ReadyToRead",
ub.ReadBook.STATUS_FINISHED: "Finished",
ub.ReadBook.STATUS_IN_PROGRESS: "Reading",
}
return reading_state
return enum_to_string_map[ub_book_read.read_status]
def get_ub_read_status(kobo_read_status):
string_to_enum_map = {
None: None,
"ReadyToRead": ub.ReadBook.STATUS_UNREAD,
"Finished": ub.ReadBook.STATUS_FINISHED,
"Reading": ub.ReadBook.STATUS_IN_PROGRESS,
}
return string_to_enum_map[kobo_read_status]
def get_or_create_reading_state(book_id):
book_read = ub.session.query(ub.ReadBook).filter(and_(ub.ReadBook.book_id == book_id,
ub.ReadBook.user_id == current_user.id)).one_or_none()
if not book_read:
book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id)
if not book_read.kobo_reading_state:
kobo_reading_state = ub.KoboReadingState(user_id=book_read.user_id, book_id=book_id)
kobo_reading_state.current_bookmark = ub.KoboBookmark()
kobo_reading_state.statistics = ub.KoboStatistics()
book_read.kobo_reading_state = kobo_reading_state
ub.session.add(book_read)
ub.session.commit()
return book_read.kobo_reading_state
def get_kobo_reading_state_response(book, kobo_reading_state):
return {
"EntitlementId": book.uuid,
"Created": convert_to_kobo_timestamp_string(book.timestamp),
"LastModified": convert_to_kobo_timestamp_string(kobo_reading_state.last_modified),
# AFAICT PriorityTimestamp is always equal to LastModified.
"PriorityTimestamp": convert_to_kobo_timestamp_string(kobo_reading_state.priority_timestamp),
"StatusInfo": get_status_info_response(kobo_reading_state.book_read_link),
"Statistics": get_statistics_response(kobo_reading_state.statistics),
"CurrentBookmark": get_current_bookmark_response(kobo_reading_state.current_bookmark),
}
def get_status_info_response(book_read):
resp = {
"LastModified": convert_to_kobo_timestamp_string(book_read.last_modified),
"Status": get_read_status_for_kobo(book_read),
"TimesStartedReading": book_read.times_started_reading,
}
if book_read.last_time_started_reading:
resp["LastTimeStartedReading"] = convert_to_kobo_timestamp_string(book_read.last_time_started_reading)
return resp
def get_statistics_response(statistics):
resp = {
"LastModified": convert_to_kobo_timestamp_string(statistics.last_modified),
}
if statistics.spent_reading_minutes:
resp["SpentReadingMinutes"] = statistics.spent_reading_minutes
if statistics.remaining_time_minutes:
resp["RemainingTimeMinutes"] = statistics.remaining_time_minutes
return resp
def get_current_bookmark_response(current_bookmark):
resp = {
"LastModified": convert_to_kobo_timestamp_string(current_bookmark.last_modified),
}
if current_bookmark.progress_percent:
resp["ProgressPercent"] = current_bookmark.progress_percent
if current_bookmark.content_source_progress_percent:
resp["ContentSourceProgressPercent"] = current_bookmark.content_source_progress_percent
if current_bookmark.location_value:
resp["Location"] = {
"Value": current_bookmark.location_value,
"Type": current_bookmark.location_type,
"Source": current_bookmark.location_source,
}
return resp
@kobo.route("/<book_uuid>/image.jpg")
@ -381,7 +538,6 @@ def TopLevelEndpoint():
# TODO: Implement the following routes
@kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])
@kobo.route("/v1/library/<book_uuid>/state", methods=["PUT"])
@kobo.route("/v1/library/tags", methods=["POST"])
@kobo.route("/v1/library/tags/<shelf_name>", methods=["POST"])
@kobo.route("/v1/library/tags/<tag_id>", methods=["DELETE"])

25
cps/services/SyncToken.py
View File

@ -42,6 +42,13 @@ def to_epoch_timestamp(datetime_object):
return (datetime_object - datetime(1970, 1, 1)).total_seconds()
def get_datetime_from_json(json_object, field_name):
try:
return datetime.utcfromtimestamp(json_object[field_name])
except KeyError:
return datetime.min
class SyncToken():
""" The SyncToken is used to persist state accross requests.
When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service.
@ -53,7 +60,8 @@ class SyncToken():
"""
SYNC_TOKEN_HEADER = "x-kobo-synctoken"
VERSION = "1-0-0"
VERSION = "1-1-0"
LAST_MODIFIED_ADDED_VERSION = "1-1-0"
MIN_VERSION = "1-0-0"
token_schema = {
@ -68,6 +76,7 @@ class SyncToken():
"raw_kobo_store_token": {"type": "string"},
"books_last_modified": {"type": "string"},
"books_last_created": {"type": "string"},
"reading_state_last_modified": {"type": "string"},
},
}
@ -76,10 +85,13 @@ class SyncToken():
raw_kobo_store_token="",
books_last_created=datetime.min,
books_last_modified=datetime.min,
reading_state_last_modified=datetime.min,
):
self.raw_kobo_store_token = raw_kobo_store_token
self.books_last_created = books_last_created
self.books_last_modified = books_last_modified
self.reading_state_last_modified = reading_state_last_modified
@staticmethod
def from_headers(headers):
@ -109,12 +121,9 @@ class SyncToken():
raw_kobo_store_token = data_json["raw_kobo_store_token"]
try:
books_last_modified = datetime.utcfromtimestamp(
data_json["books_last_modified"]
)
books_last_created = datetime.utcfromtimestamp(
data_json["books_last_created"]
)
books_last_modified = get_datetime_from_json(data_json, "books_last_modified")
books_last_created = get_datetime_from_json(data_json, "books_last_created")
reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified")
except TypeError:
log.error("SyncToken timestamps don't parse to a datetime.")
return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
@ -123,6 +132,7 @@ class SyncToken():
raw_kobo_store_token=raw_kobo_store_token,
books_last_created=books_last_created,
books_last_modified=books_last_modified,
reading_state_last_modified=reading_state_last_modified
)
def set_kobo_store_header(self, store_headers):
@ -143,6 +153,7 @@ class SyncToken():
"raw_kobo_store_token": self.raw_kobo_store_token,
"books_last_modified": to_epoch_timestamp(self.books_last_modified),
"books_last_created": to_epoch_timestamp(self.books_last_created),
"reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified)
},
}
return b64encode_json(token)

74
cps/ub.py
View File

@ -20,6 +20,7 @@
from __future__ import division, print_function, unicode_literals
import os
import datetime
import itertools
from binascii import hexlify
from flask import g
@ -31,10 +32,10 @@ try:
oauth_support = True
except ImportError:
oauth_support = False
from sqlalchemy import create_engine, exc, exists
from sqlalchemy import create_engine, exc, exists, event
from sqlalchemy import Column, ForeignKey
from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime
from sqlalchemy.orm import foreign, relationship, remote, sessionmaker
from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float
from sqlalchemy.orm import backref, foreign, relationship, remote, sessionmaker, Session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql.expression import and_
from werkzeug.security import generate_password_hash
@ -292,8 +293,12 @@ class ReadBook(Base):
id = Column(Integer, primary_key=True)
book_id = Column(Integer, unique=False)
user_id = Column(Integer, ForeignKey('user.id'), unique=False)
is_read = Column(Boolean, unique=False)
read_status = Column(Integer, unique=False, default=STATUS_UNREAD)
read_status = Column(Integer, unique=False, default=STATUS_UNREAD, nullable=False)
kobo_reading_state = relationship("KoboReadingState", uselist=False, primaryjoin="and_(ReadBook.user_id == foreign(KoboReadingState.user_id), "
"ReadBook.book_id == foreign(KoboReadingState.book_id))", cascade="all", backref=backref("book_read_link", uselist=False))
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
last_time_started_reading = Column(DateTime, nullable=True)
times_started_reading = Column(Integer, default=0, nullable=False)
class Bookmark(Base):
@ -306,6 +311,54 @@ class Bookmark(Base):
bookmark_key = Column(String)
# The Kobo ReadingState API keeps track of 4 timestamped entities:
# ReadingState, StatusInfo, Statistics, CurrentBookmark
# Which we map to the following 4 tables:
# KoboReadingState, ReadBook, KoboStatistics and KoboBookmark
class KoboReadingState(Base):
__tablename__ = 'kobo_reading_state'
id = Column(Integer, primary_key=True, autoincrement=True)
user_id = Column(Integer, ForeignKey('user.id'))
book_id = Column(Integer)
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
priority_timestamp = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
current_bookmark = relationship("KoboBookmark", uselist=False, backref="kobo_reading_state", cascade="all")
statistics = relationship("KoboStatistics", uselist=False, backref="kobo_reading_state", cascade="all")
class KoboBookmark(Base):
__tablename__ = 'kobo_bookmark'
id = Column(Integer, primary_key=True)
kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id'))
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
location_source = Column(String)
location_type = Column(String)
location_value = Column(String)
progress_percent = Column(Float)
content_source_progress_percent = Column(Float)
class KoboStatistics(Base):
__tablename__ = 'kobo_statistics'
id = Column(Integer, primary_key=True)
kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id'))
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
remaining_time_minutes = Column(Integer)
spent_reading_minutes = Column(Integer)
# Updates the last_modified timestamp in the KoboReadingState table if any of its children tables are modified.
@event.listens_for(Session, 'before_flush')
def receive_before_flush(session, flush_context, instances):
for change in itertools.chain(session.new, session.dirty):
if isinstance(change, (ReadBook, KoboStatistics, KoboBookmark)):
if change.kobo_reading_state:
change.kobo_reading_state.last_modified = datetime.datetime.utcnow()
# Baseclass representing Downloads from calibre-web in app.db
class Downloads(Base):
__tablename__ = 'downloads'
@ -358,6 +411,12 @@ def migrate_Database(session):
ReadBook.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "bookmark"):
Bookmark.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "kobo_reading_state"):
KoboReadingState.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "kobo_bookmark"):
KoboBookmark.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "kobo_statistics"):
KoboStatistics.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "registration"):
ReadBook.__table__.create(bind=engine)
conn = engine.connect()
@ -381,10 +440,13 @@ def migrate_Database(session):
session.commit()
try:
session.query(exists().where(ReadBook.read_status)).scalar()
except exc.OperationalError: # Database is not compatible, some columns are missing
except exc.OperationalError:
conn = engine.connect()
conn.execute("ALTER TABLE book_read_link ADD column 'read_status' INTEGER DEFAULT 0")
conn.execute("UPDATE book_read_link SET 'read_status' = 1 WHERE is_read")
conn.execute("ALTER TABLE book_read_link ADD column 'last_modified' DATETIME")
conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME")
conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0")
session.commit()
# Handle table exists, but no content

9
cps/web.py
View File

@ -319,11 +319,14 @@ def toggle_read(book_id):
else:
book.read_status = ub.ReadBook.STATUS_FINISHED
else:
readBook = ub.ReadBook()
readBook.user_id = int(current_user.id)
readBook.book_id = book_id
readBook = ub.ReadBook(user_id=current_user.id, book_id = book_id)
readBook.read_status = ub.ReadBook.STATUS_FINISHED
book = readBook
if not book.kobo_reading_state:
kobo_reading_state = ub.KoboReadingState(user_id=current_user.id, book_id=book_id)
kobo_reading_state.current_bookmark = ub.KoboBookmark()
kobo_reading_state.statistics = ub.KoboStatistics()
book.kobo_reading_state = kobo_reading_state
ub.session.merge(book)
ub.session.commit()
else: