NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'reading_state' into Develop

Browse Source 
# Conflicts:
#	cps/kobo.py
This commit is contained in:
Ozzieisaacs 2020-03-12 20:35:24 +01:00
commit d597e05fa9
4 changed files with 300 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
cps/kobo.py
View File

@ -17,11 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import sys
import base64
import os
import uuid
from datetime import datetime
from time import gmtime, strftime
try:
from urllib import unquote
@ -38,12 +38,13 @@ from flask import (
redirect,
abort
)
from flask_login import login_required
from flask_login import current_user, login_required
from werkzeug.datastructures import Headers
from sqlalchemy import func
from sqlalchemy.sql.expression import and_
import requests
from . import config, logger, kobo_auth, db, helper
from . import config, logger, kobo_auth, db, helper, ub
from .services import SyncToken as SyncToken
from .web import download_required
from .kobo_auth import requires_kobo_auth
@ -116,6 +117,9 @@ def redirect_or_proxy_request():
return make_response(jsonify({}))
def convert_to_kobo_timestamp_string(timestamp):
return timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
@kobo.route("/v1/library/sync")
@requires_kobo_auth
@download_required
@ -130,7 +134,8 @@ def HandleSyncRequest():
new_books_last_modified = sync_token.books_last_modified
new_books_last_created = sync_token.books_last_created
entitlements = []
new_reading_state_last_modified = sync_token.reading_state_last_modified
sync_results = []
# We reload the book database so that the user get's a fresh view of the library
# in case of external changes (e.g: adding a book through Calibre).
@ -147,37 +152,63 @@ def HandleSyncRequest():
.all()
)
reading_states_in_new_entitlements = []
for book in changed_entries:
kobo_reading_state = get_or_create_reading_state(book.id)
entitlement = {
"BookEntitlement": create_book_entitlement(book),
"BookMetadata": get_metadata(book),
"ReadingState": reading_state(book),
}
if kobo_reading_state.last_modified > sync_token.reading_state_last_modified:
entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state)
new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
reading_states_in_new_entitlements.append(book.id)
if book.timestamp > sync_token.books_last_created:
entitlements.append({"NewEntitlement": entitlement})
sync_results.append({"NewEntitlement": entitlement})
else:
entitlements.append({"ChangedEntitlement": entitlement})
sync_results.append({"ChangedEntitlement": entitlement})
new_books_last_modified = max(
book.last_modified, new_books_last_modified
)
new_books_last_created = max(book.timestamp, new_books_last_created)
changed_reading_states = (
ub.session.query(ub.KoboReadingState)
.filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
ub.KoboReadingState.user_id == current_user.id,
ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements))))
for kobo_reading_state in changed_reading_states.all():
book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one()
sync_results.append({
"ChangedReadingState": {
"ReadingState": get_kobo_reading_state_response(book, kobo_reading_state)
}
})
new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
sync_token.books_last_created = new_books_last_created
sync_token.books_last_modified = new_books_last_modified
sync_token.reading_state_last_modified = new_reading_state_last_modified
return generate_sync_response(request, sync_token, entitlements)
if config.config_kobo_proxy:
return generate_sync_response(request, sync_token, sync_results)
return make_response(jsonify(sync_results))
# Missing feature: Detect server-side book deletions.
def generate_sync_response(request, sync_token, entitlements):
def generate_sync_response(request, sync_token, sync_results):
extra_headers = {}
if config.config_kobo_proxy:
# Merge in sync results from the official Kobo store.
try:
store_response = make_request_to_kobo_store(sync_token)
store_entitlements = store_response.json()
entitlements += store_entitlements
store_sync_results = store_response.json()
sync_results += store_sync_results
sync_token.merge_from_store_response(store_response)
extra_headers["x-kobo-sync"] = store_response.headers.get("x-kobo-sync")
extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode")
@ -187,7 +218,7 @@ def generate_sync_response(request, sync_token, entitlements):
log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e))
sync_token.to_headers(extra_headers)
response = make_response(jsonify(entitlements), extra_headers)
response = make_response(jsonify(sync_results), extra_headers)
return response
@ -229,25 +260,21 @@ def create_book_entitlement(book):
book_uuid = book.uuid
return {
"Accessibility": "Full",
"ActivePeriod": {"From": current_time(),},
"Created": book.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
"ActivePeriod": {"From": convert_to_kobo_timestamp_string(datetime.datetime.now())},
"Created": convert_to_kobo_timestamp_string(book.timestamp),
"CrossRevisionId": book_uuid,
"Id": book_uuid,
"IsHiddenFromArchive": False,
"IsLocked": False,
# Setting this to true removes from the device.
"IsRemoved": False,
"LastModified": book.last_modified.strftime("%Y-%m-%dT%H:%M:%SZ"),
"LastModified": convert_to_kobo_timestamp_string(book.last_modified),
"OriginCategory": "Imported",
"RevisionId": book_uuid,
"Status": "Active",
}
def current_time():
return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
def get_description(book):
if not book.comments:
return None
@ -310,6 +337,8 @@ def get_metadata(book):
"IsSocialEnabled": True,
"Language": "en",
"PhoneticPronunciations": {},
# TODO: Fix book.pubdate to return a datetime object so that we can easily
# convert it to the format Kobo devices expect.
"PublicationDate": book.pubdate,
"Publisher": {"Imprint": "", "Name": get_publisher(book),},
"RevisionId": book_uuid,
@ -333,16 +362,148 @@ def get_metadata(book):
return metadata
def reading_state(book):
# TODO: Implement
reading_state = {
# "StatusInfo": {
# "LastModified": get_single_cc_value(book, "lastreadtimestamp"),
# "Status": get_single_cc_value(book, "reading_status"),
# }
# TODO: CurrentBookmark, Location
@kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
@login_required
def HandleStateRequest(book_uuid):
book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
if not book or not book.data:
log.info(u"Book %s not found in database", book_uuid)
return redirect_or_proxy_request()
kobo_reading_state = get_or_create_reading_state(book.id)
if request.method == "GET":
return jsonify([get_kobo_reading_state_response(book, kobo_reading_state)])
else:
update_results_response = {"EntitlementId": book_uuid}
request_data = request.json
if "ReadingStates" not in request_data:
abort(400, description="Malformed request data is missing 'ReadingStates' key")
request_reading_state = request_data["ReadingStates"][0]
request_bookmark = request_reading_state.get("CurrentBookmark")
if request_bookmark:
current_bookmark = kobo_reading_state.current_bookmark
current_bookmark.progress_percent = request_bookmark.get("ProgressPercent")
current_bookmark.content_source_progress_percent = request_bookmark.get("ContentSourceProgressPercent")
location = request_bookmark.get("Location")
if location:
current_bookmark.location_value = location.get("Value")
current_bookmark.location_type = location.get("Type")
current_bookmark.location_source = location.get("Source")
update_results_response["CurrentBookmarkResult"] = {"Result": "Success"}
request_statistics = request_reading_state.get("Statistics")
if request_statistics:
statistics = kobo_reading_state.statistics
statistics.spent_reading_minutes = request_statistics.get("SpentReadingMinutes")
statistics.remaining_time_minutes = request_statistics.get("RemainingTimeMinutes")
update_results_response["StatisticsResult"] = {"Result": "Success"}
request_status_info = request_reading_state.get("StatusInfo")
if request_status_info:
book_read = kobo_reading_state.book_read_link
new_book_read_status = get_ub_read_status(request_status_info.get("Status"))
if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS and new_book_read_status != book_read.read_status:
book_read.times_started_reading += 1
book_read.last_time_started_reading = datetime.datetime.utcnow()
book_read.read_status = new_book_read_status
update_results_response["StatusInfoResult"] = {"Result": "Success"}
ub.session.merge(kobo_reading_state)
ub.session.commit()
return jsonify({
"RequestResult": "Success",
"UpdateResults": [update_results_response],
})
def get_read_status_for_kobo(ub_book_read):
enum_to_string_map = {
None: "ReadyToRead",
ub.ReadBook.STATUS_UNREAD: "ReadyToRead",
ub.ReadBook.STATUS_FINISHED: "Finished",
ub.ReadBook.STATUS_IN_PROGRESS: "Reading",
}
return reading_state
return enum_to_string_map[ub_book_read.read_status]
def get_ub_read_status(kobo_read_status):
string_to_enum_map = {
None: None,
"ReadyToRead": ub.ReadBook.STATUS_UNREAD,
"Finished": ub.ReadBook.STATUS_FINISHED,
"Reading": ub.ReadBook.STATUS_IN_PROGRESS,
}
return string_to_enum_map[kobo_read_status]
def get_or_create_reading_state(book_id):
book_read = ub.session.query(ub.ReadBook).filter(and_(ub.ReadBook.book_id == book_id,
ub.ReadBook.user_id == current_user.id)).one_or_none()
if not book_read:
book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id)
if not book_read.kobo_reading_state:
kobo_reading_state = ub.KoboReadingState(user_id=book_read.user_id, book_id=book_id)
kobo_reading_state.current_bookmark = ub.KoboBookmark()
kobo_reading_state.statistics = ub.KoboStatistics()
book_read.kobo_reading_state = kobo_reading_state
ub.session.add(book_read)
ub.session.commit()
return book_read.kobo_reading_state
def get_kobo_reading_state_response(book, kobo_reading_state):
return {
"EntitlementId": book.uuid,
"Created": convert_to_kobo_timestamp_string(book.timestamp),
"LastModified": convert_to_kobo_timestamp_string(kobo_reading_state.last_modified),
# AFAICT PriorityTimestamp is always equal to LastModified.
"PriorityTimestamp": convert_to_kobo_timestamp_string(kobo_reading_state.priority_timestamp),
"StatusInfo": get_status_info_response(kobo_reading_state.book_read_link),
"Statistics": get_statistics_response(kobo_reading_state.statistics),
"CurrentBookmark": get_current_bookmark_response(kobo_reading_state.current_bookmark),
}
def get_status_info_response(book_read):
resp = {
"LastModified": convert_to_kobo_timestamp_string(book_read.last_modified),
"Status": get_read_status_for_kobo(book_read),
"TimesStartedReading": book_read.times_started_reading,
}
if book_read.last_time_started_reading:
resp["LastTimeStartedReading"] = convert_to_kobo_timestamp_string(book_read.last_time_started_reading)
return resp
def get_statistics_response(statistics):
resp = {
"LastModified": convert_to_kobo_timestamp_string(statistics.last_modified),
}
if statistics.spent_reading_minutes:
resp["SpentReadingMinutes"] = statistics.spent_reading_minutes
if statistics.remaining_time_minutes:
resp["RemainingTimeMinutes"] = statistics.remaining_time_minutes
return resp
def get_current_bookmark_response(current_bookmark):
resp = {
"LastModified": convert_to_kobo_timestamp_string(current_bookmark.last_modified),
}
if current_bookmark.progress_percent:
resp["ProgressPercent"] = current_bookmark.progress_percent
if current_bookmark.content_source_progress_percent:
resp["ContentSourceProgressPercent"] = current_bookmark.content_source_progress_percent
if current_bookmark.location_value:
resp["Location"] = {
"Value": current_bookmark.location_value,
"Type": current_bookmark.location_type,
"Source": current_bookmark.location_source,
}
return resp
@kobo.route("/<book_uuid>/image.jpg")
@ -367,7 +528,6 @@ def TopLevelEndpoint():
# TODO: Implement the following routes
@kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])
@kobo.route("/v1/library/<book_uuid>/state", methods=["PUT"])
@kobo.route("/v1/library/tags", methods=["POST"])
@kobo.route("/v1/library/tags/<shelf_name>", methods=["POST"])
@kobo.route("/v1/library/tags/<tag_id>", methods=["DELETE"])

25
cps/services/SyncToken.py
View File

@ -42,6 +42,13 @@ def to_epoch_timestamp(datetime_object):
return (datetime_object - datetime(1970, 1, 1)).total_seconds()
def get_datetime_from_json(json_object, field_name):
try:
return datetime.utcfromtimestamp(json_object[field_name])
except KeyError:
return datetime.min
class SyncToken():
""" The SyncToken is used to persist state accross requests.
When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service.
@ -53,7 +60,8 @@ class SyncToken():
"""
SYNC_TOKEN_HEADER = "x-kobo-synctoken"
VERSION = "1-0-0"
VERSION = "1-1-0"
LAST_MODIFIED_ADDED_VERSION = "1-1-0"
MIN_VERSION = "1-0-0"
token_schema = {
@ -68,6 +76,7 @@ class SyncToken():
"raw_kobo_store_token": {"type": "string"},
"books_last_modified": {"type": "string"},
"books_last_created": {"type": "string"},
"reading_state_last_modified": {"type": "string"},
},
}
@ -76,10 +85,13 @@ class SyncToken():
raw_kobo_store_token="",
books_last_created=datetime.min,
books_last_modified=datetime.min,
reading_state_last_modified=datetime.min,
):
self.raw_kobo_store_token = raw_kobo_store_token
self.books_last_created = books_last_created
self.books_last_modified = books_last_modified
self.reading_state_last_modified = reading_state_last_modified
@staticmethod
def from_headers(headers):
@ -109,12 +121,9 @@ class SyncToken():
raw_kobo_store_token = data_json["raw_kobo_store_token"]
try:
books_last_modified = datetime.utcfromtimestamp(
data_json["books_last_modified"]
)
books_last_created = datetime.utcfromtimestamp(
data_json["books_last_created"]
)
books_last_modified = get_datetime_from_json(data_json, "books_last_modified")
books_last_created = get_datetime_from_json(data_json, "books_last_created")
reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified")
except TypeError:
log.error("SyncToken timestamps don't parse to a datetime.")
return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
@ -123,6 +132,7 @@ class SyncToken():
raw_kobo_store_token=raw_kobo_store_token,
books_last_created=books_last_created,
books_last_modified=books_last_modified,
reading_state_last_modified=reading_state_last_modified
)
def set_kobo_store_header(self, store_headers):
@ -143,6 +153,7 @@ class SyncToken():
"raw_kobo_store_token": self.raw_kobo_store_token,
"books_last_modified": to_epoch_timestamp(self.books_last_modified),
"books_last_created": to_epoch_timestamp(self.books_last_created),
"reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified)
},
}
return b64encode_json(token)

83
cps/ub.py
View File

@ -20,6 +20,7 @@
from __future__ import division, print_function, unicode_literals
import os
import datetime
import itertools
from binascii import hexlify
from flask import g
@ -31,11 +32,12 @@ try:
oauth_support = True
except ImportError:
oauth_support = False
from sqlalchemy import create_engine, exc, exists
from sqlalchemy import create_engine, exc, exists, event
from sqlalchemy import Column, ForeignKey
from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float
from sqlalchemy.orm import backref, foreign, relationship, remote, sessionmaker, Session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql.expression import and_
from werkzeug.security import generate_password_hash
from . import constants # , config
@ -284,10 +286,19 @@ class BookShelf(Base):
class ReadBook(Base):
__tablename__ = 'book_read_link'
STATUS_UNREAD = 0
STATUS_FINISHED = 1
STATUS_IN_PROGRESS = 2
id = Column(Integer, primary_key=True)
book_id = Column(Integer, unique=False)
user_id = Column(Integer, ForeignKey('user.id'), unique=False)
is_read = Column(Boolean, unique=False)
read_status = Column(Integer, unique=False, default=STATUS_UNREAD, nullable=False)
kobo_reading_state = relationship("KoboReadingState", uselist=False, primaryjoin="and_(ReadBook.user_id == foreign(KoboReadingState.user_id), "
"ReadBook.book_id == foreign(KoboReadingState.book_id))", cascade="all", backref=backref("book_read_link", uselist=False))
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
last_time_started_reading = Column(DateTime, nullable=True)
times_started_reading = Column(Integer, default=0, nullable=False)
class Bookmark(Base):
@ -300,6 +311,54 @@ class Bookmark(Base):
bookmark_key = Column(String)
# The Kobo ReadingState API keeps track of 4 timestamped entities:
# ReadingState, StatusInfo, Statistics, CurrentBookmark
# Which we map to the following 4 tables:
# KoboReadingState, ReadBook, KoboStatistics and KoboBookmark
class KoboReadingState(Base):
__tablename__ = 'kobo_reading_state'
id = Column(Integer, primary_key=True, autoincrement=True)
user_id = Column(Integer, ForeignKey('user.id'))
book_id = Column(Integer)
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
priority_timestamp = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
current_bookmark = relationship("KoboBookmark", uselist=False, backref="kobo_reading_state", cascade="all")
statistics = relationship("KoboStatistics", uselist=False, backref="kobo_reading_state", cascade="all")
class KoboBookmark(Base):
__tablename__ = 'kobo_bookmark'
id = Column(Integer, primary_key=True)
kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id'))
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
location_source = Column(String)
location_type = Column(String)
location_value = Column(String)
progress_percent = Column(Float)
content_source_progress_percent = Column(Float)
class KoboStatistics(Base):
__tablename__ = 'kobo_statistics'
id = Column(Integer, primary_key=True)
kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id'))
last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
remaining_time_minutes = Column(Integer)
spent_reading_minutes = Column(Integer)
# Updates the last_modified timestamp in the KoboReadingState table if any of its children tables are modified.
@event.listens_for(Session, 'before_flush')
def receive_before_flush(session, flush_context, instances):
for change in itertools.chain(session.new, session.dirty):
if isinstance(change, (ReadBook, KoboStatistics, KoboBookmark)):
if change.kobo_reading_state:
change.kobo_reading_state.last_modified = datetime.datetime.utcnow()
# Baseclass representing Downloads from calibre-web in app.db
class Downloads(Base):
__tablename__ = 'downloads'
@ -352,6 +411,12 @@ def migrate_Database(session):
ReadBook.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "bookmark"):
Bookmark.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "kobo_reading_state"):
KoboReadingState.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "kobo_bookmark"):
KoboBookmark.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "kobo_statistics"):
KoboStatistics.__table__.create(bind=engine)
if not engine.dialect.has_table(engine.connect(), "registration"):
ReadBook.__table__.create(bind=engine)
conn = engine.connect()
@ -373,6 +438,16 @@ def migrate_Database(session):
conn.execute("ALTER TABLE remote_auth_token ADD column 'token_type' INTEGER DEFAULT 0")
conn.execute("update remote_auth_token set 'token_type' = 0")
session.commit()
try:
session.query(exists().where(ReadBook.read_status)).scalar()
except exc.OperationalError:
conn = engine.connect()
conn.execute("ALTER TABLE book_read_link ADD column 'read_status' INTEGER DEFAULT 0")
conn.execute("UPDATE book_read_link SET 'read_status' = 1 WHERE is_read")
conn.execute("ALTER TABLE book_read_link ADD column 'last_modified' DATETIME")
conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME")
conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0")
session.commit()
# Handle table exists, but no content
cnt = session.query(Registration).count()

21
cps/web.py
View File

@ -314,13 +314,19 @@ def toggle_read(book_id):
book = ub.session.query(ub.ReadBook).filter(and_(ub.ReadBook.user_id == int(current_user.id),
ub.ReadBook.book_id == book_id)).first()
if book:
book.is_read = not book.is_read
if book.read_status == ub.ReadBook.STATUS_FINISHED:
book.read_status = ub.ReadBook.STATUS_UNREAD
else:
book.read_status = ub.ReadBook.STATUS_FINISHED
else:
readBook = ub.ReadBook()
readBook.user_id = int(current_user.id)
readBook.book_id = book_id
readBook.is_read = True
readBook = ub.ReadBook(user_id=current_user.id, book_id = book_id)
readBook.read_status = ub.ReadBook.STATUS_FINISHED
book = readBook
if not book.kobo_reading_state:
kobo_reading_state = ub.KoboReadingState(user_id=current_user.id, book_id=book_id)
kobo_reading_state.current_bookmark = ub.KoboBookmark()
kobo_reading_state.statistics = ub.KoboStatistics()
book.kobo_reading_state = kobo_reading_state
ub.session.merge(book)
ub.session.commit()
else:
@ -980,7 +986,7 @@ def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs)
order = order or []
if not config.config_read_column:
readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\
.filter(ub.ReadBook.is_read == True).all()
.filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all()
readBookIds = [x.book_id for x in readBooks]
else:
try:
@ -1448,7 +1454,8 @@ def show_book(book_id):
if not config.config_read_column:
matching_have_read_book = ub.session.query(ub.ReadBook).\
filter(and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == book_id)).all()
have_read = len(matching_have_read_book) > 0 and matching_have_read_book[0].is_read
have_read = len(
matching_have_read_book) > 0 and matching_have_read_book[0].read_status == ub.ReadBook.STATUS_FINISHED
else:
try:
matching_have_read_book = getattr(entries, 'custom_column_'+str(config.config_read_column))