Chunked sync

cps/kobo.py

@@ -43,6 +43,7 @@ from flask_login import current_user
 from werkzeug.datastructures import Headers
 from sqlalchemy import func
 from sqlalchemy.sql.expression import and_, or_
+from sqlalchemy.orm import load_only
 from sqlalchemy.exc import StatementError
 import requests
 
@@ -56,6 +57,8 @@ KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]}
 KOBO_STOREAPI_URL = "https://storeapi.kobo.com"
 KOBO_IMAGEHOST_URL = "https://kbimages1-a.akamaihd.net"
 
+SYNC_ITEM_LIMIT = 5
+
 kobo = Blueprint("kobo", __name__, url_prefix="/kobo/<auth_token>")
 kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo)
 kobo_auth.register_url_value_preprocessor(kobo)
@@ -142,68 +145,70 @@ def HandleSyncRequest():
     new_books_last_modified = sync_token.books_last_modified
     new_books_last_created = sync_token.books_last_created
     new_reading_state_last_modified = sync_token.reading_state_last_modified
+    new_archived_last_modified = datetime.datetime.min
     sync_results = []
 
     # We reload the book database so that the user get's a fresh view of the library
     # in case of external changes (e.g: adding a book through Calibre).
     calibre_db.reconnect_db(config, ub.app_DB_path)
 
-    archived_books = (
-        ub.session.query(ub.ArchivedBook)
-        .filter(ub.ArchivedBook.user_id == int(current_user.id))
-        .all()
-    )
-
-    # We join-in books that have had their Archived bit recently modified in order to either:
-    #   * Restore them to the user's device.
-    #   * Delete them from the user's device.
-    # (Ideally we would use a join for this logic, however cross-database joins don't look trivial in SqlAlchemy.)
-    recently_restored_or_archived_books = []
-    archived_book_ids = {}
-    new_archived_last_modified = datetime.datetime.min
-    for archived_book in archived_books:
-        if archived_book.last_modified > sync_token.archive_last_modified:
-            recently_restored_or_archived_books.append(archived_book.book_id)
-        if archived_book.is_archived:
-            archived_book_ids[archived_book.book_id] = True
-        new_archived_last_modified = max(
-            new_archived_last_modified, archived_book.last_modified)
-
-    # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast.
-    # It looks like it's treating the db.Books.last_modified field as a string and may fail
-    # the comparison because of the +00:00 suffix.
     changed_entries = (
-        calibre_db.session.query(db.Books)
+        calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
-        .join(db.Data)
+        .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
-        .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified,
+        .filter(db.Books.last_modified >= sync_token.books_last_modified)
-                    db.Books.id.in_(recently_restored_or_archived_books)))
+        .filter(db.Books.id>sync_token.books_last_id)
         .filter(db.Data.format.in_(KOBO_FORMATS))
-        .all()
+        # .filter(ub.ArchivedBook.is_archived == 0)
+        .order_by(db.Books.last_modified)
+        .order_by(db.Books.id)
+        .limit(SYNC_ITEM_LIMIT)
     )
 
     reading_states_in_new_entitlements = []
     for book in changed_entries:
-        kobo_reading_state = get_or_create_reading_state(book.id)
+        kobo_reading_state = get_or_create_reading_state(book.Books.id)
         entitlement = {
-            "BookEntitlement": create_book_entitlement(book, archived=(book.id in archived_book_ids)),
+            "BookEntitlement": create_book_entitlement(book.Books, archived=(book.is_archived == True)),
-            "BookMetadata": get_metadata(book),
+            "BookMetadata": get_metadata(book.Books),
         }
 
         if kobo_reading_state.last_modified > sync_token.reading_state_last_modified:
-            entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state)
+            entitlement["ReadingState"] = get_kobo_reading_state_response(book.Books, kobo_reading_state)
             new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
-            reading_states_in_new_entitlements.append(book.id)
+            reading_states_in_new_entitlements.append(book.Books.id)
 
-        if book.timestamp > sync_token.books_last_created:
+        if book.Books.timestamp > sync_token.books_last_created:
             sync_results.append({"NewEntitlement": entitlement})
         else:
             sync_results.append({"ChangedEntitlement": entitlement})
 
         new_books_last_modified = max(
-            book.last_modified, new_books_last_modified
+            book.Books.last_modified, new_books_last_modified
         )
-        new_books_last_created = max(book.timestamp, new_books_last_created)
+        new_books_last_created = max(book.Books.timestamp, new_books_last_created)
 
+    max_change = (changed_entries
+        .from_self()
+        .filter(ub.ArchivedBook.is_archived)
+        .order_by(func.datetime(ub.ArchivedBook.last_modified).desc())
+        .first()
+    )
+    if max_change:
+        max_change = max_change.last_modified
+    else:
+        max_change = new_archived_last_modified
+    new_archived_last_modified = max(new_archived_last_modified, max_change)
+
+    # no. of books returned
+    book_count = changed_entries.count()
+
+    # last entry:
+    if book_count:
+        books_last_id = changed_entries.all()[-1].Books.id or -1
+    else:
+        books_last_id = -1
+
+    # generate reading state data
     changed_reading_states = (
         ub.session.query(ub.KoboReadingState)
         .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
@@ -225,11 +230,12 @@ def HandleSyncRequest():
     sync_token.books_last_modified = new_books_last_modified
     sync_token.archive_last_modified = new_archived_last_modified
     sync_token.reading_state_last_modified = new_reading_state_last_modified
+    sync_token.books_last_id = books_last_id
 
-    return generate_sync_response(sync_token, sync_results)
+    return generate_sync_response(sync_token, sync_results, book_count)
 
 
-def generate_sync_response(sync_token, sync_results):
+def generate_sync_response(sync_token, sync_results, set_cont=False):
     extra_headers = {}
     if config.config_kobo_proxy:
         # Merge in sync results from the official Kobo store.
@@ -245,6 +251,8 @@ def generate_sync_response(sync_token, sync_results):
 
         except Exception as e:
             log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e))
+    if set_cont:
+        extra_headers["x-kobo-sync"] = "continue"
     sync_token.to_headers(extra_headers)
 
     response = make_response(jsonify(sync_results), extra_headers)

cps/services/SyncToken.py

@@ -85,6 +85,7 @@ class SyncToken:
             "archive_last_modified": {"type": "string"},
             "reading_state_last_modified": {"type": "string"},
             "tags_last_modified": {"type": "string"},
+            "books_last_id": {"type": "integer", "optional": True}
         },
     }
 
@@ -96,6 +97,7 @@ class SyncToken:
         archive_last_modified=datetime.min,
         reading_state_last_modified=datetime.min,
         tags_last_modified=datetime.min,
+        books_last_id=-1
     ):
         self.raw_kobo_store_token = raw_kobo_store_token
         self.books_last_created = books_last_created
@@ -103,6 +105,7 @@ class SyncToken:
         self.archive_last_modified = archive_last_modified
         self.reading_state_last_modified = reading_state_last_modified
         self.tags_last_modified = tags_last_modified
+        self.books_last_id = books_last_id
 
     @staticmethod
     def from_headers(headers):
@@ -137,6 +140,7 @@ class SyncToken:
             archive_last_modified = get_datetime_from_json(data_json, "archive_last_modified")
             reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified")
             tags_last_modified = get_datetime_from_json(data_json, "tags_last_modified")
+            books_last_id = data_json["books_last_id"]
         except TypeError:
             log.error("SyncToken timestamps don't parse to a datetime.")
             return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
@@ -147,7 +151,8 @@ class SyncToken:
             books_last_modified=books_last_modified,
             archive_last_modified=archive_last_modified,
             reading_state_last_modified=reading_state_last_modified,
-            tags_last_modified=tags_last_modified
+            tags_last_modified=tags_last_modified,
+            books_last_id=books_last_id
         )
 
     def set_kobo_store_header(self, store_headers):
@@ -170,7 +175,8 @@ class SyncToken:
                 "books_last_created": to_epoch_timestamp(self.books_last_created),
                 "archive_last_modified": to_epoch_timestamp(self.archive_last_modified),
                 "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified),
-                "tags_last_modified": to_epoch_timestamp(self.tags_last_modified)
+                "tags_last_modified": to_epoch_timestamp(self.tags_last_modified),
+                "books_last_id":self.books_last_id
             },
         }
         return b64encode_json(token)