Bugfixes kobo sync

cps/admin.py

@@ -40,7 +40,7 @@ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError
 from sqlalchemy.sql.expression import func, or_, text
 
 from . import constants, logger, helper, services
-from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
+from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, kobo_sync_status
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
     valid_email, check_username
 from .gdriveutils import is_gdrive_ready, gdrive_support
@@ -1432,8 +1432,13 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
         else:
             content.sidebar_view &= ~constants.DETAIL_RANDOM
 
+        old_state = content.kobo_only_shelves_sync
         content.kobo_only_shelves_sync = int(to_save.get("kobo_only_shelves_sync") == "on") or 0
-
+        # 1 -> 0: nothing has to be done
+        # 0 -> 1: all synced books have to be added to archived books, + currently synced shelfs
+        # which don't have to be synced have to be removed (added to Shelf archive)
+        if old_state == 0 and content.kobo_only_shelves_sync == 1:
+            kobo_sync_status.update_on_sync_shelfs(content.id)
         if to_save.get("default_language"):
             content.default_language = to_save["default_language"]
         if to_save.get("locale"):

cps/kobo.py

@@ -173,11 +173,9 @@ def HandleSyncRequest():
                 .join(ub.KoboSyncedBooks, ub.KoboSyncedBooks.book_id == db.Books.id, isouter=True)
                 .filter(or_(ub.KoboSyncedBooks.user_id != current_user.id,
                             ub.KoboSyncedBooks.book_id == None))
-                #.filter(or_(db.Books.last_modified > sync_token.books_last_modified,
+                .filter(ub.BookShelf.date_added > sync_token.books_last_modified)
-                #            ub.BookShelf.date_added > sync_token.books_last_modified))
-                .filter(ub.BookShelf.date_added > sync_token.books_last_modified) #?? or also or from above
                 .filter(db.Data.format.in_(KOBO_FORMATS))
-                .filter(calibre_db.common_filters())
+                .filter(calibre_db.common_filters(allow_show_archived=True))
                 .order_by(db.Books.id)
                 .order_by(ub.ArchivedBook.last_modified)
                 .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id)
@@ -204,8 +202,6 @@ def HandleSyncRequest():
                    .order_by(db.Books.id)
         )
 
-    #if sync_token.books_last_id > -1:
-    #    changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id)
 
     reading_states_in_new_entitlements = []
     if sqlalchemy_version2:

cps/kobo_sync_status.py

@@ -19,6 +19,8 @@
 
 from flask_login import current_user
 from . import ub
+import datetime
+from sqlalchemy.sql.expression import or_
 
 
 def add_synced_books(book_id):
@@ -32,3 +34,38 @@ def add_synced_books(book_id):
 def remove_synced_book(book_id):
     ub.session.query(ub.KoboSyncedBooks).filter(ub.KoboSyncedBooks.book_id == book_id).delete()
     ub.session_commit()
+
+def add_archived_books(book_id):
+    archived_book = (
+        ub.session.query(ub.ArchivedBook)
+        .filter(ub.ArchivedBook.book_id == book_id)
+        .first()
+    )
+    if not archived_book:
+        archived_book = ub.ArchivedBook(user_id=current_user.id, book_id=book_id)
+    archived_book.is_archived = True
+    archived_book.last_modified = datetime.datetime.utcnow()
+
+    ub.session.merge(archived_book)
+    ub.session_commit()
+
+
+
+# select all books which are synced by the current user and do not belong to a synced shelf and them to archive
+# select all shelfs from current user which are synced and do not belong to the "only sync" shelfs
+def update_on_sync_shelfs(content_id):
+        books_to_archive = (ub.session.query(ub.KoboSyncedBooks)
+                            .join(ub.BookShelf, ub.KoboSyncedBooks.book_id == ub.BookShelf.book_id, isouter=True)
+                            .join(ub.Shelf, ub.Shelf.user_id == content_id, isouter=True)
+                            .filter(or_(ub.Shelf.kobo_sync == 0, ub.Shelf.kobo_sync == None))
+                            .filter(ub.KoboSyncedBooks.user_id == content_id).all())
+        for b in books_to_archive:
+            add_archived_books(b.book_id)
+            ub.session.query(ub.KoboSyncedBooks).filter(ub.KoboSyncedBooks.book_id == b.book_id).filter(ub.KoboSyncedBooks.user_id == content_id).delete()
+            ub.session_commit()
+
+        shelfs_to_archive = ub.session.query(ub.Shelf).filter(ub.Shelf.user_id == content_id).filter(
+            ub.Shelf.kobo_sync == 0).all()
+        for a in shelfs_to_archive:
+            ub.session.add(ub.ShelfArchive(uuid=a.uuid, user_id=content_id))
+            ub.session_commit()

cps/web.py

@@ -46,7 +46,7 @@ from werkzeug.security import generate_password_hash, check_password_hash
 
 from . import constants, logger, isoLanguages, services
 from . import babel, db, ub, config, get_locale, app
-from . import calibre_db
+from . import calibre_db, kobo_sync_status
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
 from .helper import check_valid_domain, render_task_status, check_email, check_username, \
     get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
@@ -1628,7 +1628,13 @@ def change_profile(kobo_support, local_oauth_check, oauth_status, translations,
             current_user.default_language = to_save["default_language"]
         if to_save.get("locale"):
             current_user.locale = to_save["locale"]
+        old_state = current_user.kobo_only_shelves_sync
+        # 1 -> 0: nothing has to be done
+        # 0 -> 1: all synced books have to be added to archived books, + currently synced shelfs which
+        # don't have to be synced have to be removed (added to Shelf archive)
         current_user.kobo_only_shelves_sync = int(to_save.get("kobo_only_shelves_sync") == "on") or 0
+        if old_state == 0 and current_user.kobo_only_shelves_sync == 1:
+            kobo_sync_status.update_on_sync_shelfs(current_user.id)
 
     except Exception as ex:
         flash(str(ex), category="error")
@@ -1754,12 +1760,6 @@ def show_book(book_id):
         for index in range(0, len(entries.languages)):
             entries.languages[index].language_name = isoLanguages.get_language_name(get_locale(), entries.languages[
                 index].lang_code)
-            #try:
-            #    entries.languages[index].language_name = isoLanguages.get_language_name(get_locale(), LC.parse(entries.languages[index].lang_code)
-            #                                                                            .get_language_name(get_locale())
-            #except UnknownLocaleError:
-            #    entries.languages[index].language_name = _(
-            #        isoLanguages.get(part3=entries.languages[index].lang_code).name)
         cc = get_cc_columns(filter_config_custom_read=True)
         book_in_shelfs = []
         shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all()

test/Calibre-Web TestSummary_Linux.html

@@ -37,14 +37,14 @@
       <div class="row">
         <div class="col-xs-6 col-md-6 col-sm-offset-3" style="margin-top:50px;">
             
-            <p class='text-justify attribute'><strong>Start Time: </strong>2021-10-07 20:52:54</p>
+            <p class='text-justify attribute'><strong>Start Time: </strong>2021-10-09 19:48:39</p>
             
         </div>
       </div>
       <div class="row">
         <div class="col-xs-6 col-md-6 col-sm-offset-3">
             
-            <p class='text-justify attribute'><strong>Stop Time: </strong>2021-10-08 00:16:00</p>
+            <p class='text-justify attribute'><strong>Stop Time: </strong>2021-10-09 23:11:14</p>
             
         </div>
       </div>
@@ -592,8 +592,8 @@
                     </div>
                     <div class="text-left pull-left">
                         <pre class="text-left">Traceback (most recent call last):
-  File &#34;/home/ozzie/Development/calibre-web-test/test/test_ebook_convert_gdrive.py&#34;, line 330, in test_convert_only
+  File &#34;/home/ozzie/Development/calibre-web-test/test/test_ebook_convert_gdrive.py&#34;, line 332, in test_convert_only
-    self.assertEqual(ret[-3][&#39;result&#39;], &#39;Finished&#39;)
+    self.assertEqual(ret[-2][&#39;result&#39;], &#39;Finished&#39;)
 AssertionError: &#39;Failed&#39; != &#39;Finished&#39;
 - Failed
 + Finished</pre>
@@ -1810,13 +1810,13 @@ AttributeError: &#39;bool&#39; object has no attribute &#39;text&#39;</pre>
     
 
 
-    <tr id="su" class="errorClass">
+    <tr id="su" class="skipClass">
         <td>TestKoboSync</td>
         <td class="text-center">10</td>
-        <td class="text-center">6</td>
+        <td class="text-center">9</td>
-        <td class="text-center">3</td>
-        <td class="text-center">1</td>
         <td class="text-center">0</td>
+        <td class="text-center">0</td>
+        <td class="text-center">1</td>
         <td class="text-center">
             <a onclick="showClassDetail('c21', 10)">Detail</a>
         </td>
@@ -1842,31 +1842,11 @@ AttributeError: &#39;bool&#39; object has no attribute &#39;text&#39;</pre>
     
     
     
-        <tr id="et21.3" class="none bg-info">
+        <tr id='st21.3' class='none bg-warning'>
             <td>
                 <div class='testcase'>TestKoboSync - test_kobo_sync_selected_shelfs</div>
             </td>
-            <td colspan='6'>
+            <td colspan='6' align='center'>SKIP</td>
-                <div class="text-center">
-                    <a class="popup_link text-center" onfocus='blur()' onclick="showTestDetail('div_et21.3')">ERROR</a>
-                </div>
-                <!--css div popup start-->
-                <div id="div_et21.3" class="popup_window test_output" style="display:block;">
-                    <div class='close_button pull-right'>
-                        <button type="button" class="close" aria-label="Close" onfocus="this.blur();"
-                                onclick="document.getElementById('div_et21.3').style.display='none'"><span
-                                aria-hidden="true">&times;</span></button>
-                    </div>
-                    <div class="text-left pull-left">
-                        <pre class="text-left">Traceback (most recent call last):
-  File &#34;/home/ozzie/Development/calibre-web-test/test/test_kobo_sync.py&#34;, line 618, in test_kobo_sync_selected_shelfs
-    self.assertIn(&#34;NewTag&#34;, data3[2])
-IndexError: list index out of range</pre>
-                    </div>
-                    <div class="clearfix"></div>
-                </div>
-                <!--css div popup end-->
-            </td>
         </tr>
     
     
@@ -1880,31 +1860,11 @@ IndexError: list index out of range</pre>
     
     
     
-        <tr id="ft21.5" class="none bg-danger">
+        <tr id='pt21.5' class='hiddenRow bg-success'>
             <td>
                 <div class='testcase'>TestKoboSync - test_sync_changed_book</div>
             </td>
-            <td colspan='6'>
+            <td colspan='6' align='center'>PASS</td>
-                <div class="text-center">
-                    <a class="popup_link text-center" onfocus='blur()' onclick="showTestDetail('div_ft21.5')">FAIL</a>
-                </div>
-                <!--css div popup start-->
-                <div id="div_ft21.5" class="popup_window test_output" style="display:block;">
-                    <div class='close_button pull-right'>
-                        <button type="button" class="close" aria-label="Close" onfocus="this.blur();"
-                                onclick="document.getElementById('div_ft21.5').style.display='none'"><span
-                                aria-hidden="true">&times;</span></button>
-                    </div>
-                    <div class="text-left pull-left">
-                        <pre class="text-left">Traceback (most recent call last):
-  File &#34;/home/ozzie/Development/calibre-web-test/test/test_kobo_sync.py&#34;, line 285, in test_sync_changed_book
-    self.assertEqual(1, len(data))
-AssertionError: 1 != 0</pre>
-                    </div>
-                    <div class="clearfix"></div>
-                </div>
-                <!--css div popup end-->
-            </td>
         </tr>
     
     
@@ -1927,31 +1887,11 @@ AssertionError: 1 != 0</pre>
     
     
     
-        <tr id="ft21.8" class="none bg-danger">
+        <tr id='pt21.8' class='hiddenRow bg-success'>
             <td>
                 <div class='testcase'>TestKoboSync - test_sync_shelf</div>
             </td>
-            <td colspan='6'>
+            <td colspan='6' align='center'>PASS</td>
-                <div class="text-center">
-                    <a class="popup_link text-center" onfocus='blur()' onclick="showTestDetail('div_ft21.8')">FAIL</a>
-                </div>
-                <!--css div popup start-->
-                <div id="div_ft21.8" class="popup_window test_output" style="display:block;">
-                    <div class='close_button pull-right'>
-                        <button type="button" class="close" aria-label="Close" onfocus="this.blur();"
-                                onclick="document.getElementById('div_ft21.8').style.display='none'"><span
-                                aria-hidden="true">&times;</span></button>
-                    </div>
-                    <div class="text-left pull-left">
-                        <pre class="text-left">Traceback (most recent call last):
-  File &#34;/home/ozzie/Development/calibre-web-test/test/test_kobo_sync.py&#34;, line 303, in test_sync_shelf
-    self.assertEqual(1, len(data))
-AssertionError: 1 != 0</pre>
-                    </div>
-                    <div class="clearfix"></div>
-                </div>
-                <!--css div popup end-->
-            </td>
         </tr>
     
     
@@ -1965,31 +1905,11 @@ AssertionError: 1 != 0</pre>
     
     
     
-        <tr id="ft21.10" class="none bg-danger">
+        <tr id='pt21.10' class='hiddenRow bg-success'>
             <td>
                 <div class='testcase'>TestKoboSync - test_sync_upload</div>
             </td>
-            <td colspan='6'>
+            <td colspan='6' align='center'>PASS</td>
-                <div class="text-center">
-                    <a class="popup_link text-center" onfocus='blur()' onclick="showTestDetail('div_ft21.10')">FAIL</a>
-                </div>
-                <!--css div popup start-->
-                <div id="div_ft21.10" class="popup_window test_output" style="display:block;">
-                    <div class='close_button pull-right'>
-                        <button type="button" class="close" aria-label="Close" onfocus="this.blur();"
-                                onclick="document.getElementById('div_ft21.10').style.display='none'"><span
-                                aria-hidden="true">&times;</span></button>
-                    </div>
-                    <div class="text-left pull-left">
-                        <pre class="text-left">Traceback (most recent call last):
-  File &#34;/home/ozzie/Development/calibre-web-test/test/test_kobo_sync.py&#34;, line 269, in test_sync_upload
-    self.assertEqual(1, len(data))
-AssertionError: 1 != 0</pre>
-                    </div>
-                    <div class="clearfix"></div>
-                </div>
-                <!--css div popup end-->
-            </td>
         </tr>
     
     
@@ -4093,10 +4013,10 @@ AssertionError: True is not false</pre>
     <tr id='total_row' class="text-center bg-grey">
         <td>Total</td>
         <td>353</td>
-        <td>341</td>
+        <td>344</td>
-        <td>5</td>
         <td>2</td>
-        <td>5</td>
+        <td>1</td>
+        <td>6</td>
         <td>&nbsp;</td>
     </tr>
 </table>
@@ -4496,7 +4416,7 @@ AssertionError: True is not false</pre>
 </div>
 
 <script>
-    drawCircle(341, 5, 2, 5);
+    drawCircle(344, 2, 1, 6);
     showCase(5);
 </script>