Backup metadata 4th step

cps/admin.py

@@ -136,28 +136,38 @@ def admin_forbidden():
 @admin_required
 def shutdown():
     task = request.get_json().get('parameter', -1)
-    showtext = {}
+    show_text = {}
     if task in (0, 1):  # valid commandos received
         # close all database connections
         calibre_db.dispose()
         ub.dispose()
 
         if task == 0:
-            showtext['text'] = _(u'Server restarted, please reload page')
+            show_text['text'] = _(u'Server restarted, please reload page')
         else:
-            showtext['text'] = _(u'Performing shutdown of server, please close window')
+            show_text['text'] = _(u'Performing shutdown of server, please close window')
         # stop gevent/tornado server
         web_server.stop(task == 0)
-        return json.dumps(showtext)
+        return json.dumps(show_text)
 
     if task == 2:
         log.warning("reconnecting to calibre database")
         calibre_db.reconnect_db(config, ub.app_DB_path)
-        showtext['text'] = _(u'Reconnect successful')
+        show_text['text'] = _(u'Reconnect successful')
-        return json.dumps(showtext)
+        return json.dumps(show_text)
 
-    showtext['text'] = _(u'Unknown command')
+    show_text['text'] = _(u'Unknown command')
-    return json.dumps(showtext), 400
+    return json.dumps(show_text), 400
+
+@admi.route("/metadata_backup", methods=["POST"])
+@login_required
+@admin_required
+def queue_metadata_backup():
+    show_text = {}
+    log.warning("Queuing all books for metadata backup")
+    helper.set_all_metadata_dirty()
+    show_text['text'] = _(u'Books successfully queued fo Metadata Backup')
+    return json.dumps(show_text)
 
 
 # method is available without login and not protected by CSRF to make it easy reachable, is per default switched off

cps/db.py

@@ -399,7 +399,7 @@ class CustomColumns(Base):
         display_dict = json.loads(self.display)
         return display_dict
 
-    def to_json(self, value, extra):
+    def to_json(self, value, extra, sequence):
         content = dict()
         content['table'] = "custom_column_" + str(self.id)
         content['column'] = "value"
@@ -417,7 +417,7 @@ class CustomColumns(Base):
         content['category_sort'] = "value"
         content['is_csp'] = False
         content['is_editable'] = self.editable
-        content['rec_index'] = self.id + 22     # toDo why ??
+        content['rec_index'] = sequence + 22     # toDo why ??
         content['#value#'] = value
         content['#extra#'] = extra
         content['is_multiple2'] = {}

cps/helper.py

@@ -1037,6 +1037,7 @@ def update_thumbnail_cache():
 
 def set_all_metadata_dirty():
     WorkerThread.add(None, TaskBackupMetadata(export_language=get_locale(),
-                                              translated_title=_("cover"),
+                                              translated_title=_("Cover"),
-                                              set_dirty=True),
+                                              set_dirty=True,
+                                              task_message=N_("Queue all books for metadata backup")),
                      hidden=False)

cps/schedule.py

@@ -34,7 +34,7 @@ def get_scheduled_tasks(reconnect=True):
 
     # ToDo make configurable. Generate metadata.opf file for each changed book
     if True:
-        tasks.append([lambda: TaskBackupMetadata(), "en", 'backup metadata', False])
+        tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False])
 
     # Generate all missing book cover thumbnails
     if config.schedule_generate_book_covers:

cps/static/js/main.js

@@ -503,6 +503,23 @@ $(function() {
             }
         });
     });
+    $("#metadata_backup").click(function() {
+        $("#DialogHeader").addClass("hidden");
+        $("#DialogFinished").addClass("hidden");
+        $("#DialogContent").html("");
+        $("#spinner2").show();
+        $.ajax({
+            method: "post",
+            contentType: "application/json; charset=utf-8",
+            dataType: "json",
+            url: getPath() + "/metadata_backup",
+            success: function success(data) {
+                $("#spinner2").hide();
+                $("#DialogContent").html(data.text);
+                $("#DialogFinished").removeClass("hidden");
+            }
+        });
+    });
     $("#perform_update").click(function() {
         $("#DialogHeader").removeClass("hidden");
         $("#spinner2").show();

cps/tasks/metadata_backup.py

@@ -42,7 +42,7 @@ NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
 class TaskBackupMetadata(CalibreTask):
 
     def __init__(self, export_language="en",
-                 translated_title="cover",
+                 translated_title="Cover",
                  set_dirty=False,
                  task_message=N_('Backing up Metadata')):
         super(TaskBackupMetadata, self).__init__(task_message)
@@ -62,7 +62,8 @@ class TaskBackupMetadata(CalibreTask):
         try:
             books = self.calibre_db.session.query(db.Books).all()
             for book in books:
-                self.calibre_db.set_metadata_dirty(book)
+                self.calibre_db.set_metadata_dirty(book.id)
+            self.calibre_db.session.commit()
             self._handleSuccess()
         except Exception as ex:
             self.log.debug('Error adding book for backup: ' + str(ex))
@@ -74,21 +75,25 @@ class TaskBackupMetadata(CalibreTask):
         try:
             metadata_backup = self.calibre_db.session.query(db.Metadata_Dirtied).all()
             custom_columns = self.calibre_db.session.query(db.CustomColumns).order_by(db.CustomColumns.label).all()
+            count = len(metadata_backup)
+            i = 0
             for backup in metadata_backup:
                 book = self.calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none()
-                self.calibre_db.session.query(db.Metadata_Dirtied).filter(db.Metadata_Dirtied == backup.id).delete()
+                self.calibre_db.session.query(db.Metadata_Dirtied).filter(
+                    db.Metadata_Dirtied.book == backup.book).delete()
                 self.calibre_db.session.commit()
                 if book:
                     self.open_metadata(book, custom_columns)
-                    self._handleSuccess()
-                    self.calibre_db.session.close()
                 else:
                     self.log.error("Book {} not found in database".format(backup.book))
-                    self._handleError("Book {} not found in database".format(backup.book))
+                    # self._handleError("Book {} not found in database".format(backup.book))
+                i += 1
+                self.progress = (1.0 / count) * i
+            self._handleSuccess()
             self.calibre_db.session.close()
 
         except Exception as ex:
-            self.log.debug('Error creating metadata backup: ' + str(ex))
+            self.log.debug('Error creating metadata backup for book {}: '.format(book.id) + str(ex))
             self._handleError('Error creating metadata backup: ' + str(ex))
             self.calibre_db.session.rollback()
             self.calibre_db.session.close()
@@ -155,35 +160,36 @@ class TaskBackupMetadata(CalibreTask):
         title.text = book.title
         for author in book.authors:
             creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP)
-            creator.text = str(author)
+            creator.text = str(author.name)
             creator.set(OPF + "file-as", book.author_sort)     # ToDo Check
             creator.set(OPF + "role", "aut")
         contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP)
         contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]"
         contributor.set(OPF + "file-as", "calibre")     # ToDo Check
-        contributor.set(OPF + "role", "bpk")
+        contributor.set(OPF + "role", "bkp")
 
         date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP)
         date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate)
-
+        if book.comments:
+            for b in book.comments:
+                description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP)
+                description.text = b.text
         if not book.languages:
             language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
             language.text = self.export_language
         else:
             for b in book.languages:
                 language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
-                language.text = str(b.languages)
+                language.text = str(b.lang_code)
         for b in book.tags:
             subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP)
-            subject.text = str(b.tags)
+            subject.text = str(b.name)
-        if book.comments:
-            description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP)
-            description.text = escape(str(book.comments))
         etree.SubElement(metadata, "meta", name="calibre:author_link_map",
-                         content="{" + escape(",".join(['"' + str(a) + '":""' for a in book.authors])) + "}",
+                         content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}",
                          nsmap=NSMAP)
+        for b in book.series:
             etree.SubElement(metadata, "meta", name="calibre:series",
-                         content=str(book.series),
+                             content=str(str(b.name)),
                              nsmap=NSMAP)
         etree.SubElement(metadata, "meta", name="calibre:series_index",
                          content=str(book.series_index),
@@ -195,6 +201,7 @@ class TaskBackupMetadata(CalibreTask):
         etree.SubElement(metadata, "meta", name="calibre:title_sort",
                          content=book.sort,
                          nsmap=NSMAP)
+        sequence = 0
         for cc in custom_columns:
             value = None
             extra = None
@@ -203,8 +210,9 @@ class TaskBackupMetadata(CalibreTask):
                 value = cc_entry[0].get("value")
                 extra = cc_entry[0].get("extra")
             etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label),
-                             content=escape(cc.to_json(value, extra)),
+                             content=cc.to_json(value, extra, sequence),
                              nsmap=NSMAP)
+            sequence += 1
 
         # generate guide element and all sub elements of it
         # Title is translated from default export language
@@ -213,15 +221,24 @@ class TaskBackupMetadata(CalibreTask):
 
         # prepare finalize everything and output
         doc = etree.ElementTree(package)
+        # doc = etree.tostring(package, xml_declaration=True, encoding='utf-8', pretty_print=True) # .replace(b""", b""")
         try:
             with open(book_metadata_filepath, 'wb') as f:
+                # f.write(doc)
                 doc.write(f, xml_declaration=True, encoding='utf-8', pretty_print=True)
         except Exception:
             # ToDo: Folder not writeable error
             pass
     @property
     def name(self):
-        return "Backing up Metadata"
+        return "Metadata backup"
+
+    # needed for logging
+    def __str__(self):
+        if self.set_dirty:
+            return "Queue all books for metadata backup"
+        else:
+            return "Perform metadata backup"
 
     @property
     def is_cancellable(self):

cps/templates/admin.html

@@ -207,6 +207,9 @@
     <div class="btn btn-default" id="admin_restart" data-toggle="modal" data-target="#RestartDialog">{{_('Restart')}}</div>
     <div class="btn btn-default" id="admin_stop" data-toggle="modal" data-target="#ShutdownDialog">{{_('Shutdown')}}</div>
   </div>
+  <div class="row form-group">
+    <div class="btn btn-default" id="metadata_backup" data-toggle="modal" data-target="#StatusDialog">{{_('Queue all books for metadata backup')}}</div>
+  </div>
 
   <div class="row">
     <div class="col">