Code cleaning Stats page

Enable calibre's ebook-convert as converter for mobi files (#411, #533)

cps/book_formats.py

@@ -127,11 +127,11 @@ def get_versions():
     else:
         IVersion = _(u'not installed')
     if use_pdf_meta:
-        PVersion=PyPdfVersion
+        PVersion='v'+PyPdfVersion
     else:
         PVersion=_(u'not installed')
     if lxmlversion:
-        XVersion = '.'.join(map(str, lxmlversion))
+        XVersion = 'v'+'.'.join(map(str, lxmlversion))
     else:
         XVersion = _(u'not installed')
-    return {'ImageVersion': IVersion, 'PyPdfVersion': PVersion, 'LxmlVersion':XVersion}
+    return {'Image Magick': IVersion, 'PyPdf': PVersion, 'lxml':XVersion}

cps/converter.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import sys
+import subprocess
+import ub
+import db
+import re
+import web
+from flask_babel import gettext as _
+
+
+RET_FAIL = 0
+RET_SUCCESS = 1
+
+
+def versionKindle():
+    versions = _(u'not installed')
+    if os.path.exists(ub.config.config_converterpath):
+        try:
+            p = subprocess.Popen(ub.config.config_converterpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p.wait()
+            for lines in p.stdout.readlines():
+                if isinstance(lines, bytes):
+                    lines = lines.decode('utf-8')
+                if re.search('Amazon kindlegen\(', lines):
+                    versions = lines
+        except Exception:
+            versions = _(u'Excecution permissions missing')
+    return {'kindlegen' : versions}
+
+
+def versionCalibre():
+    versions = _(u'not installed')
+    if os.path.exists(ub.config.config_converterpath):
+        try:
+            p = subprocess.Popen(ub.config.config_converterpath + ' --version', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p.wait()
+            for lines in p.stdout.readlines():
+                if isinstance(lines, bytes):
+                    lines = lines.decode('utf-8')
+                if re.search('.*\(calibre', lines):
+                    versions = lines
+        except Exception:
+            versions = _(u'Excecution permissions missing')
+    return {'Calibre converter' : versions}
+
+
+def convert_kindlegen(file_path, book):
+    error_message = None
+    # vendorpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) +
+    #                                           os.sep + "../vendor" + os.sep))
+    #if sys.platform == "win32":
+    #    kindlegen = (os.path.join(vendorpath, u"kindlegen.exe")).encode(sys.getfilesystemencoding())
+    #else:
+    #    kindlegen = (os.path.join(vendorpath, u"kindlegen")).encode(sys.getfilesystemencoding())
+    if not os.path.exists(ub.config.config_converterpath):
+        error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=ub.config.config_converterpath)
+        web.app.logger.error("convert_kindlegen: " + error_message)
+        return error_message, RET_FAIL
+    try:
+        p = subprocess.Popen((ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    except Exception as e:
+        error_message = _(u"kindlegen failed, no execution permissions")
+        web.app.logger.error("convert_kindlegen: " + error_message)
+        return error_message, RET_FAIL
+    # Poll process for new output until finished
+    while True:
+        nextline = p.stdout.readline()
+        if nextline == '' and p.poll() is not None:
+            break
+        if nextline != "\r\n":
+            # Format of error message (kindlegen translates its output texts):
+            # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
+            conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
+            # If error occoures, log in every case
+            if conv_error:
+                error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
+                                  error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
+                web.app.logger.info("convert_kindlegen: " + error_message)
+                web.app.logger.info(nextline.strip('\r\n'))
+            else:
+                web.app.logger.debug(nextline.strip('\r\n'))
+
+    check = p.returncode
+    if not check or check < 2:
+        book.data.append(db.Data(
+            name=book.data[0].name,
+            book_format="MOBI",
+            book=book.id,
+            uncompressed_size=os.path.getsize(file_path + ".mobi")
+        ))
+        db.session.commit()
+        return file_path + ".mobi", RET_SUCCESS
+    else:
+        web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
+        if not error_message:
+            error_message = 'kindlegen failed, no excecution permissions'
+        return error_message, RET_FAIL
+
+
+def convert_calibre(file_path, book):
+    error_message = None
+    if not os.path.exists(ub.config.config_converterpath):
+        error_message = _(u"Ebook-convert binary %(converterpath)s not found", converterpath=ub.config.config_converterpath)
+        web.app.logger.error("convert_calibre: " + error_message)
+        return error_message, RET_FAIL
+    try:
+        command = ("\""+ub.config.config_converterpath + "\" " + ub.config.config_calibre +
+                  " \"" + file_path + u".epub\" \"" + file_path + u".mobi\"").encode(sys.getfilesystemencoding())
+        p = subprocess.Popen(command,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    except Exception as e:
+        error_message = _(u"Ebook-convert failed, no execution permissions")
+        web.app.logger.error("convert_calibre: " + error_message)
+        return error_message, RET_FAIL
+    # Poll process for new output until finished
+    while True:
+        nextline = p.stdout.readline()
+        if nextline == '' and p.poll() is not None:
+            break
+        web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
+
+    check = p.returncode
+    if check == 0 :
+        book.data.append(db.Data(
+            name=book.data[0].name,
+            book_format="MOBI",
+            book=book.id,
+            uncompressed_size=os.path.getsize(file_path + ".mobi")
+        ))
+        db.session.commit()
+        return file_path + ".mobi", RET_SUCCESS
+    else:
+        web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
+        if not error_message:
+            error_message = 'Ebook-convert failed, no excecution permissions'
+        return error_message, RET_FAIL
+
+
+def versioncheck():
+    if ub.config.config_ebookconverter == 1:
+        return versionKindle()
+    elif ub.config.config_ebookconverter == 2:
+        return versionCalibre()
+    else:
+        return {'ebook_converter':''}
+
+
+def convert_mobi(file_path, book):
+    if ub.config.config_ebookconverter == 2:
+        return convert_calibre(file_path, book)
+    else:
+        return convert_kindlegen(file_path, book)

cps/helper.py

@@ -14,6 +14,7 @@ import traceback
 import re
 import unicodedata
 from io import BytesIO
+import converter
 
 try:
     from StringIO import StringIO
@@ -57,17 +58,6 @@ RET_FAIL = 0
 
 
 def make_mobi(book_id, calibrepath):
-    error_message = None
-    vendorpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) +
-                                               os.sep + "../vendor" + os.sep))
-    if sys.platform == "win32":
-        kindlegen = (os.path.join(vendorpath, u"kindlegen.exe")).encode(sys.getfilesystemencoding())
-    else:
-        kindlegen = (os.path.join(vendorpath, u"kindlegen")).encode(sys.getfilesystemencoding())
-    if not os.path.exists(kindlegen):
-        error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=kindlegen)
-        app.logger.error("make_mobi: " + error_message)
-        return error_message, RET_FAIL
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
     data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
     if not data:
@@ -77,45 +67,7 @@ def make_mobi(book_id, calibrepath):
 
     file_path = os.path.join(calibrepath, book.path, data.name)
     if os.path.exists(file_path + u".epub"):
-        try:
+        return converter.convert_mobi(file_path, book)
-            p = subprocess.Popen((kindlegen + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-        except Exception:
-            error_message = _(u"kindlegen failed, no execution permissions")
-            app.logger.error("make_mobi: " + error_message)
-            return error_message, RET_FAIL
-        # Poll process for new output until finished
-        while True:
-            nextline = p.stdout.readline()
-            if nextline == '' and p.poll() is not None:
-                break
-            if nextline != "\r\n":
-                # Format of error message (kindlegen translates its output texts):
-                # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
-                conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
-                # If error occoures, log in every case
-                if conv_error:
-                    error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
-                                      error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
-                    app.logger.info("make_mobi: " + error_message)
-                    app.logger.info(nextline.strip('\r\n'))
-                app.logger.debug(nextline.strip('\r\n'))
-
-        check = p.returncode
-        if not check or check < 2:
-            book.data.append(db.Data(
-                    name=book.data[0].name,
-                    book_format="MOBI",
-                    book=book.id,
-                    uncompressed_size=os.path.getsize(file_path + ".mobi")
-                ))
-            db.session.commit()
-            return file_path + ".mobi", RET_SUCCESS
-        else:
-            app.logger.info("make_mobi: kindlegen failed with error while converting book")
-            if not error_message:
-                error_message = 'kindlegen failed, no excecution permissions'
-            return error_message, RET_FAIL
     else:
         error_message = "make_mobi: epub not found: %s.epub" % file_path
         return error_message, RET_FAIL

cps/server.py

@@ -92,9 +92,9 @@ class server:
 
     def getNameVersion(self):
         if gevent_present:
-            return {'gevent':geventVersion}
+            return {'Gevent':'v'+geventVersion}
         else:
-            return {'tornado':tornadoVersion}
+            return {'Tornado':'v'+tornadoVersion}
 
 
 # Start Instance of Server

cps/templates/config_edit.html

@@ -159,6 +159,38 @@
       </div>
     </div>
   </div>
+  <div class="panel panel-default">
+    <div class="panel-heading">
+      <h4 class="panel-title">
+        <div class="accordion-toggle" data-toggle="collapse" href="#collapseeight">
+          <span class="glyphicon glyphicon-plus"></span>
+           {{_('E-Book converter')}}
+        </div>
+      </h4>
+    </div>
+    <div id="collapseeight" class="panel-collapse collapse">
+      <div class="panel-body">
+          <div class="form-group">
+              <div><input type="radio" name="config_ebookconverter" id="converter0" value="0" {% if content.config_ebookconverter == 0 %}checked{% endif %}>
+              <label for="converter0">{{_('No converter')}}</label></div>
+              <div><label><input type="radio" name="config_ebookconverter" id="converter1" value="1" {% if content.config_ebookconverter == 1 %}checked{% endif %}>
+              <label for="converter1">{{_('Use Kindlegen')}}</label></div>
+              <div><label><input type="radio" name="config_ebookconverter" id="converter2" value="2" {% if content.config_ebookconverter == 2 %}checked{% endif %}>
+              <label for="converter2">{{_('Use calibre\'s ebook converter')}}</label></div>
+          </div>
+          <div data-related="calibre">
+            <div class="form-group">
+                <label for="config_calibre">{{_('E-Book converter settings')}}</label>
+                <input type="text" class="form-control" id="config_calibre" name="config_calibre" value="{% if content.config_calibre != None %}{{ content.config_calibre }}{% endif %}" autocomplete="off">
+            </div>
+            <div class="form-group">
+                <label for="config_calibre">{{_('Path to convertertool')}}</label>
+                <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if content.config_converterpath != None %}{{ content.config_converterpath }}{% endif %}" autocomplete="off">
+            </div>
+          </div>
+      </div>
+    </div>
+  </div>
 </div>
 
 

cps/templates/stats.html

@@ -34,75 +34,12 @@
     </tr>
   </thead>
   <tbody>
+  {% for library,version in versions.iteritems() %}
     <tr>
-      <th>Python</th>
+      <th>{{library}}</th>
-      <td>{{versions['PythonVersion']}}</td>
+      <td>{{version}}</td>
     </tr>
-    {% if 'tornado' in versions %}
+  {% endfor %}
-    <tr>
-      <th>Tornado web server</th>
-      <td>v{{versions['tornado']}}</td>
-    </tr>
-    {% endif %}
-    {% if 'gevent' in versions %}
-    <tr>
-      <th>Gevent web server</th>
-      <td>v{{versions['gevent']}}</td>
-    </tr>
-    {% endif %}
-    <tr>
-      <th>Kindlegen</th>
-      <td>{{versions['KindlegenVersion']}}</td>
-    </tr>
-    <tr>
-      <th>ImageMagick</th>
-      <td>{{versions['ImageVersion']}}</td>
-    </tr>
-    <tr>
-      <th>PyPDF2</th>
-      <td>v{{versions['PyPdfVersion']}}</td>
-    </tr>
-    <tr>
-      <th>Babel</th>
-      <td>v{{versions['babel']}}</td>
-    </tr>
-    <tr>
-      <th>SqlAlchemy</th>
-      <td>v{{versions['sqlalchemy']}}</td>
-    </tr>
-    <tr>
-      <th>Flask</th>
-      <td>v{{versions['flask']}}</td>
-    </tr>
-    <tr>
-      <th>Flask Login</th>
-      <td>v{{versions['flasklogin']}}</td>
-    </tr>
-    <tr>
-      <th>Flask Principal</th>
-      <td>v{{versions['flask_principal']}}</td>
-    </tr>
-    <tr>
-      <th>ISO639 Languages</th>
-      <td>v{{versions['iso639']}}</td>
-    </tr>
-    <tr>
-      <th>Requests</th>
-      <td>v{{versions['requests']}}</td>
-    </tr>
-    <tr>
-      <th>SQlite</th>
-      <td>v{{versions['sqlite']}}</td>
-    </tr>
-    <tr>
-      <th>Pysqlite</th>
-      <td>v{{versions['pysqlite']}}</td>
-    </tr>
-    <tr>
-      <th>lxml</th>
-      <td>v{{versions['LxmlVersion']}}</td>
-    </tr>
-
   </tbody>
 </table>
 {% endblock %}

cps/ub.py

@@ -301,8 +301,11 @@ class Settings(Base):
     config_use_goodreads = Column(Boolean)
     config_goodreads_api_key = Column(String)
     config_goodreads_api_secret = Column(String)
-    config_mature_content_tags = Column(String)  # type: str
+    config_mature_content_tags = Column(String)
     config_logfile = Column(String)
+    config_ebookconverter = Column(Integer, default=0)
+    config_converterpath = Column(String)
+    config_calibre = Column(String)
 
     def __repr__(self):
         pass
@@ -355,6 +358,9 @@ class Config:
         self.config_columns_to_ignore = data.config_columns_to_ignore
         self.config_use_google_drive = data.config_use_google_drive
         self.config_google_drive_folder = data.config_google_drive_folder
+        self.config_ebookconverter = data.config_ebookconverter
+        self.config_converterpath = data.config_converterpath
+        self.config_calibre = data.config_calibre
         if data.config_google_drive_watch_changes_response:
             self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
         else:
@@ -657,6 +663,16 @@ def migrate_Database():
         conn = engine.connect()
         conn.execute("ALTER TABLE Settings ADD column `config_read_column` INTEGER DEFAULT 0")
         session.commit()
+    try:
+        session.query(exists().where(Settings.config_ebookconverter)).scalar()
+        session.commit()
+    except exc.OperationalError:  # Database is not compatible, some rows are missing
+        conn = engine.connect()
+        conn.execute("ALTER TABLE Settings ADD column `config_ebookconverter` INTEGER DEFAULT 0")
+        conn.execute("ALTER TABLE Settings ADD column `config_converterpath` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_calibre` String DEFAULT ''")
+        session.commit()
+
     # Remove login capability of user Guest
     conn = engine.connect()
     conn.execute("UPDATE user SET password='' where nickname = 'Guest' and password !=''")

cps/web.py

@@ -66,15 +66,16 @@ from iso639 import __version__ as iso639Version
 from uuid import uuid4
 import os.path
 import sys
-import subprocess
+
 import re
 import db
 from shutil import move, copyfile
 import shutil
 import gdriveutils
+import converter
 import tempfile
 import hashlib
-from redirect import redirect_back, is_safe_url
+from redirect import redirect_back
 
 try:
     from urllib.parse import quote
@@ -1430,40 +1431,23 @@ def admin_forbidden():
 @app.route("/stats")
 @login_required
 def stats():
-    counter = len(db.session.query(db.Books).all())
+    counter = db.session.query(db.Books).count()
-    authors = len(db.session.query(db.Authors).all())
+    authors = db.session.query(db.Authors).count()
-    categorys = len(db.session.query(db.Tags).all())
+    categorys = db.session.query(db.Tags).count()
-    series = len(db.session.query(db.Series).all())
+    series = db.session.query(db.Series).count()
     versions = uploader.book_formats.get_versions()
-    vendorpath = os.path.join(config.get_main_dir, "vendor")
+    versions['Babel'] = 'v'+babelVersion
-    if sys.platform == "win32":
+    versions['Sqlalchemy'] = 'v'+sqlalchemyVersion
-        kindlegen = os.path.join(vendorpath, u"kindlegen.exe")
+    versions['Flask'] = 'v'+flaskVersion
-    else:
+    versions['Flask Login'] = 'v'+flask_loginVersion
-        kindlegen = os.path.join(vendorpath, u"kindlegen")
+    versions['Flask Principal'] = 'v'+flask_principalVersion
-    versions['KindlegenVersion'] = _('not installed')
+    versions['Iso 639'] = 'v'+iso639Version
-    if os.path.exists(kindlegen):
+    versions['Requests'] = 'v'+requests.__version__
-        try:
+    versions['pySqlite'] = 'v'+db.engine.dialect.dbapi.version
-            p = subprocess.Popen(kindlegen, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    versions['Sqlite'] = 'v'+db.engine.dialect.dbapi.sqlite_version
-            p.wait()
+    versions.update(converter.versioncheck())
-            for lines in p.stdout.readlines():
-                if isinstance(lines, bytes):
-                    lines = lines.decode('utf-8')
-                if re.search('Amazon kindlegen\(', lines):
-                    versions['KindlegenVersion'] = lines
-        except Exception:
-            versions['KindlegenVersion'] = _(u'Excecution permissions missing')
-    versions['PythonVersion'] = sys.version
-    versions['babel'] = babelVersion
-    versions['sqlalchemy'] = sqlalchemyVersion
-    versions['flask'] = flaskVersion
-    versions['flasklogin'] = flask_loginVersion
-    versions['flask_principal'] = flask_principalVersion
     versions.update(server.Server.getNameVersion())
-    # versions['tornado'] = tornadoVersion
+    versions['Python'] = sys.version
-    versions['iso639'] = iso639Version
-    versions['requests'] = requests.__version__
-    versions['pysqlite'] = db.engine.dialect.dbapi.version
-    versions['sqlite'] = db.engine.dialect.dbapi.sqlite_version
 
     return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions,
                                  categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")
@@ -2572,6 +2556,14 @@ def view_configuration():
         ub.session.commit()
         flash(_(u"Calibre-web configuration updated"), category="success")
         config.loadSettings()
+        if reboot_required:
+            # db.engine.dispose() # ToDo verify correct
+            ub.session.close()
+            ub.engine.dispose()
+            # stop Server
+            server.Server.setRestartTyp(True)
+            server.Server.stopServer()
+            app.logger.info('Reboot required, restarting')
     readColumn = db.session.query(db.Custom_Columns)\
             .filter(db.and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
     return render_title_template("config_view_edit.html", content=config, readColumns=readColumn,
@@ -2680,6 +2672,13 @@ def configuration_helper(origin):
         if "config_public_reg" in to_save and to_save["config_public_reg"] == "on":
             content.config_public_reg = 1
 
+        if "config_converterpath" in to_save:
+            content.config_converterpath = to_save["config_converterpath"].strip()
+        if "config_calibre" in to_save:
+            content.config_calibre = to_save["config_calibre"].strip()
+        if "config_ebookconverter" in to_save:
+            content.config_ebookconverter = int(to_save["config_ebookconverter"])
+
         # Remote login configuration
         content.config_remote_login = ("config_remote_login" in to_save and to_save["config_remote_login"] == "on")
         if not content.config_remote_login: