Bugfix Get gdrive instances (#554, #525)

Metadata.db download works again
removed DEVELOPMENT constant
removed db logging in debug mode (too, noisy, to less information)
code refactoring url_for_other_page
feed languge set to en-EN
update status shos local time instead of UTC
Error handling (back to index page) in case of gdrive authenticate aborted
Mistyping page register fixed
Mistyping bokk fixed
Added uploaded books to tasklist (#442)
Error handling for failed file update added
Code refactoring worker thread
Tasks now never show any decimal values
Converter function unified
removed shell from subprocess call
preparation for limiting domain for registering emails
Book series can now increased in 0.1 steps (#562)
Accordion panels in config are now usable on touch devices like iPad (#545)
Gdrive authenticate button only visible after logged in (#525)
Fixed misstyping in german translation

cps/gdriveutils.py

@@ -3,8 +3,9 @@ try:
     from pydrive.drive import GoogleDrive
     from pydrive.auth import RefreshError
     from apiclient import errors
+    gdrive_support = True
 except ImportError:
-    pass
+    gdrive_support = False
 
 import os
 from ub import config
@@ -259,6 +260,13 @@ def copyDriveFileRemote(drive, origin_file_id, copy_title):
         print ('An error occurred: %s' % error)
     return None
 
+
+# Download metadata.db from gdrive
+def downloadFile(path, filename, output):
+    f = getFileFromEbooksFolder(path, filename)
+    f.GetContentFile(output)
+
+
 def moveGdriveFolderRemote(origin_file, target_folder):
     drive = getDrive(Gdrive.Instance().drive)
     previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')])
@@ -339,7 +347,7 @@ def uploadFileToEbooksFolder(destFile, f):
 
 def watchChange(drive, channel_id, channel_type, channel_address,
               channel_token=None, expiration=None):
-    drive = getDrive(drive)
+    # drive = getDrive(drive)
     # Watch for all changes to a user's Drive.
     # Args:
     # service: Drive API service instance.
@@ -382,7 +390,7 @@ def watchFile(drive, file_id, channel_id, channel_type, channel_address,
     Raises:
     apiclient.errors.HttpError: if http request to create channel fails.
     """
-    drive = getDrive(drive)
+    # drive = getDrive(drive)
 
     body = {
         'id': channel_id,
@@ -405,7 +413,7 @@ def stopChannel(drive, channel_id, resource_id):
     Raises:
     apiclient.errors.HttpError: if http request to create channel fails.
     """
-    drive = getDrive(drive)
+    # drive = getDrive(drive)
     # service=drive.auth.service
     body = {
         'id': channel_id,
@@ -415,7 +423,7 @@ def stopChannel(drive, channel_id, resource_id):
 
 
 def getChangeById (drive, change_id):
-    drive = getDrive(drive)
+    # drive = getDrive(drive)
     # Print a single Change resource information.
     #
     # Args:

cps/templates/admin.html

@@ -53,6 +53,9 @@
   </table>
 
     <div class="btn btn-default" id="admin_edit_email"><a href="{{url_for('edit_mailsettings')}}">{{_('Change SMTP settings')}}</a></div>
+    {% if g.allow_registration %}
+    <div class="btn btn-default" id="admin_register_domain"><a href="{{url_for('edit_register_domains')}}">{{_('Edit allowed domains')}}</a></div>
+    {% endif %}
 
   <h2>{{_('Configuration')}}</h2>
   <table class="table table-striped" id="table_configuration">
@@ -79,7 +82,6 @@
   <div class="btn btn-default"><a href="{{url_for('configuration')}}">{{_('Basic Configuration')}}</a></div>
   <div class="btn btn-default"><a href="{{url_for('view_configuration')}}">{{_('UI Configuration')}}</a></div>
   <h2>{{_('Administration')}}</h2>
-      {% if not development %}
     <div>{{_('Current commit timestamp')}}: <span>{{commit}} </span></div>
     <div class="hidden" id="update_info">{{_('Newest commit timestamp')}}: <span></span></div>
     <p></p>
@@ -88,7 +90,6 @@
     <div class="btn btn-default" data-toggle="modal" data-target="#ShutdownDialog">{{_('Stop Calibre-web')}}</div>
     <div class="btn btn-default" id="check_for_update">{{_('Check for update')}}</div>
     <div class="btn btn-default hidden" id="perform_update" data-toggle="modal" data-target="#UpdateprogressDialog">{{_('Perform Update')}}</div>
-      {% endif %}
 </div>
 <!-- Modal -->
 <div id="RestartDialog" class="modal fade" role="dialog">

cps/templates/book_edit.html

@@ -50,7 +50,7 @@
     </div>
     <div class="form-group">
       <label for="series_index">{{_('Series id')}}</label>
-      <input type="number" step="1" min="0" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
+      <input type="number" step="0.1" min="0" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
     </div>
     <div class="form-group">
       <label for="rating">{{_('Rating')}}</label>

cps/templates/config_edit.html

@@ -7,10 +7,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapseOne">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapseOne" style="text-decoration:none;">
           <span class="glyphicon glyphicon-minus"></span>
           {{_('Library Configuration')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapseOne" class="panel-collapse collapse in">
@@ -31,11 +31,15 @@
           </label>
         </div>
       {% else %}
-        {% if show_authenticate_google_drive %}
+        {% if show_authenticate_google_drive and g.user.is_authenticated %}
           <div class="form-group required">
             <a href="{{ url_for('authenticate_google_drive') }}" class="btn btn-primary">{{_('Authenticate Google Drive')}}</a>
           </div>
         {% else %}
+          {% if show_authenticate_google_drive and not g.user.is_authenticated %}
+          <div >{{_('Please finish Google Drive setup after login')}}</div>
+          {% endif %}
+          {% if not show_authenticate_google_drive %}
           <div class="form-group required">
             <label for="config_google_drive_folder">{{_('Google Drive Calibre folder')}}</label>
               <select name="config_google_drive_folder" id="config_google_drive_folder" class="form-control">
@@ -55,17 +59,19 @@
           {% endif %}
           {% endif %}
         {% endif %}
+      {% endif %}
     </div>
       </div>
     </div>
   </div>
+
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapsetwo">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapsetwo" style="text-decoration:none;">
           <span class="glyphicon glyphicon-plus"></span>
           {{_('Server Configuration')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapsetwo" class="panel-collapse collapse">
@@ -88,10 +94,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapsethree">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapsethree" style="text-decoration:none;">
           <span class="glyphicon glyphicon-plus"></span>
           {{_('Logfile Configuration')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapsethree" class="panel-collapse collapse">
@@ -115,10 +121,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapsefive">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapsefive">
           <span class="glyphicon glyphicon-plus"></span>
           {{_('Feature Configuration')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapsefive" class="panel-collapse collapse">
@@ -162,10 +168,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapseeight">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapseeight" style="text-decoration:none;">
           <span class="glyphicon glyphicon-plus"></span>
            {{_('E-Book converter')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapseeight" class="panel-collapse collapse">

cps/templates/config_view_edit.html

@@ -7,10 +7,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapsefour">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapsefour" style="text-decoration:none;">
           <span class="glyphicon glyphicon-plus"></span>
           {{_('View Configuration')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapsefour" class="panel-collapse collapse">
@@ -57,10 +57,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapsesix">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapsesix" style="text-decoration:none;">
           <span class="glyphicon glyphicon-plus"></span>
           {{_('Default settings for new users')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapsesix" class="panel-collapse collapse">
@@ -99,10 +99,10 @@
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
-        <div class="accordion-toggle" data-toggle="collapse" href="#collapseseven">
+        <a class="accordion-toggle" data-toggle="collapse" href="#collapseseven" style="text-decoration:none;">
           <span class="glyphicon glyphicon-plus"></span>
           {{_('Default visibilities for new users')}}
-        </div>
+        </a>
       </h4>
     </div>
     <div id="collapseseven" class="panel-collapse collapse">

cps/templates/layout.html

@@ -186,13 +186,13 @@
           {% if pagination and (pagination.has_next or pagination.has_prev) %}
             <div class="pagination">
               {% if pagination.has_prev %}
-              <a class="previous" href="{{ url_for_other_page(pagination.page - 1)
+              <a class="previous" href="{{ (pagination.page - 1)|url_for_other_page
                 }}">&laquo; {{_('Previous')}}</a>
               {% endif %}
             {% for page in pagination.iter_pages() %}
               {% if page %}
                 {% if page != pagination.page %}
-                  <a href="{{ url_for_other_page(page) }}">{{ page }}</a>
+                  <a href="{{ (page)|url_for_other_page }}">{{ page }}</a>
                 {% else %}
                   <strong>{{ page }}</strong>
                 {% endif %}
@@ -201,7 +201,7 @@
               {% endif %}
             {% endfor %}
             {% if pagination.has_next %}
-              <a class="next" href="{{ url_for_other_page(pagination.page + 1)
+              <a class="next" href="{{ (pagination.page + 1)|url_for_other_page
                 }}">{{_('Next')}} &raquo;</a>
             {% endif %}
             </div>

cps/translations/de/LC_MESSAGES/messages.po

@@ -1287,7 +1287,7 @@ msgstr "Über"
 
 #: cps/templates/layout.html:187
 msgid "Previous"
-msgstr "Voerheriger"
+msgstr "Vorheriger"
 
 #: cps/templates/layout.html:214
 msgid "Book Details"

cps/ub.py

@@ -46,7 +46,6 @@ DEFAULT_PASS = "admin123"
 DEFAULT_PORT = int(os.environ.get("CALIBRE_PORT", 8083))
 
 
-DEVELOPMENT = False
 
 
 class UserBase:
@@ -291,10 +290,7 @@ class Settings(Base):
     config_default_show = Column(SmallInteger, default=2047)
     config_columns_to_ignore = Column(String)
     config_use_google_drive = Column(Boolean)
-    # config_google_drive_client_id = Column(String)
-    # config_google_drive_client_secret = Column(String)
     config_google_drive_folder = Column(String)
-    # config_google_drive_calibre_url_base = Column(String)
     config_google_drive_watch_changes_response = Column(String)
     config_remote_login = Column(Boolean)
     config_use_goodreads = Column(Boolean)
@@ -556,9 +552,6 @@ def migrate_Database():
     except exc.OperationalError:
         conn = engine.connect()
         conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0")
-        # conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''")
-        # conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''")
-        # conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0")
         conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''")
         conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''")
     try:

cps/web.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 try:
-    from pydrive.auth import GoogleAuth
     from googleapiclient.errors import HttpError
-    gdrive_support = True
+    # gdrive_support = True
 except ImportError:
-    gdrive_support = False
+    # gdrive_support = False
+    pass
 
 try:
     from goodreads.client import GoodreadsClient
@@ -47,6 +47,8 @@ from flask_principal import Principal
 from flask_principal import __version__ as flask_principalVersion
 from flask_babel import Babel
 from flask_babel import gettext as _
+import pytz
+# from tzlocal import get_localzone
 import requests
 from werkzeug.security import generate_password_hash, check_password_hash
 from werkzeug.datastructures import Headers
@@ -64,11 +66,9 @@ from iso639 import __version__ as iso639Version
 from uuid import uuid4
 import os.path
 import sys
-
 import re
 import db
 from shutil import move, copyfile
-# import shutil
 import gdriveutils
 import converter
 import tempfile
@@ -185,13 +185,13 @@ lm.anonymous_user = ub.Anonymous
 app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT')
 db.setup_db()
 
-if config.config_log_level == logging.DEBUG:
+'''if config.config_log_level == logging.DEBUG:
     logging.getLogger("sqlalchemy.engine").addHandler(file_handler)
     logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
     logging.getLogger("sqlalchemy.pool").addHandler(file_handler)
     logging.getLogger("sqlalchemy.pool").setLevel(config.config_log_level)
     logging.getLogger("sqlalchemy.orm").addHandler(file_handler)
-    logging.getLogger("sqlalchemy.orm").setLevel(config.config_log_level)
+    logging.getLogger("sqlalchemy.orm").setLevel(config.config_log_level)'''
 
 
 def is_gdrive_ready():
@@ -309,16 +309,6 @@ class Pagination(object):
                 last = num
 
 
-# pagination links in jinja
-def url_for_other_page(page):
-    args = request.view_args.copy()
-    args['page'] = page
-    return url_for(request.endpoint, **args)
-
-
-app.jinja_env.globals['url_for_other_page'] = url_for_other_page
-
-
 def login_required_if_no_ano(func):
     @wraps(func)
     def decorated_view(*args, **kwargs):
@@ -344,6 +334,15 @@ def remote_login_required(f):
 
 
 # custom jinja filters
+
+# pagination links in jinja
+@app.template_filter('url_for_other_page')
+def url_for_other_page(page):
+    args = request.view_args.copy()
+    args['page'] = page
+    return url_for(request.endpoint, **args)
+
+# shortentitles to at longest nchar, shorten longer words if necessary
 @app.template_filter('shortentitle')
 def shortentitle_filter(s,nchar=20):
     text = s.split()
@@ -354,7 +353,7 @@ def shortentitle_filter(s,nchar=20):
             res += '...'
             break
         # if word longer than 20 chars truncate line and append '...', otherwise add whole word to result
-        # string, and summarize total length to stop at 60 chars
+        # string, and summarize total length to stop at chars given by nchar
         if len(line) > nchar:
             res += line[:(nchar-3)] + '[..] '
             suml += nchar+3
@@ -583,6 +582,7 @@ def feed_search(term):
         entriescount = len(entries) if len(entries) > 0 else 1
         pagination = Pagination(1, entriescount, entriescount)
         xml = render_title_template('feed.xml', searchterm=term, entries=entries, pagination=pagination)
+        
     else:
         xml = render_title_template('feed.xml', searchterm="")
     response = make_response(xml)
@@ -596,8 +596,6 @@ def render_title_template(*args, **kwargs):
 
 @app.before_request
 def before_request():
-    if ub.DEVELOPMENT:
-        reload(ub)
     g.user = current_user
     g.allow_registration = config.config_public_reg
     g.allow_upload = config.config_uploading
@@ -620,7 +618,7 @@ def feed_index():
 @app.route("/opds/osd")
 @requires_basic_auth_if_no_ano
 def feed_osd():
-    xml = render_title_template('osd.xml', lang='de-DE')
+    xml = render_title_template('osd.xml', lang='en-EN')
     response = make_response(xml)
     response.headers["Content-Type"] = "application/xml; charset=utf-8"
     return response
@@ -998,6 +996,7 @@ def get_matching_tags():
 @login_required_if_no_ano
 def get_update_status():
     status = {}
+    tz = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone
     if request.method == "GET":
         # should be automatically replaced by git with current commit hash
         commit_id = '$Format:%H$'
@@ -1007,7 +1006,7 @@ def get_update_status():
             status['status'] = True
             commitdate = requests.get('https://api.github.com/repos/janeczku/calibre-web/git/commits/'+commit['object']['sha']).json()
             if "committer" in commitdate:
-                form_date=datetime.datetime.strptime(commitdate['committer']['date'],"%Y-%m-%dT%H:%M:%SZ")
+                form_date=datetime.datetime.strptime(commitdate['committer']['date'],"%Y-%m-%dT%H:%M:%SZ") - datetime.timedelta(seconds=tz)
                 status['commit'] = format_datetime(form_date, format='short', locale=get_locale())
             else:
                 status['commit'] = u'Unknown'
@@ -1552,9 +1551,13 @@ def authenticate_google_drive():
 @app.route("/gdrive/callback")
 def google_drive_callback():
     auth_code = request.args.get('code')
+    try:
         credentials = gdriveutils.Gauth.Instance().auth.flow.step2_exchange(auth_code)
         with open(os.path.join(config.get_main_dir,'gdrive_credentials'), 'w') as f:
             f.write(credentials.to_json())
+    except ValueError as error:
+        app.logger.error(error)
+    finally:
         return redirect(url_for('configuration'))
 
 
@@ -1829,6 +1832,7 @@ def advanced_search():
         q = q.all()
         return render_title_template('search.html', searchterm=searchterm,
                                      entries=q, title=_(u"search"), page="search")
+    # prepare data for search-form
     tags = db.session.query(db.Tags).order_by(db.Tags.name).all()
     series = db.session.query(db.Series).order_by(db.Series.name).all()
     if current_user.filter_language() == u"all":
@@ -2063,7 +2067,7 @@ def register():
             flash(_(u"This username or email address is already in use."), category="error")
             return render_title_template('register.html', title=_(u"register"), page="register")
 
-    return render_title_template('register.html', title=_(u"register"), page="regsiter")
+    return render_title_template('register.html', title=_(u"register"), page="register")
 
 
 @app.route('/login', methods=['GET', 'POST'])
@@ -2520,7 +2524,7 @@ def admin():
     content = ub.session.query(ub.User).all()
     settings = ub.session.query(ub.Settings).first()
     return render_title_template("admin.html", content=content, email=settings, config=config, commit=commit,
-                                 development=ub.DEVELOPMENT, title=_(u"Admin page"), page="admin")
+                                 title=_(u"Admin page"), page="admin")
 
 
 @app.route("/admin/config", methods=["GET", "POST"])
@@ -2626,7 +2630,7 @@ def configuration_helper(origin):
     gdriveError=None
     db_change = False
     success = False
-    if gdrive_support == False:
+    if gdriveutils.gdrive_support == False:
         gdriveError = _('Import of optional Google Drive requirements missing')
     else:
         if not os.path.isfile(os.path.join(config.get_main_dir,'client_secrets.json')):
@@ -2665,7 +2669,7 @@ def configuration_helper(origin):
             else:
                 flash(_(u'client_secrets.json is not configured for web application'), category="error")
                 return render_title_template("config_edit.html", content=config, origin=origin,
-                                             gdrive=gdrive_support, gdriveError=gdriveError,
+                                             gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
                                              goodreads=goodreads_support, title=_(u"Basic Configuration"),
                                              page="config")
         # always show google drive settings, but in case of error deny support
@@ -2691,7 +2695,7 @@ def configuration_helper(origin):
                     ub.session.commit()
                     flash(_(u'Keyfile location is not valid, please enter correct path'), category="error")
                     return render_title_template("config_edit.html", content=config, origin=origin,
-                                                 gdrive=gdrive_support, gdriveError=gdriveError,
+                                                 gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
                                                  goodreads=goodreads_support, title=_(u"Basic Configuration"),
                                                  page="config")
         if "config_certfile" in to_save:
@@ -2703,7 +2707,7 @@ def configuration_helper(origin):
                     ub.session.commit()
                     flash(_(u'Certfile location is not valid, please enter correct path'), category="error")
                     return render_title_template("config_edit.html", content=config, origin=origin,
-                                                 gdrive=gdrive_support, gdriveError=gdriveError,
+                                                 gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
                                                  goodreads=goodreads_support, title=_(u"Basic Configuration"),
                                                  page="config")
         content.config_uploading = 0
@@ -2746,7 +2750,7 @@ def configuration_helper(origin):
                     ub.session.commit()
                     flash(_(u'Logfile location is not valid, please enter correct path'), category="error")
                     return render_title_template("config_edit.html", content=config, origin=origin,
-                                                 gdrive=gdrive_support, gdriveError=gdriveError,
+                                                 gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
                                                  goodreads=goodreads_support, title=_(u"Basic Configuration"),
                                                  page="config")
             else:
@@ -2766,14 +2770,14 @@ def configuration_helper(origin):
             logging.getLogger("book_formats").setLevel(config.config_log_level)
         except Exception as e:
             flash(e, category="error")
-            return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdrive_support,
+            return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdriveutils.gdrive_support,
                                          gdriveError=gdriveError, goodreads=goodreads_support,
                                          title=_(u"Basic Configuration"), page="config")
         if db_change:
             reload(db)
             if not db.setup_db():
                 flash(_(u'DB location is not valid, please enter correct path'), category="error")
-                return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdrive_support,
+                return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdriveutils.gdrive_support,
                                              gdriveError=gdriveError, goodreads=goodreads_support,
                                              title=_(u"Basic Configuration"), page="config")
         if reboot_required:
@@ -2785,12 +2789,12 @@ def configuration_helper(origin):
             app.logger.info('Reboot required, restarting')
         if origin:
             success = True
-    if is_gdrive_ready() and gdrive_support == True:
+    if is_gdrive_ready() and gdriveutils.gdrive_support == True:
         gdrivefolders=gdriveutils.listRootFolders()
     else:
         gdrivefolders=None
     return render_title_template("config_edit.html", origin=origin, success=success, content=config,
-                                 show_authenticate_google_drive=not is_gdrive_ready(), gdrive=gdrive_support,
+                                 show_authenticate_google_drive=not is_gdrive_ready(), gdrive=gdriveutils.gdrive_support,
                                  gdriveError=gdriveError, gdrivefolders=gdrivefolders,
                                  goodreads=goodreads_support, title=_(u"Basic Configuration"), page="config")
 
@@ -3106,10 +3110,14 @@ def edit_book(book_id):
         is_format = db.session.query(db.Data).filter(db.Data.book == book_id).filter(db.Data.format == file_ext.upper()).first()
         if is_format:
             # Format entry already exists, no need to update the database
-            app.logger.info('Bokk format already existing')
+            app.logger.info('Book format already existing')
         else:
             db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
             db.session.add(db_format)
+        uploadText=_(u"File format %s added to %s" % (file_ext.upper(),book.title))
+        helper.global_WorkerThread.add_upload(current_user.nickname, 
+            "<a href=\""+ url_for('show_book', book_id=book.id) +"\">"+ uploadText + "</a>")
+            
 
     to_save = request.form.to_dict()
 
@@ -3307,9 +3315,9 @@ def edit_book(book_id):
 def upload():
     if not config.config_uploading:
         abort(404)
-    # create the function for sorting...
     if request.method == 'POST' and 'btn-upload' in request.files:
         for requested_file in request.files.getlist("btn-upload"):
+            # create the function for sorting...
             db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
             db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
             if '.' in requested_file.filename:
@@ -3407,7 +3415,7 @@ def upload():
             db_book.data.append(db_data)
 
             db.session.add(db_book)
-            db.session.flush()  # flush content get db_book.id avalible
+            db.session.flush()  # flush content get db_book.id available
 
             # add comment
             book_id = db_book.id
@@ -3425,10 +3433,12 @@ def upload():
                 gdriveutils.updateGdriveCalibreFromLocal()
 
             error = helper.update_dir_stucture(book.id, config.config_calibre_dir)
-            # ToDo: Handle error
-            if error:
-                pass
             
+            if error:
+                flash(error, category="error")
+            uploadText=_(u"File %s uploaded" % book.title)
+            helper.global_WorkerThread.add_upload(current_user.nickname, 
+                "<a href=\"" + url_for('show_book', book_id=book.id) + "\">" + uploadText + "</a>")
 
             if db_language is not None:  # display Full name instead of iso639.part3
                 book.languages[0].language_name = _(meta.languages)
@@ -3436,15 +3446,12 @@ def upload():
             for author in db_book.authors:
                 author_names.append(author.name)
             if len(request.files.getlist("btn-upload")) < 2:
-                # db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
                 cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
                 if current_user.role_edit() or current_user.role_admin():
                     return render_title_template('book_edit.html', book=book, authors=author_names,
-                                                 cc=cc,title=_(u"edit metadata"), page="upload")
+                                                 cc=cc, title=_(u"edit metadata"), page="upload")
                 book_in_shelfs = []
                 return render_title_template('detail.html', entry=book, cc=cc,
                                              title=book.title, books_shelfs=book_in_shelfs, page="upload")
     return redirect(url_for("index"))
-    else:
-        return redirect(url_for("index"))
 

cps/worker.py

@@ -13,7 +13,6 @@ import os
 from email.generator import Generator
 import web
 from flask_babel import gettext as _
-# from babel.dates import format_datetime
 import re
 import gdriveutils as gd
 import subprocess
@@ -42,6 +41,7 @@ STAT_FINISH_SUCCESS = 3
 
 TASK_EMAIL = 1
 TASK_CONVERT = 2
+TASK_UPLOAD = 3
 
 RET_FAIL = 0
 RET_SUCCESS = 1
@@ -55,7 +55,6 @@ def get_attachment(bookpath, filename):
     if web.ub.config.config_use_google_drive:
         df = gd.getFileFromEbooksFolder(bookpath, filename)
         if df:
-
             datafile = os.path.join(calibrepath, bookpath, filename)
             if not os.path.exists(os.path.join(calibrepath, bookpath)):
                 os.makedirs(os.path.join(calibrepath, bookpath))
@@ -127,7 +126,7 @@ class emailbase():
         if self.transferSize:
             lock2 = threading.Lock()
             lock2.acquire()
-            value = round(float(self.progress) / float(self.transferSize),2)*100
+            value = int((float(self.progress) / float(self.transferSize))*100)
             lock2.release()
             return str(value) + ' %'
         else:
@@ -173,6 +172,7 @@ class WorkerThread(threading.Thread):
                     self.send_raw_email()
                 if self.queue[self.current]['typ'] == TASK_CONVERT:
                     self.convert_mobi()
+                # TASK_UPLOAD is handled implicitly
                 self.current += 1
             else:
                 doLock.release()
@@ -212,10 +212,7 @@ class WorkerThread(threading.Thread):
         self.UIqueue[self.current]['status'] = _('Started')
         self.queue[self.current]['starttime'] = datetime.now()
         self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
-        if web.ub.config.config_ebookconverter == 2:
+        filename=self.convert()
-            filename = self.convert_calibre()
-        else:
-            filename = self.convert_kindlegen()
         if web.ub.config.config_use_google_drive:
             gd.updateGdriveCalibreFromLocal()
         if(filename):
@@ -223,19 +220,95 @@ class WorkerThread(threading.Thread):
                        self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
                        self.UIqueue[self.current]['user'], _(u"E-Mail: %s" % self.queue[self.current]['title']))
 
-    def convert_kindlegen(self):
+    def convert(self):
+        error_message = None
+        file_path = self.queue[self.current]['file_path']
+        bookid = self.queue[self.current]['bookid']
+        # check if converter-excecutable is existing
+        if not os.path.exists(web.ub.config.config_converterpath):
+            self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath))
+            return
+        try:
+            # check which converter to use kindlegen is "1"
+            if web.ub.config.config_ebookconverter == 1:
+                command = (web.ub.config.config_converterpath + u" \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding())
+            else:           
+                command = (u"\"" + web.ub.config.config_converterpath + u"\" \"" + file_path + u".epub\" \""
+                    + file_path + u".mobi\" " + web.ub.config.config_calibre).encode(sys.getfilesystemencoding())
+
+            if sys.version_info > (3, 0):
+                command = command.decode('Utf-8')
+        
+            p = subprocess.Popen(command, stdout=subprocess.PIPE)
+        except Exception as e:
+            self._handleError(_(u"Ebook-converter failed, no execution permissions"))
+            return
+        if web.ub.config.config_ebookconverter == 1:
+            nextline = p.communicate()[0]
+            # Format of error message (kindlegen translates its output texts):
+            # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
+            conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
+            # If error occoures, log in every case
+            if conv_error:
+                error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
+                                  error=conv_error.group(1), message=conv_error.group(2).strip())
+                web.app.logger.info("convert_kindlegen: " + error_message)
+            else:
+                web.app.logger.debug("convert_kindlegen: " + nextline)
+
+        else:
+            while p.poll() is None:
+                nextline = p.stdout.readline()
+                if sys.version_info > (3, 0):
+                    nextline = nextline.decode('Utf-8', 'backslashreplace')
+                # if nextline == '' and p.poll() is not None:
+                #    break
+                # while p.poll() is None:
+                web.app.logger.debug(nextline.strip('\r\n'))
+                # parse calibre-converter
+                progress = re.search("(\d+)%\s.*", nextline)
+                if progress:
+                    self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
+                # if nextline != "\r\n" and web.ub.config.config_ebookconverter == 1:
+
+        #process returncode
+        check = p.returncode
+        # kindlegen returncodes
+        # 0 = Info(prcgen):I1036: Mobi file built successfully
+        # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
+        # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors! 
+        if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \
+                (check == 0 and web.ub.config.config_ebookconverter == 2):
+            cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
+            new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
+                                     book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
+            cur_book.data.append(new_format)
+            web.db.session.commit()
+            self.queue[self.current]['path'] = cur_book.path
+            self.queue[self.current]['title'] = cur_book.title
+            if web.ub.config.config_use_google_drive:
+                os.remove(file_path + u".epub")
+            self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
+            self.UIqueue[self.current]['status'] = _('Finished')
+            self.UIqueue[self.current]['progress'] = "100 %"
+            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
+                                                    datetime.now() - self.queue[self.current]['starttime'])
+            return file_path + ".mobi" 
+        else:
+            web.app.logger.info("ebook converter failed with error while converting book")
+            if not error_message:       # ToDo Check         
+                error_message = 'Ebook converter failed with unknown error'
+            self._handleError(error_message)
+            return
+                    
+
+                       
+    '''def convert_kindlegen(self):
         error_message = None
         file_path = self.queue[self.current]['file_path']
         bookid = self.queue[self.current]['bookid']
         if not os.path.exists(web.ub.config.config_converterpath):
-            error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=web.ub.config.config_converterpath)
+            self._handleError(_(u"kindlegen binary %(kindlepath)s not found", kindlepath=web.ub.config.config_converterpath))
-            web.app.logger.error("convert_kindlegen: " + error_message)
-            self.queue[self.current]['status'] = STAT_FAIL
-            self.UIqueue[self.current]['status'] = _('Failed')
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            self.UIqueue[self.current]['message'] = error_message
             return
         try:
             command = (web.ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding())
@@ -245,14 +318,7 @@ class WorkerThread(threading.Thread):
                 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
 
         except Exception:
-            error_message = _(u"kindlegen failed, no execution permissions")
+            self._handleError(_(u"kindlegen failed, no execution permissions"))
-            web.app.logger.error("convert_kindlegen: " + error_message)
-            self.queue[self.current]['status'] = STAT_FAIL
-            self.UIqueue[self.current]['status'] = _('Failed')
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            self.UIqueue[self.current]['message'] = error_message
             return
         # Poll process for new output until finished
         while True:
@@ -295,33 +361,21 @@ class WorkerThread(threading.Thread):
             self.UIqueue[self.current]['progress'] = "100 %"
             self.UIqueue[self.current]['runtime'] = self._formatRuntime(
                                                     datetime.now() - self.queue[self.current]['starttime'])
-            return file_path + ".mobi" #, RET_SUCCESS
+            return file_path + ".mobi" 
         else:
             web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
             if not error_message:                
                 error_message = 'kindlegen failed, no excecution permissions'
-            self.queue[self.current]['status'] = STAT_FAIL
+            self._handleError(error_message)
-            self.UIqueue[self.current]['status'] = _('Failed')
+            return
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            self.UIqueue[self.current]['message'] = error_message
-            return # error_message, RET_FAIL
 
     def convert_calibre(self):
         error_message = None
         file_path = self.queue[self.current]['file_path']
         bookid = self.queue[self.current]['bookid']
         if not os.path.exists(web.ub.config.config_converterpath):
-            error_message = _(u"Ebook-convert binary %(converterpath)s not found",
+            self._handleError(_(u"Ebook-convert binary %(converterpath)s not found",
-                              converterpath=web.ub.config.config_converterpath)
+                              converterpath=web.ub.config.config_converterpath))
-            web.app.logger.error("convert_calibre: " + error_message)
-            self.queue[self.current]['status'] = STAT_FAIL
-            self.UIqueue[self.current]['status'] = _('Failed')
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            self.UIqueue[self.current]['message'] = error_message
             return
         try:
             command = (u"\"" + web.ub.config.config_converterpath + u"\" \"" + file_path + u".epub\" \""
@@ -330,16 +384,9 @@ class WorkerThread(threading.Thread):
                 p = subprocess.Popen(command.decode('Utf-8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
             else:
                 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-        except Exception as e:
+        except Exception:
-            error_message = _(u"Ebook-convert failed, no execution permissions")
+            self._handleError(_(u"Ebook-convert failed, no execution permissions"))
-            web.app.logger.error("convert_calibre: " + error_message)
+            return
-            self.queue[self.current]['status'] = STAT_FAIL
-            self.UIqueue[self.current]['status'] = _('Failed')
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            self.UIqueue[self.current]['message'] = error_message
-            return # error_message, RET_FAIL
         # Poll process for new output until finished
         while True:
             nextline = p.stdout.readline()
@@ -354,8 +401,6 @@ class WorkerThread(threading.Thread):
                 web.app.logger.debug(nextline.strip('\r\n'))
             else:
                 web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
-
-
         check = p.returncode
         if check == 0:
             cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
@@ -377,13 +422,8 @@ class WorkerThread(threading.Thread):
             web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
             if not error_message:
                 error_message = 'Ebook-convert failed, no excecution permissions'
-            self.queue[self.current]['status'] = STAT_FAIL
+            self._handleError(error_message)                
-            self.UIqueue[self.current]['status'] = _('Failed')
+            return'''
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            self.UIqueue[self.current]['message'] = error_message
-            return # error_message, RET_FAIL
 
     def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail):
         addLock = threading.Lock()
@@ -418,7 +458,27 @@ class WorkerThread(threading.Thread):
         self.last=len(self.queue)
         addLock.release()
 
+    def add_upload(self, user_name, typ):
+        # if more than 20 entries in the list, clean the list
+        addLock = threading.Lock()
+        addLock.acquire()
+        if self.last >= 20:
+            self.delete_completed_tasks()
+        # progress=100%, runtime=0, and status finished
+        self.queue.append({'starttime': datetime.now(), 'status': STAT_FINISH_SUCCESS, 'typ': TASK_UPLOAD})
+        self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': "100 %", 'type': typ,
+                             'runtime': '0 s', 'status': _('Finished'),'id': self.id })
+        self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
+        self.id += 1
+        self.last=len(self.queue)
+        addLock.release()
+    
+        
     def send_raw_email(self):
+        self.queue[self.current]['starttime'] = datetime.now()
+        self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] 
+        self.queue[self.current]['status'] = STAT_STARTED
+        self.UIqueue[self.current]['status'] = _('Started')
         obj=self.queue[self.current]
         # create MIME message
         msg = MIMEMultipart()
@@ -432,13 +492,7 @@ class WorkerThread(threading.Thread):
             if result:
                 msg.attach(result)
             else:
-                self.queue[self.current]['status'] = STAT_FAIL
+                self._handleError(u"Attachment not found")
-                self.UIqueue[self.current]['status'] = _('Failed')
-                self.UIqueue[self.current]['progress'] = "100 %"
-                self.queue[self.current]['starttime'] = datetime.now()
-                self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
-                self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                    datetime.now() - self.queue[self.current]['starttime'])
                 return
 
         msg['From'] = obj['settings']["mail_from"]
@@ -459,12 +513,6 @@ class WorkerThread(threading.Thread):
             org_stderr = sys.stderr
             sys.stderr = StderrLogger()
 
-            self.queue[self.current]['status'] = STAT_STARTED
-            self.UIqueue[self.current]['status'] = _('Started')
-            self.queue[self.current]['starttime'] = datetime.now()
-            self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
-
-
             if use_ssl == 2:
                 self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
             else:
@@ -490,13 +538,8 @@ class WorkerThread(threading.Thread):
             sys.stderr = org_stderr
 
         except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e:
-            self.queue[self.current]['status'] = STAT_FAIL
+            self._handleError(error_message)
-            self.UIqueue[self.current]['status'] = _('Failed')
+            return None
-            self.UIqueue[self.current]['progress'] = "100 %"
-            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            web.app.logger.error(e)
-        # return None
 
     def _formatRuntime(self, runtime):
         self.UIqueue[self.current]['rt'] = runtime.total_seconds()
@@ -510,6 +553,17 @@ class WorkerThread(threading.Thread):
             retVal = '0 s'
         return retVal
     
+    def _handleError(self, error_message):
+        web.app.logger.error(error_message)
+        self.queue[self.current]['status'] = STAT_FAIL
+        self.UIqueue[self.current]['status'] = _('Failed')
+        self.UIqueue[self.current]['progress'] = "100 %"
+        self.UIqueue[self.current]['runtime'] = self._formatRuntime(
+                                                datetime.now() - self.queue[self.current]['starttime'])
+        self.UIqueue[self.current]['message'] = error_message
+
+
+
 class StderrLogger(object):
 
     buffer = ''