Merge branch 'master' into Develop

# Conflicts:
#	cps.py
#	cps/web.py

cps.py

@@ -72,7 +72,6 @@ def main():
     app.register_blueprint(admi)
     app.register_blueprint(remotelogin)
     app.register_blueprint(meta)
-    # if config.config_use_google_drive:
     app.register_blueprint(gdrive)
     app.register_blueprint(editbook)
     if kobo_available:

cps/__init__.py

@@ -37,6 +37,11 @@ from . import config_sql, logger, cache_buster, cli, ub, db
 from .reverseproxy import ReverseProxied
 from .server import WebServer
 
+try:
+    import lxml
+    lxml_present = True
+except ImportError:
+    lxml_present = False
 
 mimetypes.init()
 mimetypes.add_type('application/xhtml+xml', '.xhtml')
@@ -90,6 +95,16 @@ db.CalibreDB.setup_db(config.config_calibre_dir, cli.settingspath)
 calibre_db = db.CalibreDB()
 
 def create_app():
+    if sys.version_info < (3, 0):
+        log.info(
+            '*** Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2, please update your installation to Python3 ***')
+        print(
+            '*** Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2, please update your installation to Python3 ***')
+        sys.exit(5)
+    if not lxml_present:
+        log.info('*** "lxml" is needed for calibre-web to run. Please install it using pip: "pip install lxml" ***')
+        print('*** "lxml" is needed for calibre-web to run. Please install it using pip: "pip install lxml" ***')
+        sys.exit(6)
     app.wsgi_app = ReverseProxied(app.wsgi_app)
     # For python2 convert path to unicode
     if sys.version_info < (3, 0):
@@ -99,12 +114,8 @@ def create_app():
 
     if os.environ.get('FLASK_DEBUG'):
         cache_buster.init_cache_busting(app)
-
     log.info('Starting Calibre Web...')
-    if sys.version_info < (3, 0):
+
-        log.info('*** Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2, please update your installation to Python3 ***')
-        print('*** Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2, please update your installation to Python3 ***')
-        sys.exit(5)
     Principal(app)
     lm.init_app(app)
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))

cps/admin.py

@@ -888,7 +888,7 @@ def list_restriction(res_type, user_id):
     else:
         json_dumps = ""
     js = json.dumps(json_dumps)
-    response = make_response(js.replace("'", '"'))
+    response = make_response(js) #.replace("'", '"')
     response.headers["Content-Type"] = "application/json; charset=utf-8"
     return response
 

cps/editbooks.py

@@ -26,16 +26,19 @@ from datetime import datetime
 import json
 from shutil import copyfile
 from uuid import uuid4
-
-# Improve this to check if scholarly is available in a global way, like other pythonic libraries
-have_scholar = True
 try:
-    from scholarly import scholarly
+    from lxml.html.clean import clean_html
 except ImportError:
-    have_scholar = False
     pass
 
 
+# Improve this to check if scholarly is available in a global way, like other pythonic libraries
+try:
+    from scholarly import scholarly
+    have_scholar = True
+except ImportError:
+    have_scholar = False
+
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
@@ -57,6 +60,8 @@ except ImportError:
     pass  # We're not using Python 3
 
 
+
+
 editbook = Blueprint('editbook', __name__)
 log = logger.create()
 
@@ -459,9 +464,11 @@ def edit_book_series_index(series_index, book):
 # Handle book comments/description
 def edit_book_comments(comments, book):
     modif_date = False
+    if comments:
+        comments = clean_html(comments)
     if len(book.comments):
         if book.comments[0].text != comments:
-            book.comments[0].text = comments
+            book.comments[0].text = clean_html(comments)
             modif_date = True
     else:
         if comments:
@@ -515,6 +522,8 @@ def edit_cc_data_value(book_id, book, c, to_save, cc_db_value, cc_string):
         to_save[cc_string] = 1 if to_save[cc_string] == 'True' else 0
     elif c.datatype == 'comments':
         to_save[cc_string] = Markup(to_save[cc_string]).unescape()
+        if to_save[cc_string]:
+            to_save[cc_string] = clean_html(to_save[cc_string])
     elif c.datatype == 'datetime':
         try:
             to_save[cc_string] = datetime.strptime(to_save[cc_string], "%Y-%m-%d")

cps/helper.py

@@ -38,6 +38,7 @@ from flask_login import current_user
 from sqlalchemy.sql.expression import true, false, and_, text, func
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash
+from markupsafe import escape
 
 try:
     from urllib.parse import quote
@@ -97,10 +98,11 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
         settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
     else:
         settings = dict()
-    txt = (u"%s -> %s: %s" % (
+    link = '<a href="{}">{}</a>"'.format(url_for('web.show_book', book_id=book.id), escape(book.title))  # prevent xss
+    txt = u"{} -> {}: {}".format(
            old_book_format,
            new_book_format,
-           "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + book.title + "</a>"))
+           link)
     settings['old_book_format'] = old_book_format
     settings['new_book_format'] = new_book_format
     WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail, user_id))
@@ -773,7 +775,7 @@ def render_task_status(tasklist):
 
             ret['taskMessage'] = "{}: {}".format(_(task.name), task.message)
             ret['progress'] = "{} %".format(int(task.progress * 100))
-            ret['user'] = user
+            ret['user'] = escape(user)  # prevent xss
             renderedtasklist.append(ret)
 
     return renderedtasklist

cps/jinjia.py

@@ -31,7 +31,7 @@ from babel.dates import format_date
 from flask import Blueprint, request, url_for
 from flask_babel import get_locale
 from flask_login import current_user
-
+from markupsafe import escape
 from . import logger
 
 
@@ -129,6 +129,10 @@ def formatseriesindex_filter(series_index):
             return series_index
     return 0
 
+@jinjia.app_template_filter('escapedlink')
+def escapedlink_filter(url, text):
+    return "<a href='{}'>{}</a>".format(url, escape(text))
+
 @jinjia.app_template_filter('uuidfilter')
 def uuidfilter(var):
     return uuid4()

cps/static/js/main.js

@@ -20,6 +20,15 @@ function getPath() {
     var jsFileLocation = $("script[src*=jquery]").attr("src");  // the js file path
     return jsFileLocation.substr(0, jsFileLocation.search("/static/js/libs/jquery.min.js"));  // the js folder path
 }
+
+function elementSorter(a, b) {
+    a = +a.slice(0, -2);
+    b = +b.slice(0, -2);
+    if (a > b) return 1;
+    if (a < b) return -1;
+    return 0;
+}
+
 // Generic control/related handler to show/hide fields based on a checkbox' value
 // e.g.
 //  <input type="checkbox" data-control="stuff-to-show">
@@ -712,3 +721,4 @@ $(function() {
         });
     });
 });
+

cps/static/js/table.js

@@ -22,6 +22,26 @@ var selections = [];
 var reload = false;
 
 $(function() {
+    $('#tasktable').bootstrapTable({
+        formatNoMatches: function () {
+            return '';
+        },
+        striped: true
+    });
+    if ($('#tasktable').length) {
+        setInterval(function () {
+            $.ajax({
+                method: "get",
+                url: getPath() + "/ajax/emailstat",
+                async: true,
+                timeout: 900,
+                success: function (data) {
+                    $('#table').bootstrapTable("load", data);
+                }
+            });
+        }, 2000);
+    }
+
     $("#books-table").on("check.bs.table check-all.bs.table uncheck.bs.table uncheck-all.bs.table",
         function (e, rowsAfter, rowsBefore) {
             var rows = rowsAfter;
@@ -611,6 +631,7 @@ function checkboxFormatter(value, row){
     else
         return '<input type="checkbox" class="chk" data-pk="' + row.id + '" data-name="' + this.field + '" onchange="checkboxChange(this, ' + row.id + ', \'' + this.name + '\', ' + this.column + ')">';
 }
+
 function singlecheckboxFormatter(value, row){
     if(value)
         return '<input type="checkbox" class="chk" data-pk="' + row.id + '" data-name="' + this.field + '" checked onchange="checkboxChange(this, ' + row.id + ', \'' + this.name + '\', 0)">';

cps/templates/detail.html

@@ -120,9 +120,8 @@
         </p>
         </div>
       {% endif %}
-
       {% if entry.series|length > 0 %}
-        <p>{{_("Book %(index)s of %(range)s", index=entry.series_index|formatfloat(2), range=("<a href='" + url_for('web.books_list', data='series', sort_param='stored', book_id=entry.series[0].id) + "'>" + entry.series[0].name + "</a>")|safe) }}</p>
+          <p>{{_("Book %(index)s of %(range)s", index=entry.series_index | formatfloat(2), range=(url_for('web.books_list', data='series', sort_param='stored', book_id=entry.series[0].id)|escapedlink(entry.series[0].name))|safe)}}</p>
 
       {% endif %}
 

cps/templates/index.html

@@ -63,7 +63,7 @@
 </div>
 {% endif %}
 <div class="discover load-more">
-  <h2 class="{{title}}">{{_(title)}}</h2>
+  <h2 class="{{title}}">{{title}}</h2>
     <div class="filterheader hidden-xs hidden-sm">
       <a data-toggle="tooltip" title="{{_('Sort according to book date, newest first')}}" id="new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='new')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
       <a data-toggle="tooltip" title="{{_('Sort according to book date, oldest first')}}" id="old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='old')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>

cps/templates/tasks.html

@@ -5,7 +5,7 @@
 {% block body %}
 <div class="discover">
     <h2>{{_('Tasks')}}</h2>
-    <table class="table table-no-bordered" id="table" data-url="{{  url_for('web.get_email_status_json') }}"  data-sort-name="starttime" data-sort-order="asc">
+    <table class="table table-no-bordered" id="tasktable" data-url="{{  url_for('web.get_email_status_json') }}"  data-sort-name="starttime" data-sort-order="asc">
       <thead>
         <tr>
             {% if g.user.role_admin() %}
@@ -25,32 +25,9 @@
 {% endblock %}
 {% block js %}
 <script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-table.min.js') }}"></script>
-        <script>
+<script src="{{ url_for('static', filename='js/table.js') }}"></script>
-            // ToDo: Move to js file
+<script>
-            $('#table').bootstrapTable({
+    // ToDo: Move to js file
-               formatNoMatches: function () {
+</script>
-                return '';
-                },
-                striped: true
-            });
-            setInterval(function() {
-                $.ajax({
-                    method:"get",
-                    url: "{{ url_for('web.get_email_status_json')}}",
-                    async: true,
-                    timeout: 900,
-                    success:function(data){
-                    $('#table').bootstrapTable("load", data);
-                }
-                });
-            }, 1000);
-            function elementSorter(a, b) {
-                a = +a.slice(0, -2);
-                b = +b.slice(0, -2);
-                if (a > b) return 1;
-                if (a < b) return -1;
-                return 0;
-            }
-        </script>
 
 {% endblock %}

cps/ub.py

@@ -91,9 +91,9 @@ def store_user_session():
                 user_session = User_Sessions(flask_session.get('_user_id', ""), flask_session.get('_id', ""))
                 session.add(user_session)
                 session.commit()
-                log.info("Login and store session : " + flask_session.get('_id', ""))
+                log.debug("Login and store session : " + flask_session.get('_id', ""))
             else:
-                log.info("Found stored session : " + flask_session.get('_id', ""))
+                log.debug("Found stored session: " + flask_session.get('_id', ""))
         except (exc.OperationalError, exc.InvalidRequestError) as e:
             session.rollback()
             log.exception(e)
@@ -102,7 +102,7 @@ def store_user_session():
 
 def delete_user_session(user_id, session_key):
     try:
-        log.info("Deleted session_key : " + session_key)
+        log.debug("Deleted session_key: " + session_key)
         session.query(User_Sessions).filter(User_Sessions.user_id==user_id,
                                             User_Sessions.session_key==session_key).delete()
         session.commit()

optional-requirements.txt

@@ -30,7 +30,6 @@ Flask-Dance>=2.0.0,<5.1.0
 SQLAlchemy-Utils>=0.33.5,<0.38.0
 
 # extracting metadata
-lxml>=3.8.0,<4.7.0
 rarfile>=2.7
 scholarly>=1.2.0, <1.3
 

requirements.txt

@@ -12,3 +12,4 @@ SQLAlchemy>=1.3.0,<1.5.0
 tornado>=4.1,<6.2
 Wand>=0.4.4,<0.7.0
 unidecode>=0.04.19,<1.3.0
+lxml>=3.8.0,<4.7.0