Merge branch 'metadata' into Develop

cps/about.py

@@ -49,9 +49,9 @@ sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefol
 
 def collect_stats():
     if constants.NIGHTLY_VERSION[0] == "$Format:%H$":
-        calibre_web_version = constants.STABLE_VERSION['version']
+        calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta")
     else:
-        calibre_web_version = (constants.STABLE_VERSION['version'] + ' - '
+        calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - '
                                + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - '
                                + constants.NIGHTLY_VERSION[1].replace('%', '%%'))
 

cps/admin.py

@@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param
 from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \
     kobo_sync_status, schedule
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
-    valid_email, check_username
+    valid_email, check_username, get_calibre_binarypath
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .render_template import render_title_template, get_sidebar_config
 from .services.worker import WorkerThread
@@ -217,7 +217,7 @@ def admin():
                     form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:]))
             commit = format_datetime(form_date - tz, format='short')
         else:
-            commit = version['version']
+            commit = version['version'].replace("b", " Beta")
 
     all_user = ub.session.query(ub.User).all()
     # email_settings = mail_config.get_mail_settings()
@@ -1751,6 +1751,7 @@ def _configuration_update_helper():
 
         _config_checkbox_int(to_save, "config_uploading")
         _config_checkbox_int(to_save, "config_unicode_filename")
+        _config_checkbox_int(to_save, "config_embed_metadata")
         # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case
         reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse")
                             and config.config_login_type == constants.LOGIN_LDAP)
@@ -1767,8 +1768,14 @@ def _configuration_update_helper():
             constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',')
 
         _config_string(to_save, "config_calibre")
-        _config_string(to_save, "config_converterpath")
+        _config_string(to_save, "config_binariesdir")
         _config_string(to_save, "config_kepubifypath")
+        if "config_binariesdir" in to_save:
+            calibre_status = helper.check_calibre(config.config_binariesdir)
+            if calibre_status:
+                return _configuration_result(calibre_status)
+            to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert")
+            _config_string(to_save, "config_converterpath")
 
         reboot_required |= _config_int(to_save, "config_login_type")
 

cps/cli.py

@@ -29,8 +29,8 @@ from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE
 
 def version_info():
     if _NIGHTLY_VERSION[1].startswith('$Format'):
-        return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version']
+        return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta")
-    return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1])
+    return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1])
 
 
 class CliParameter(object):

cps/config_sql.py

@@ -34,6 +34,7 @@ except ImportError:
     from sqlalchemy.ext.declarative import declarative_base
 
 from . import constants, logger
+from .subproc_wrapper import process_wait
 
 
 log = logger.create()
@@ -140,10 +141,12 @@ class _Settings(_Base):
 
     config_kepubifypath = Column(String, default=None)
     config_converterpath = Column(String, default=None)
+    config_binariesdir = Column(String, default=None)
     config_calibre = Column(String)
     config_rarfile_location = Column(String, default=None)
     config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
     config_unicode_filename = Column(Boolean, default=False)
+    config_embed_metadata = Column(Boolean, default=True)
 
     config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
 
@@ -186,9 +189,11 @@ class ConfigSQL(object):
         self.load()
 
         change = False
-        if self.config_converterpath == None:  # pylint: disable=access-member-before-definition
+
+        if self.config_binariesdir == None: # pylint: disable=access-member-before-definition
             change = True
-            self.config_converterpath = autodetect_calibre_binary()
+            self.config_binariesdir = autodetect_calibre_binaries()
+            self.config_converterpath = autodetect_converter_binary(self.config_binariesdir)
 
         if self.config_kepubifypath == None:  # pylint: disable=access-member-before-definition
             change = True
@@ -474,17 +479,32 @@ def _migrate_table(session, orm_class, secret_key=None):
             session.rollback()
 
 
-def autodetect_calibre_binary():
+def autodetect_calibre_binaries():
     if sys.platform == "win32":
-        calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe",
+        calibre_path = ["C:\\program files\\calibre\\",
-                        "C:\\program files(x86)\\calibre\\ebook-convert.exe",
+                        "C:\\program files(x86)\\calibre\\",
-                        "C:\\program files(x86)\\calibre2\\ebook-convert.exe",
+                        "C:\\program files(x86)\\calibre2\\",
-                        "C:\\program files\\calibre2\\ebook-convert.exe"]
+                        "C:\\program files\\calibre2\\"]
     else:
-        calibre_path = ["/opt/calibre/ebook-convert"]
+        calibre_path = ["/opt/calibre/"]
     for element in calibre_path:
-        if os.path.isfile(element) and os.access(element, os.X_OK):
+        supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()]
-            return element
+        if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths):
+            values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths]
+            if all(values):
+                version = values[0].group(1)
+                log.debug("calibre version %s", version)
+                return element 
+    return ""
+
+
+def autodetect_converter_binary(calibre_path):
+    if sys.platform == "win32":
+        converter_path = os.path.join(calibre_path, "ebook-convert.exe")
+    else:
+        converter_path = os.path.join(calibre_path, "ebook-convert")
+    if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK):
+        return converter_path
     return ""
 
 
@@ -526,6 +546,7 @@ def load_configuration(session, secret_key):
         session.commit()
 
 
+
 def get_flask_session_key(_session):
     flask_settings = _session.query(_Flask_Settings).one_or_none()
     if flask_settings == None:

cps/constants.py

@@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr'
                      'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg',
                      'opus', 'wav', 'flac', 'm4a', 'm4b'}
 
+_extension = ""
+if sys.platform == "win32":
+    _extension = ".exe"
+SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]}
+
 
 def has_flag(value, bit_flag):
     return bit_flag == (bit_flag & (value or 0))
@@ -169,13 +174,11 @@ BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, d
                                   'series_id, languages, publisher, pubdate, identifiers')
 
 # python build process likes to have x.y.zbw -> b for beta and w a counting number
-STABLE_VERSION = {'version': '0.6.22 Beta'}
+STABLE_VERSION = {'version': '0.6.22b'}
 
 NIGHTLY_VERSION = dict()
 NIGHTLY_VERSION[0] = '$Format:%H$'
 NIGHTLY_VERSION[1] = '$Format:%cI$'
-# NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57'
-# NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00'
 
 # CACHE
 CACHE_TYPE_THUMBNAILS    = 'thumbnails'

cps/epub.py

@@ -23,10 +23,12 @@ from lxml import etree
 from . import isoLanguages, cover
 from . import config, logger
 from .helper import split_authors
+from .epub_helper import get_content_opf, default_ns
 from .constants import BookMeta
 
 log = logger.create()
 
+
 def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name):
     if cover_file is None:
         return None
@@ -44,24 +46,14 @@ def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name):
     return cover.cover_processing(tmp_file_name, cf, extension)
 
 def get_epub_layout(book, book_data):
-    ns = {
-        'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
-        'pkg': 'http://www.idpf.org/2007/opf',
-    }
     file_path = os.path.normpath(os.path.join(config.get_book_path(),
                                               book.path, book_data.name + "." + book_data.format.lower()))
 
     try:
-        epubZip = zipfile.ZipFile(file_path)
+        tree, __ = get_content_opf(file_path, default_ns)
-        txt = epubZip.read('META-INF/container.xml')
+        p = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
-        tree = etree.fromstring(txt)
-        cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
-        cf = epubZip.read(cfname)
 
-        tree = etree.fromstring(cf)
+        layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=default_ns)
-        p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]
-
-        layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=ns)
     except (etree.XMLSyntaxError, KeyError, IndexError) as e:
         log.error("Could not parse epub metadata of book {} during kobo sync: {}".format(book.id, e))
         layout = []
@@ -80,12 +72,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     }
 
     epub_zip = zipfile.ZipFile(tmp_file_path)
-
+    tree, cf_name = get_content_opf(epub_zip, ns)
-    txt = epub_zip.read('META-INF/container.xml')
-    tree = etree.fromstring(txt)
-    cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
-    cf = epub_zip.read(cf_name)
-    tree = etree.fromstring(cf)
 
     cover_path = os.path.dirname(cf_name)
 

cps/epub_helper.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018 lemmsh, Kennyl, Kyosfonica, matthazinski
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import zipfile
+from lxml import etree
+
+from . import isoLanguages
+
+default_ns = {
+    'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
+    'pkg': 'http://www.idpf.org/2007/opf',
+}
+
+OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
+PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/"
+
+OPF = "{%s}" % OPF_NAMESPACE
+PURL = "{%s}" % PURL_NAMESPACE
+
+etree.register_namespace("opf", OPF_NAMESPACE)
+etree.register_namespace("dc", PURL_NAMESPACE)
+
+OPF_NS = {None: OPF_NAMESPACE}  # the default namespace (no prefix)
+NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
+
+
+def updateEpub(src, dest, filename, data, ):
+    # create a temp copy of the archive without filename
+    with zipfile.ZipFile(src, 'r') as zin:
+        with zipfile.ZipFile(dest, 'w') as zout:
+            zout.comment = zin.comment # preserve the comment
+            for item in zin.infolist():
+                if item.filename != filename:
+                    zout.writestr(item, zin.read(item.filename))
+
+    # now add filename with its new data
+    with zipfile.ZipFile(dest, mode='a', compression=zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr(filename, data)
+
+
+def get_content_opf(file_path, ns=default_ns):
+    epubZip = zipfile.ZipFile(file_path)
+    txt = epubZip.read('META-INF/container.xml')
+    tree = etree.fromstring(txt)
+    cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
+    cf = epubZip.read(cf_name)
+
+    return etree.fromstring(cf), cf_name
+
+
+def create_new_metadata_backup(book,  custom_columns, export_language, translated_cover_name, lang_type=3):
+    # generate root package element
+    package = etree.Element(OPF + "package", nsmap=OPF_NS)
+    package.set("unique-identifier", "uuid_id")
+    package.set("version", "2.0")
+
+    # generate metadata element and all sub elements of it
+    metadata = etree.SubElement(package, "metadata", nsmap=NSMAP)
+    identifier = etree.SubElement(metadata, PURL + "identifier", id="calibre_id", nsmap=NSMAP)
+    identifier.set(OPF + "scheme", "calibre")
+    identifier.text = str(book.id)
+    identifier2 = etree.SubElement(metadata, PURL + "identifier", id="uuid_id", nsmap=NSMAP)
+    identifier2.set(OPF + "scheme", "uuid")
+    identifier2.text = book.uuid
+    title = etree.SubElement(metadata, PURL + "title", nsmap=NSMAP)
+    title.text = book.title
+    for author in book.authors:
+        creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP)
+        creator.text = str(author.name)
+        creator.set(OPF + "file-as", book.author_sort)     # ToDo Check
+        creator.set(OPF + "role", "aut")
+    contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP)
+    contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]"
+    contributor.set(OPF + "file-as", "calibre")     # ToDo Check
+    contributor.set(OPF + "role", "bkp")
+
+    date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP)
+    date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate)
+    if book.comments and book.comments[0].text:
+        for b in book.comments:
+            description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP)
+            description.text = b.text
+    for b in book.publishers:
+        publisher = etree.SubElement(metadata, PURL + "publisher", nsmap=NSMAP)
+        publisher.text = str(b.name)
+    if not book.languages:
+        language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
+        language.text = export_language
+    else:
+        for b in book.languages:
+            language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
+            language.text = str(b.lang_code) if lang_type == 3 else isoLanguages.get(part3=b.lang_code).part1
+    for b in book.tags:
+        subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP)
+        subject.text = str(b.name)
+    etree.SubElement(metadata, "meta", name="calibre:author_link_map",
+                     content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}",
+                     nsmap=NSMAP)
+    for b in book.series:
+        etree.SubElement(metadata, "meta", name="calibre:series",
+                         content=str(str(b.name)),
+                         nsmap=NSMAP)
+    if book.series:
+        etree.SubElement(metadata, "meta", name="calibre:series_index",
+                         content=str(book.series_index),
+                         nsmap=NSMAP)
+    if len(book.ratings) and book.ratings[0].rating > 0:
+        etree.SubElement(metadata, "meta", name="calibre:rating",
+                         content=str(book.ratings[0].rating),
+                         nsmap=NSMAP)
+    etree.SubElement(metadata, "meta", name="calibre:timestamp",
+                     content='{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(
+                         d=book.timestamp),
+                     nsmap=NSMAP)
+    etree.SubElement(metadata, "meta", name="calibre:title_sort",
+                     content=book.sort,
+                     nsmap=NSMAP)
+    sequence = 0
+    for cc in custom_columns:
+        value = None
+        extra = None
+        cc_entry = getattr(book, "custom_column_" + str(cc.id))
+        if cc_entry.__len__():
+            value = [c.value for c in cc_entry] if cc.is_multiple else cc_entry[0].value
+            extra = cc_entry[0].extra if hasattr(cc_entry[0], "extra") else None
+        etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label),
+                         content=cc.to_json(value, extra, sequence),
+                         nsmap=NSMAP)
+        sequence += 1
+
+    # generate guide element and all sub elements of it
+    # Title is translated from default export language
+    guide = etree.SubElement(package, "guide")
+    etree.SubElement(guide, "reference", type="cover", title=translated_cover_name, href="cover.jpg")
+
+    return package
+
+def replace_metadata(tree, package):
+    rep_element = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
+    new_element = package.xpath('//metadata', namespaces=default_ns)[0]
+    tree.replace(rep_element, new_element)
+    return etree.tostring(tree,
+                   xml_declaration=True,
+                   encoding='utf-8',
+                   pretty_print=True).decode('utf-8')
+
+

cps/file_helper.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2023 OzzieIsaacs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from tempfile import gettempdir
+import os
+import shutil
+
+def get_temp_dir():
+    tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+    if not os.path.isdir(tmp_dir):
+        os.mkdir(tmp_dir)
+    return tmp_dir
+
+
+def del_temp_dir():
+    tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+    shutil.rmtree(tmp_dir)

cps/gdrive.py

@@ -23,7 +23,6 @@
 import os
 import hashlib
 import json
-import tempfile
 from uuid import uuid4
 from time import time
 from shutil import move, copyfile
@@ -34,6 +33,7 @@ from flask_login import login_required
 
 from . import logger, gdriveutils, config, ub, calibre_db, csrf
 from .admin import admin_required
+from .file_helper import get_temp_dir
 
 gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive')
 log = logger.create()
@@ -139,9 +139,7 @@ try:
                 dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode()
                 if not response['deleted'] and response['file']['title'] == 'metadata.db' \
                     and response['file']['md5Checksum'] != hashlib.md5(dbpath):  # nosec
-                    tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web')
+                    tmp_dir = get_temp_dir()
-                    if not os.path.isdir(tmp_dir):
-                        os.mkdir(tmp_dir)
 
                     log.info('Database file updated')
                     copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time())))

cps/gdriveutils.py

@@ -34,7 +34,6 @@ except ImportError:
     from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError
 from sqlalchemy.orm.exc import StaleDataError
-from sqlalchemy.sql.expression import text
 
 try:
     from httplib2 import __version__ as httplib2_version

cps/helper.py

@@ -25,9 +25,10 @@ import re
 import shutil
 import socket
 from datetime import datetime, timedelta
-from tempfile import gettempdir
 import requests
 import unidecode
+from uuid import uuid4
+from lxml import etree
 
 from flask import send_from_directory, make_response, redirect, abort, url_for
 from flask_babel import gettext as _
@@ -54,12 +55,14 @@ from . import calibre_db, cli_param
 from .tasks.convert import TaskConvert
 from . import logger, config, db, ub, fs
 from . import gdriveutils as gd
-from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES
+from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES
-from .subproc_wrapper import process_wait
+from .subproc_wrapper import process_wait, process_open
 from .services.worker import WorkerThread
 from .tasks.mail import TaskEmail
 from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails
 from .tasks.metadata_backup import TaskBackupMetadata
+from .file_helper import get_temp_dir
+from .epub_helper import get_content_opf, create_new_metadata_backup, updateEpub, replace_metadata
 
 log = logger.create()
 
@@ -921,10 +924,7 @@ def save_cover(img, book_path):
             return False, _("Only jpg/jpeg files are supported as coverfile")
 
     if config.config_use_google_drive:
-        tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+        tmp_dir = get_temp_dir()
-
-        if not os.path.isdir(tmp_dir):
-            os.mkdir(tmp_dir)
         ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img)
         if ret is True:
             gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"),
@@ -938,29 +938,87 @@ def save_cover(img, book_path):
 
 
 def do_download_file(book, book_format, client, data, headers):
+    book_name = data.name
     if config.config_use_google_drive:
         # startTime = time.time()
-        df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format)
+        df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format)
         # log.debug('%s', time.time() - startTime)
         if df:
-            return gd.do_gdrive_download(df, headers)
+            if config.config_embed_metadata and (
+                 (book_format == "kepub" and config.config_kepubifypath ) or
+                 (book_format != "kepub" and config.config_binariesdir)):
+                output_path = os.path.join(config.config_calibre_dir, book.path)
+                if not os.path.exists(output_path):
+                    os.makedirs(output_path)
+                output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format)
+                gd.downloadFile(book.path, book_name + "." + book_format, output)
+                if book_format == "kepub" and config.config_kepubifypath:
+                    filename, download_name = do_kepubify_metadata_replace(book, output)
+                elif book_format != "kepub" and config.config_binariesdir:
+                    filename, download_name = do_calibre_export(book.id, book_format)
+            else:
+                return gd.do_gdrive_download(df, headers)
         else:
             abort(404)
     else:
         filename = os.path.join(config.get_book_path(), book.path)
-        if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)):
+        if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)):
             # ToDo: improve error handling
-            log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format))
+            log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format))
 
         if client == "kobo" and book_format == "kepub":
             headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
 
-        response = make_response(send_from_directory(filename, data.name + "." + book_format))
+        if book_format == "kepub" and config.config_kepubifypath and config.config_embed_metadata:
-        # ToDo Check headers parameter
+            filename, download_name = do_kepubify_metadata_replace(book, os.path.join(filename,
-        for element in headers:
+                                                                                      book_name + "." + book_format))
-            response.headers[element[0]] = element[1]
+        elif book_format != "kepub" and config.config_binariesdir and config.config_embed_metadata:
-        log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format)))
+                filename, download_name = do_calibre_export(book.id, book_format)
-        return response
+        else:
+            download_name = book_name
+
+    response = make_response(send_from_directory(filename, download_name + "." + book_format))
+    # ToDo Check headers parameter
+    for element in headers:
+        response.headers[element[0]] = element[1]
+    log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format)))
+    return response
+
+
+def do_kepubify_metadata_replace(book, file_path):
+    custom_columns = (calibre_db.session.query(db.CustomColumns)
+                      .filter(db.CustomColumns.mark_for_delete == 0)
+                      .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions))
+                      .order_by(db.CustomColumns.label).all())
+
+    tree, cf_name = get_content_opf(file_path)
+    package = create_new_metadata_backup(book, custom_columns, current_user.locale, _("Cover"), lang_type=2)
+    content = replace_metadata(tree, package)
+    tmp_dir = get_temp_dir()
+    temp_file_name = str(uuid4())
+    # open zipfile and replace metadata block in content.opf
+    updateEpub(file_path, os.path.join(tmp_dir, temp_file_name + ".kepub"), cf_name, content)
+    return tmp_dir, temp_file_name
+
+
+def do_calibre_export(book_id, book_format, ):
+    try:
+        quotes = [3, 5, 7, 9]
+        tmp_dir = get_temp_dir()
+        calibredb_binarypath = get_calibre_binarypath("calibredb")
+        temp_file_name = str(uuid4())
+        opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir,
+                       '--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name),
+                       str(book_id)]
+        p = process_open(opf_command, quotes)
+        _, err = p.communicate()
+        if err:
+            log.error('Metadata embedder encountered an error: %s', err)
+        return tmp_dir, temp_file_name
+    except OSError as ex:
+        # ToDo real error handling
+        log.error_or_exception(ex)
+
 
 ##################################
 
@@ -984,6 +1042,47 @@ def check_unrar(unrar_location):
         return _('Error executing UnRar')
 
 
+def check_calibre(calibre_location):
+    if not calibre_location:
+        return
+
+    if not os.path.exists(calibre_location):
+        return _('Could not find the specified directory')
+
+    if not os.path.isdir(calibre_location):
+        return _('Please specify a directory, not a file')
+
+    try:
+        supported_binary_paths = [os.path.join(calibre_location, binary)
+                                  for binary in SUPPORTED_CALIBRE_BINARIES.values()]
+        binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths]
+        binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths]
+        if all(binaries_available) and all(binaries_executable):
+            values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)')
+                      for binary_path in supported_binary_paths]
+            if all(values):
+                version = values[0].group(1)
+                log.debug("calibre version %s", version)
+            else:
+                return _('Calibre binaries not viable')
+        else:
+            ret_val = []
+            missing_binaries=[path for path, available in
+                              zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available]
+
+            missing_perms=[path for path, available in
+                           zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available]
+            if missing_binaries:
+                ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)))
+            if missing_perms:
+                ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms)))
+            return ", ".join(ret_val)
+
+    except (OSError, UnicodeDecodeError) as err:
+        log.error_or_exception(err)
+        return _('Error excecuting Calibre')
+
+
 def json_serial(obj):
     """JSON serializer for objects not serializable by default json code"""
 
@@ -1008,7 +1107,7 @@ def tags_filters():
 
 
 # checks if domain is in database (including wildcards)
-# example SELECT * FROM @TABLE WHERE  'abcdefg' LIKE Name;
+# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name;
 # from https://code.luasoftware.com/tutorials/flask/execute-raw-sql-in-flask-sqlalchemy/
 # in all calls the email address is checked for validity
 def check_valid_domain(domain_text):
@@ -1042,6 +1141,17 @@ def get_download_link(book_id, book_format, client):
     abort(404)
 
 
+def get_calibre_binarypath(binary):
+        binariesdir = config.config_binariesdir
+        if binariesdir:
+            try:
+                return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary])
+            except KeyError as ex:
+                log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary])
+                pass
+        return ""
+
+
 def clear_cover_thumbnail_cache(book_id):
     if config.schedule_generate_book_covers:
         WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True)

cps/schedule.py

@@ -21,6 +21,7 @@ import datetime
 from . import config, constants
 from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler
 from .tasks.database import TaskReconnectDatabase
+from .tasks.tempFolder import TaskDeleteTempFolder
 from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache
 from .services.worker import WorkerThread
 from .tasks.metadata_backup import TaskBackupMetadata
@@ -31,6 +32,9 @@ def get_scheduled_tasks(reconnect=True):
     if reconnect:
         tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False])
 
+    # Delete temp folder
+    tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True])
+
     # Generate metadata.opf file for each changed book
     if config.schedule_metadata_backup:
         tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False])
@@ -86,6 +90,8 @@ def register_startup_tasks():
         # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks
         if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration):
             scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False))
+        else:
+            scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]])
 
 
 def should_task_be_running(start, duration):

cps/tasks/convert.py

@@ -19,8 +19,10 @@
 import os
 import re
 from glob import glob
-from shutil import copyfile
+from shutil import copyfile, copyfileobj
 from markupsafe import escape
+from time import time
+from uuid import uuid4
 
 from sqlalchemy.exc import SQLAlchemyError
 from flask_babel import lazy_gettext as N_
@@ -32,13 +34,15 @@ from cps.subproc_wrapper import process_open
 from flask_babel import gettext as _
 from cps.kobo_sync_status import remove_synced_book
 from cps.ub import init_db_thread
+from cps.file_helper import get_temp_dir
 
 from cps.tasks.mail import TaskEmail
-from cps import gdriveutils
+from cps import gdriveutils, helper
-
+from cps.constants import SUPPORTED_CALIBRE_BINARIES
 
 log = logger.create()
 
+current_milli_time = lambda: int(round(time() * 1000))
 
 class TaskConvert(CalibreTask):
     def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None):
@@ -61,24 +65,33 @@ class TaskConvert(CalibreTask):
             data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
             df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
                                                      data.name + "." + self.settings['old_book_format'].lower())
+            df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg")
             if df:
                 datafile = os.path.join(config.get_book_path(),
                                         cur_book.path,
                                         data.name + "." + self.settings['old_book_format'].lower())
+                if df_cover:
+                    datafile_cover = os.path.join(config.get_book_path(),
+                                            cur_book.path, "cover.jpg")
                 if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)):
                     os.makedirs(os.path.join(config.get_book_path(), cur_book.path))
                 df.GetContentFile(datafile)
+                if df_cover:
+                    df_cover.GetContentFile(datafile_cover)
                 worker_db.session.close()
             else:
+                # ToDo Include cover in error handling
                 error_message = _("%(format)s not found on Google Drive: %(fn)s",
                                   format=self.settings['old_book_format'],
                                   fn=data.name + "." + self.settings['old_book_format'].lower())
                 worker_db.session.close()
-                return error_message
+                return self._handleError(self, error_message)
 
         filename = self._convert_ebook_format()
         if config.config_use_google_drive:
             os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
+            if df_cover:
+                os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg"))
 
         if filename:
             if config.config_use_google_drive:
@@ -112,7 +125,7 @@ class TaskConvert(CalibreTask):
 
         # check to see if destination format already exists - or if book is in database
         # if it does - mark the conversion task as complete and return a success
-        # this will allow send to E-Reader workflow to continue to work
+        # this will allow to send to E-Reader workflow to continue to work
         if os.path.isfile(file_path + format_new_ext) or\
                 local_db.get_book_format(self.book_id, self.settings['new_book_format']):
             log.info("Book id %d already converted to %s", book_id, format_new_ext)
@@ -152,7 +165,8 @@ class TaskConvert(CalibreTask):
             if not os.path.exists(config.config_converterpath):
                 self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
                 return
-            check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext)
+            has_cover = local_db.get_book(book_id).has_cover
+            check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover)
 
         if check == 0:
             cur_book = local_db.get_book(book_id)
@@ -194,8 +208,15 @@ class TaskConvert(CalibreTask):
         return
 
     def _convert_kepubify(self, file_path, format_old_ext, format_new_ext):
+        if config.config_embed_metadata:
+            tmp_dir, temp_file_name = helper.do_calibre_export(self.book_id, format_old_ext[1:])
+            filename = os.path.join(tmp_dir, temp_file_name + format_old_ext)
+            temp_file_path = tmp_dir
+        else:
+            filename = file_path + format_old_ext
+            temp_file_path = os.path.dirname(file_path)
         quotes = [1, 3]
-        command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)]
+        command = [config.config_kepubifypath, filename, '-o', temp_file_path, '-i']
         try:
             p = process_open(command, quotes)
         except OSError as e:
@@ -209,13 +230,12 @@ class TaskConvert(CalibreTask):
             if p.poll() is not None:
                 break
 
-        # ToD Handle
         # process returncode
         check = p.returncode
 
         # move file
         if check == 0:
-            converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub"))
+            converted_file = glob(os.path.splitext(filename)[0] + "*.kepub.epub")
             if len(converted_file) == 1:
                 copyfile(converted_file[0], (file_path + format_new_ext))
                 os.unlink(converted_file[0])
@@ -224,16 +244,28 @@ class TaskConvert(CalibreTask):
                             folder=os.path.dirname(file_path))
         return check, None
 
-    def _convert_calibre(self, file_path, format_old_ext, format_new_ext):
+    def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover):
         try:
-            # Linux py2.7 encode as list without quotes no empty element for parameters
+            # path_tmp_opf = self._embed_metadata()
-            # linux py3.x no encode and as list without quotes no empty element for parameters
+            if config.config_embed_metadata:
-            # windows py2.7 encode as string with quotes empty element for parameters is okay
+                quotes = [3, 5]
-            # windows py 3.x no encode and as string with quotes empty element for parameters is okay
+                tmp_dir = get_temp_dir()
-            # separate handling for windows and linux
+                calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"])
-            quotes = [1, 2]
+                opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id),
+                               '--with-library', config.config_calibre_dir]
+                p = process_open(opf_command, quotes)
+                p.wait()
+                path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf")
+                with open(path_tmp_opf, 'w') as fd:
+                    copyfileobj(p.stdout, fd)
+
+            quotes = [1, 2, 4, 6]
             command = [config.config_converterpath, (file_path + format_old_ext),
                        (file_path + format_new_ext)]
+            if config.config_embed_metadata:
+                command.extend(['--from-opf', path_tmp_opf])
+            if has_cover:
+                command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')])
             quotes_index = 3
             if config.config_calibre:
                 parameters = config.config_calibre.split(" ")

cps/tasks/metadata_backup.py

@@ -17,26 +17,13 @@
 #   along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 import os
-from urllib.request import urlopen
 from lxml import etree
 
-
 from cps import config, db, gdriveutils, logger
 from cps.services.worker import CalibreTask
 from flask_babel import lazy_gettext as N_
 
-OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
+from ..epub_helper import create_new_metadata_backup
-PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/"
-
-OPF = "{%s}" % OPF_NAMESPACE
-PURL = "{%s}" % PURL_NAMESPACE
-
-etree.register_namespace("opf", OPF_NAMESPACE)
-etree.register_namespace("dc", PURL_NAMESPACE)
-
-OPF_NS = {None: OPF_NAMESPACE}  # the default namespace (no prefix)
-NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
-
 
 class TaskBackupMetadata(CalibreTask):
 
@@ -101,7 +88,8 @@ class TaskBackupMetadata(CalibreTask):
             self.calibre_db.session.close()
 
     def open_metadata(self, book, custom_columns):
-        package = self.create_new_metadata_backup(book, custom_columns)
+        # package = self.create_new_metadata_backup(book, custom_columns)
+        package = create_new_metadata_backup(book, custom_columns, self.export_language)
         if config.config_use_google_drive:
             if not gdriveutils.is_gdrive_ready():
                 raise Exception('Google Drive is configured but not ready')
@@ -123,7 +111,7 @@ class TaskBackupMetadata(CalibreTask):
             except Exception as ex:
                 raise Exception('Writing Metadata failed with error: {} '.format(ex))
 
-    def create_new_metadata_backup(self, book,  custom_columns):
+    '''def create_new_metadata_backup(self, book,  custom_columns):
         # generate root package element
         package = etree.Element(OPF + "package", nsmap=OPF_NS)
         package.set("unique-identifier", "uuid_id")
@@ -208,7 +196,7 @@ class TaskBackupMetadata(CalibreTask):
         guide = etree.SubElement(package, "guide")
         etree.SubElement(guide, "reference", type="cover", title=self.translated_title, href="cover.jpg")
 
-        return package
+        return package'''
 
     @property
     def name(self):

cps/tasks/tempFolder.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2023 OzzieIsaacs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from urllib.request import urlopen
+
+from flask_babel import lazy_gettext as N_
+
+from cps import logger, file_helper
+from cps.services.worker import CalibreTask
+
+
+class TaskDeleteTempFolder(CalibreTask):
+    def __init__(self, task_message=N_('Delete temp folder contents')):
+        super(TaskDeleteTempFolder, self).__init__(task_message)
+        self.log = logger.create()
+
+    def run(self, worker_thread):
+        try:
+            file_helper.del_temp_dir()
+        except FileNotFoundError:
+            pass
+        except (PermissionError, OSError) as e:
+            self.log.error("Error deleting temp folder: {}".format(e))
+        self._handleSuccess()
+
+    @property
+    def name(self):
+        return "Delete Temp Folder"
+
+    @property
+    def is_cancellable(self):
+        return False

cps/templates/config_edit.html

@@ -103,6 +103,10 @@
       <input type="checkbox" id="config_unicode_filename" name="config_unicode_filename" {% if config.config_unicode_filename %}checked{% endif %}>
       <label for="config_unicode_filename">{{_('Convert non-English characters in title and author while saving to disk')}}</label>
     </div>
+    <div class="form-group">
+      <input type="checkbox" id="config_embed_metadata" name="config_embed_metadata" {% if config.config_embed_metadata %}checked{% endif %}>
+      <label for="config_embed_metadata">{{_('Embed Metadata to Ebook File on Download and Conversion (needs Calibre/Kepubify binaries)')}}</label>
+    </div>
     <div class="form-group">
         <input type="checkbox" id="config_uploading" data-control="upload_settings" name="config_uploading" {% if config.config_uploading %}checked{% endif %}>
         <label for="config_uploading">{{_('Enable Uploads')}} {{_('(Please ensure that users also have upload permissions)')}}</label>
@@ -323,12 +327,12 @@
     </div>
     <div id="collapsefive" class="panel-collapse collapse">
       <div class="panel-body">
-           <label for="config_converterpath">{{_('Path to Calibre E-Book Converter')}}</label>
+           <label for="config_binariesdir">{{_('Path to Calibre Binaries')}}</label>
            <div class="form-group input-group">
-            <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
+           <input type="text" class="form-control" id="config_binariesdir" name="config_binariesdir" value="{% if config.config_binariesdir != None %}{{ config.config_binariesdir }}{% endif %}" autocomplete="off">
-            <span class="input-group-btn">
+           <span class="input-group-btn">
-              <button type="button" data-toggle="modal" id="converter_modal_path" data-link="config_converterpath" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+             <button type="button" data-toggle="modal" id="binaries_modal_path" data-link="config_binariesdir" data-folderonly="true" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
-            </span>
+           </span>
            </div>
            <div class="form-group">
               <label for="config_calibre">{{_('Calibre E-Book Converter Settings')}}</label>

cps/updater.py

@@ -25,13 +25,13 @@ import threading
 import time
 import zipfile
 from io import BytesIO
-from tempfile import gettempdir
-
 import requests
+
 from flask_babel import format_datetime
 from flask_babel import gettext as _
 
 from . import constants, logger  #  config, web_server
+from .file_helper import get_temp_dir
 
 
 log = logger.create()
@@ -85,7 +85,7 @@ class Updater(threading.Thread):
             z = zipfile.ZipFile(BytesIO(r.content))
             self.status = 3
             log.debug('Extracting zipfile')
-            tmp_dir = gettempdir()
+            tmp_dir = get_temp_dir()
             z.extractall(tmp_dir)
             folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1]
             if not os.path.isdir(folder_name):
@@ -566,7 +566,7 @@ class Updater(threading.Thread):
                 try:
                     current_version[2] = int(current_version[2])
                 except ValueError:
-                    current_version[2] = int(current_version[2].split(' ')[0])-1
+                    current_version[2] = int(current_version[2].replace("b", "").split(' ')[0])-1
 
                 # Check if major versions are identical search for newest non-equal commit and update to this one
                 if major_version_update == current_version[0]:

cps/uploader.py

@@ -18,12 +18,12 @@
 
 import os
 import hashlib
-from tempfile import gettempdir
 from flask_babel import gettext as _
 
 from . import logger, comic, isoLanguages
 from .constants import BookMeta
 from .helper import split_authors
+from .file_helper import get_temp_dir
 
 log = logger.create()
 
@@ -249,10 +249,7 @@ def get_magick_version():
 
 
 def upload(uploadfile, rar_excecutable):
-    tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+    tmp_dir = get_temp_dir()
-
-    if not os.path.isdir(tmp_dir):
-        os.mkdir(tmp_dir)
 
     filename = uploadfile.filename
     filename_root, file_extension = os.path.splitext(filename)

setup.cfg

@@ -66,7 +66,7 @@ include = cps/services*
 
 [options.extras_require]
 gdrive = 
-	google-api-python-client>=1.7.11,<2.98.0
+	google-api-python-client>=1.7.11,<2.108.0
 	gevent>20.6.0,<24.0.0
 	greenlet>=0.4.17,<2.1.0
 	httplib2>=0.9.2,<0.23.0
@@ -79,7 +79,7 @@ gdrive =
 	rsa>=3.4.2,<4.10.0
 gmail = 
 	google-auth-oauthlib>=0.4.3,<1.1.0
-	google-api-python-client>=1.7.11,<2.98.0
+	google-api-python-client>=1.7.11,<2.108.0
 goodreads = 
 	goodreads>=0.3.2,<0.4.0
 	python-Levenshtein>=0.12.0,<0.22.0