cleaner worker api
the worker thread now stops on its own
This commit is contained in:
Daniel Pavel
parent
a836df9a5a
commit
63634961d4
|
|
@ -34,8 +34,9 @@ from flask_login import LoginManager
|
|||
from flask_babel import Babel
|
||||
from flask_principal import Principal
|
||||
|
||||
from . import logger, cache_buster, cli, config_sql, ub
|
||||
from . import logger, cache_buster, cli, config_sql, ub, db, services
|
||||
from .reverseproxy import ReverseProxied
|
||||
from .server import WebServer
|
||||
|
||||
|
||||
mimetypes.init()
|
||||
|
|
@ -66,14 +67,8 @@ lm.anonymous_user = ub.Anonymous
|
|||
ub.init_db(cli.settingspath)
|
||||
# pylint: disable=no-member
|
||||
config = config_sql.load_configuration(ub.session)
|
||||
from . import db, services
|
||||
|
||||
searched_ids = {}
|
||||
|
||||
from .worker import WorkerThread
|
||||
global_WorkerThread = WorkerThread()
|
||||
|
||||
from .server import WebServer
|
||||
web_server = WebServer()
|
||||
|
||||
babel = Babel()
|
||||
|
|
@ -109,7 +104,6 @@ def create_app():
|
|||
if services.goodreads:
|
||||
services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads)
|
||||
|
||||
global_WorkerThread.start()
|
||||
return app
|
||||
|
||||
@babel.localeselector
|
||||
|
|
|
|||
|
|
@ -30,12 +30,12 @@ from uuid import uuid4
|
|||
|
||||
from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
|
||||
from flask_babel import gettext as _
|
||||
from flask_login import current_user
|
||||
from flask_login import current_user, login_required
|
||||
|
||||
from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
|
||||
from . import config, get_locale, db, ub, global_WorkerThread
|
||||
from . import config, get_locale, db, ub, worker
|
||||
from .helper import order_authors, common_filters
|
||||
from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required
|
||||
from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
|
||||
|
||||
|
||||
editbook = Blueprint('editbook', __name__)
|
||||
|
|
@ -358,7 +358,7 @@ def upload_single_file(request, book, book_id):
|
|||
|
||||
# Queue uploader info
|
||||
uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
|
||||
global_WorkerThread.add_upload(current_user.nickname,
|
||||
worker.add_upload(current_user.nickname,
|
||||
"<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
|
||||
|
||||
|
||||
|
|
@ -667,7 +667,7 @@ def upload():
|
|||
if error:
|
||||
flash(error, category="error")
|
||||
uploadText=_(u"File %(file)s uploaded", file=book.title)
|
||||
global_WorkerThread.add_upload(current_user.nickname,
|
||||
worker.add_upload(current_user.nickname,
|
||||
"<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
|
||||
|
||||
# create data for displaying display Full language name instead of iso639.part3language
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ try:
|
|||
except ImportError:
|
||||
use_PIL = False
|
||||
|
||||
from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLanguages
|
||||
from . import logger, config, get_locale, db, ub, isoLanguages, worker
|
||||
from . import gdriveutils as gd
|
||||
from .constants import STATIC_DIR as _STATIC_DIR
|
||||
from .pagination import Pagination
|
||||
|
|
@ -112,7 +112,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
|
|||
text = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
|
||||
settings['old_book_format'] = old_book_format
|
||||
settings['new_book_format'] = new_book_format
|
||||
global_WorkerThread.add_convert(file_path, book.id, user_id, text, settings, kindle_mail)
|
||||
worker.add_convert(file_path, book.id, user_id, text, settings, kindle_mail)
|
||||
return None
|
||||
else:
|
||||
error_message = _(u"%(format)s not found: %(fn)s",
|
||||
|
|
@ -121,9 +121,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
|
|||
|
||||
|
||||
def send_test_mail(kindle_mail, user_name):
|
||||
global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(),
|
||||
kindle_mail, user_name, _(u"Test e-mail"),
|
||||
_(u'This e-mail has been sent via Calibre-Web.'))
|
||||
worker.add_email(_(u'Calibre-Web test e-mail'), None, None,
|
||||
config.get_mail_settings(), kindle_mail, user_name,
|
||||
_(u"Test e-mail"), _(u'This e-mail has been sent via Calibre-Web.'))
|
||||
return
|
||||
|
||||
|
||||
|
|
@ -138,8 +138,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
|
|||
text += "Don't forget to change your password after first login.\r\n"
|
||||
text += "Sincerely\r\n\r\n"
|
||||
text += "Your Calibre-Web team"
|
||||
global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(),
|
||||
e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text)
|
||||
worker.add_email(_(u'Get Started with Calibre-Web'), None, None,
|
||||
config.get_mail_settings(), e_mail, None,
|
||||
_(u"Registration e-mail for user: %(name)s", name=user_name), text)
|
||||
return
|
||||
|
||||
|
||||
|
|
@ -207,15 +208,15 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
|
|||
if convert:
|
||||
# returns None if success, otherwise errormessage
|
||||
return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail)
|
||||
else:
|
||||
for entry in iter(book.data):
|
||||
if entry.format.upper() == book_format.upper():
|
||||
result = entry.name + '.' + book_format.lower()
|
||||
global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(),
|
||||
kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title),
|
||||
_(u'This e-mail has been sent via Calibre-Web.'))
|
||||
return
|
||||
return _(u"The requested file could not be read. Maybe wrong permissions?")
|
||||
|
||||
for entry in iter(book.data):
|
||||
if entry.format.upper() == book_format.upper():
|
||||
converted_file_name = entry.name + '.' + book_format.lower()
|
||||
worker.add_email(_(u"Send to Kindle"), book.path, converted_file_name,
|
||||
config.get_mail_settings(), kindle_mail, user_id,
|
||||
_(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.'))
|
||||
return
|
||||
return _(u"The requested file could not be read. Maybe wrong permissions?")
|
||||
|
||||
|
||||
def get_valid_filename(value, replace_whitespace=True):
|
||||
|
|
@ -232,7 +233,7 @@ def get_valid_filename(value, replace_whitespace=True):
|
|||
value = value.replace(u'§', u'SS')
|
||||
value = value.replace(u'ß', u'ss')
|
||||
value = unicodedata.normalize('NFKD', value)
|
||||
re_slugify = re.compile('[\W\s-]', re.UNICODE)
|
||||
re_slugify = re.compile(r'[\W\s-]', re.UNICODE)
|
||||
if isinstance(value, str): # Python3 str, Python2 unicode
|
||||
value = re_slugify.sub('', value).strip()
|
||||
else:
|
||||
|
|
@ -254,7 +255,7 @@ def get_valid_filename(value, replace_whitespace=True):
|
|||
def get_sorted_author(value):
|
||||
try:
|
||||
if ',' not in value:
|
||||
regexes = ["^(JR|SR)\.?$", "^I{1,3}\.?$", "^IV\.?$"]
|
||||
regexes = [r"^(JR|SR)\.?$", r"^I{1,3}\.?$", r"^IV\.?$"]
|
||||
combined = "(" + ")|(".join(regexes) + ")"
|
||||
value = value.split(" ")
|
||||
if re.match(combined, value[-1].upper()):
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ except ImportError:
|
|||
VERSION = {'Tornado': 'v' + _version}
|
||||
_GEVENT = False
|
||||
|
||||
from . import logger, global_WorkerThread
|
||||
from . import logger
|
||||
|
||||
|
||||
log = logger.create()
|
||||
|
|
@ -179,7 +179,6 @@ class WebServer:
|
|||
return False
|
||||
finally:
|
||||
self.wsgiserver = None
|
||||
global_WorkerThread.stop()
|
||||
|
||||
if not self.restart:
|
||||
log.info("Performing shutdown of Calibre-Web")
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ from werkzeug.exceptions import default_exceptions
|
|||
from werkzeug.datastructures import Headers
|
||||
from werkzeug.security import generate_password_hash, check_password_hash
|
||||
|
||||
from . import constants, logger, isoLanguages, services
|
||||
from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app
|
||||
from . import constants, logger, isoLanguages, services, worker
|
||||
from . import searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app
|
||||
from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
|
||||
from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \
|
||||
order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \
|
||||
|
|
@ -245,7 +245,7 @@ def before_request():
|
|||
@web.route("/ajax/emailstat")
|
||||
@login_required
|
||||
def get_email_status_json():
|
||||
tasks = global_WorkerThread.get_taskstatus()
|
||||
tasks = worker.get_taskstatus()
|
||||
answer = render_task_status(tasks)
|
||||
js = json.dumps(answer, default=json_serial)
|
||||
response = make_response(js)
|
||||
|
|
@ -760,7 +760,7 @@ def category_list():
|
|||
@login_required
|
||||
def get_tasks_status():
|
||||
# if current user admin, show all email, otherwise only own emails
|
||||
tasks = global_WorkerThread.get_taskstatus()
|
||||
tasks = worker.get_taskstatus()
|
||||
answer = render_task_status(tasks)
|
||||
return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks")
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import smtplib
|
|||
import socket
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from StringIO import StringIO
|
||||
|
|
@ -66,6 +66,13 @@ RET_FAIL = 0
|
|||
RET_SUCCESS = 1
|
||||
|
||||
|
||||
def _get_main_thread():
|
||||
for t in threading.enumerate():
|
||||
if t.__class__.__name__ == '_MainThread':
|
||||
return t
|
||||
raise Exception("main thread not found?!")
|
||||
|
||||
|
||||
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
|
||||
# it in MIME Base64 encoded to
|
||||
def get_attachment(bookpath, filename):
|
||||
|
|
@ -173,19 +180,19 @@ class email_SSL(emailbase, smtplib.SMTP_SSL):
|
|||
class WorkerThread(threading.Thread):
|
||||
|
||||
def __init__(self):
|
||||
self._stopevent = threading.Event()
|
||||
threading.Thread.__init__(self)
|
||||
self.status = 0
|
||||
self.current = 0
|
||||
self.last = 0
|
||||
self.queue = list()
|
||||
self.UIqueue = list()
|
||||
self.asyncSMTP=None
|
||||
self.asyncSMTP = None
|
||||
self.id = 0
|
||||
|
||||
# Main thread loop starting the different tasks
|
||||
def run(self):
|
||||
while not self._stopevent.isSet():
|
||||
main_thread = _get_main_thread()
|
||||
while main_thread.is_alive():
|
||||
doLock = threading.Lock()
|
||||
doLock.acquire()
|
||||
if self.current != self.last:
|
||||
|
|
@ -200,10 +207,8 @@ class WorkerThread(threading.Thread):
|
|||
self.current += 1
|
||||
else:
|
||||
doLock.release()
|
||||
time.sleep(1)
|
||||
|
||||
def stop(self):
|
||||
self._stopevent.set()
|
||||
if main_thread.is_alive():
|
||||
time.sleep(1)
|
||||
|
||||
def get_send_status(self):
|
||||
if self.asyncSMTP:
|
||||
|
|
@ -317,7 +322,7 @@ class WorkerThread(threading.Thread):
|
|||
nextline = p.communicate()[0]
|
||||
# Format of error message (kindlegen translates its output texts):
|
||||
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
|
||||
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
|
||||
conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
|
||||
# If error occoures, store error message for logfile
|
||||
if conv_error:
|
||||
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
|
||||
|
|
@ -332,7 +337,7 @@ class WorkerThread(threading.Thread):
|
|||
nextline = nextline.decode('utf-8')
|
||||
log.debug(nextline.strip('\r\n'))
|
||||
# parse progress string from calibre-converter
|
||||
progress = re.search("(\d+)%\s.*", nextline)
|
||||
progress = re.search(r"(\d+)%\s.*", nextline)
|
||||
if progress:
|
||||
self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
|
||||
|
||||
|
|
@ -511,3 +516,23 @@ class WorkerThread(threading.Thread):
|
|||
self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS
|
||||
self.UIqueue[self.current]['progress'] = "100 %"
|
||||
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
|
||||
|
||||
|
||||
_worker = WorkerThread()
|
||||
_worker.start()
|
||||
|
||||
|
||||
def get_taskstatus():
|
||||
return _worker.get_taskstatus()
|
||||
|
||||
|
||||
def add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text):
|
||||
return _worker.add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text)
|
||||
|
||||
|
||||
def add_upload(user_name, taskMessage):
|
||||
return _worker.add_upload(user_name, taskMessage)
|
||||
|
||||
|
||||
def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
|
||||
return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user