From b0cbd0a37a85bee137f4ae3d9a538b9a990ef8f1 Mon Sep 17 00:00:00 2001 From: OzzieIsaacs Date: Mon, 30 Jul 2018 20:12:41 +0200 Subject: [PATCH] Added feature to send emails in a background-task --- cps/asyncmail.py | 236 ++++++++++++++++++++++++++++++++++++++ cps/helper.py | 86 +++----------- cps/templates/layout.html | 4 + cps/templates/tasks.html | 48 ++++++++ cps/ub.py | 1 - cps/web.py | 68 +++++++++-- 6 files changed, 367 insertions(+), 76 deletions(-) create mode 100644 cps/asyncmail.py create mode 100644 cps/templates/tasks.html diff --git a/cps/asyncmail.py b/cps/asyncmail.py new file mode 100644 index 00000000..d8ded413 --- /dev/null +++ b/cps/asyncmail.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import smtplib +import threading +from datetime import datetime +import logging +import time +import socket +import sys +from email.generator import Generator +import web +from flask_babel import gettext as _ +# from babel.dates import format_datetime +import re + +try: + from StringIO import StringIO +except ImportError as e: + from io import StringIO + +chunksize = 8192 + +STAT_WAITING = 0 +STAT_FAIL = 1 +STAT_STARTED = 2 +STAT_FINISH_SUCCESS = 3 + + +class email(smtplib.SMTP): + + transferSize = 0 + progress = 0 + + def __init__(self, *args, **kwargs): + smtplib.SMTP.__init__(self, *args, **kwargs) + + def data(self, msg): + self.transferSize = len(msg) + (code, resp) = smtplib.SMTP.data(self, msg) + self.progress = 0 + return (code, resp) + + def send(self, str): + """Send `str' to the server.""" + if self.debuglevel > 0: + print>> sys.stderr, 'send:', repr(str) + if hasattr(self, 'sock') and self.sock: + try: + if self.transferSize: + lock=threading.Lock() + lock.acquire() + self.transferSize = len(str) + lock.release() + for i in range(0, self.transferSize, chunksize): + self.sock.send(str[i:i+chunksize]) + lock.acquire() + self.progress = i + lock.release() + else: + self.sock.sendall(str) + except socket.error: + self.close() + raise smtplib.SMTPServerDisconnected('Server not connected') + else: + raise smtplib.SMTPServerDisconnected('please run connect() first') + + def getTransferStatus(self): + if self.transferSize: + lock2 = threading.Lock() + lock2.acquire() + value = round(float(self.progress) / float(self.transferSize),2)*100 + lock2.release() + return str(value) + ' %' + else: + return "100 %" + +class email_SSL(email): + + def __init__(self, *args, **kwargs): + smtplib.SMTP_SSL.__init__(self, *args, **kwargs) + + +class EMailThread(threading.Thread): + + def __init__(self): + threading.Thread.__init__(self) + self.status = 0 + self.current = 0 + self.last = 0 + self.queue=list() + self.UIqueue = list() + self.asyncSMTP=None + + def run(self): + while 1: + doLock = threading.Lock() + doLock.acquire() + if self.current != self.last: + doLock.release() + self.send_raw_email() + self.current += 1 + + time.sleep(1) + + def get_send_status(self): + if self.asyncSMTP: + return self.asyncSMTP.getTransferStatus() + else: + return "0 %" + + def delete_completed_tasks(self): + # muss gelockt werden + for index, task in reversed(list(enumerate(self.UIqueue))): + if task['progress'] == "100 %": + # delete tasks + self.queue.pop(index) + self.UIqueue.pop(index) + # if we are deleting entries before the current index, adjust the index + # if self.current >= index: + self.current -= 1 + self.last = len(self.queue) + + def get_taskstatus(self): + if self.current < len(self.queue): + if self.queue[self.current]['status'] == STAT_STARTED: + self.UIqueue[self.current]['progress'] = self.get_send_status() + self.UIqueue[self.current]['runtime'] = self._formatRuntime( + datetime.now() - self.queue[self.current]['starttime']) + + return self.UIqueue + + def add_email(self, data, settings, recipient, user_name): + # if more than 50 entries in the list, clean the list + addLock = threading.Lock() + addLock.acquire() + if self.last >= 3: + self.delete_completed_tasks() + # progress, runtime, and status = 0 + self.queue.append({'data':data, 'settings':settings, 'recipent':recipient, 'starttime': 0, + 'status': STAT_WAITING}) + self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': 'E-Mail', + 'runtime': '0 s', 'status': _('Waiting') }) + # access issue + self.last=len(self.queue) + addLock.release() + + def send_raw_email(self): + obj=self.queue[self.current] + # settings = ub.get_mail_settings() + + obj['data']['From'] = obj['settings']["mail_from"] + obj['data']['To'] = obj['recipent'] + + use_ssl = int(obj['settings'].get('mail_use_ssl', 0)) + + # convert MIME message to string + fp = StringIO() + gen = Generator(fp, mangle_from_=False) + gen.flatten(obj['data']) + obj['data'] = fp.getvalue() + + # send email + try: + timeout = 600 # set timeout to 5mins + + org_stderr = sys.stderr + #org_stderr2 = smtplib.stderr + sys.stderr = StderrLogger() + #smtplib.stderr = StderrLogger() + + self.queue[self.current]['status'] = STAT_STARTED + self.UIqueue[self.current]['status'] = _('Started') + self.queue[self.current]['starttime'] = datetime.now() + self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] + + + if use_ssl == 2: + self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout) + else: + self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout) + + # link to logginglevel + if web.ub.config.config_log_level != logging.DEBUG: + self.asyncSMTP.set_debuglevel(0) + else: + self.asyncSMTP.set_debuglevel(1) + if use_ssl == 1: + self.asyncSMTP.starttls() + if obj['settings']["mail_password"]: + self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"])) + self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], obj['data']) + self.asyncSMTP.quit() + self.queue[self.current]['status'] = STAT_FINISH_SUCCESS + self.UIqueue[self.current]['status'] = _('Finished') + self.UIqueue[self.current]['progress'] = "100 %" + self.UIqueue[self.current]['runtime'] = self._formatRuntime( + datetime.now() - self.queue[self.current]['starttime']) + + sys.stderr = org_stderr + #smtplib.stderr = org_stderr2 + + except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e: + self.queue[self.current]['status'] = STAT_FAIL + self.UIqueue[self.current]['status'] = _('Failed') + self.UIqueue[self.current]['progress'] = "100 %" + self.UIqueue[self.current]['runtime'] = self._formatRuntime( + datetime.now() - self.queue[self.current]['starttime']) + web.app.logger.error(e) + return None + + def _formatRuntime(self, runtime): + val = re.split('\:|\.', str(runtime))[0:3] + erg = list() + for v in val: + if int(v) > 0: + erg.append(v) + retVal = (':'.join(erg)).lstrip('0') + ' s' + if retVal == ' s': + retVal = '0 s' + return retVal + +class StderrLogger(object): + + buffer = '' + + def __init__(self): + self.logger = web.app.logger + + def write(self, message): + if message == '\n': + self.logger.debug(self.buffer) + print self.buffer + self.buffer = '' + else: + self.buffer += message diff --git a/cps/helper.py b/cps/helper.py index 6c5c81c4..0c5cc2d2 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -5,9 +5,7 @@ import db import ub from flask import current_app as app import logging -import smtplib from tempfile import gettempdir -import socket import sys import os import traceback @@ -15,6 +13,7 @@ import re import unicodedata from io import BytesIO import converter +import asyncmail try: from StringIO import StringIO @@ -28,11 +27,9 @@ except ImportError as e: from email.mime.text import MIMEText from email import encoders -from email.generator import Generator from email.utils import formatdate from email.utils import make_msgid from flask_babel import gettext as _ -import subprocess import threading import shutil import requests @@ -52,11 +49,22 @@ except ImportError: # Global variables updater_thread = None +global_eMailThread = asyncmail.EMailThread() +global_eMailThread.start() RET_SUCCESS = 1 RET_FAIL = 0 +def update_download(book_id, user_id): + check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id == + book_id).first() + + if not check: + new_download = ub.Downloads(user_id=user_id, book_id=book_id) + ub.session.add(new_download) + ub.session.commit() + def make_mobi(book_id, calibrepath): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first() @@ -73,74 +81,16 @@ def make_mobi(book_id, calibrepath): return error_message, RET_FAIL -class StderrLogger(object): - - buffer = '' - - def __init__(self): - self.logger = logging.getLogger('cps.web') - - def write(self, message): - if message == '\n': - self.logger.debug(self.buffer) - self.buffer = '' - else: - self.buffer += message - - -def send_raw_email(kindle_mail, msg): - settings = ub.get_mail_settings() - - msg['From'] = settings["mail_from"] - msg['To'] = kindle_mail - - use_ssl = int(settings.get('mail_use_ssl', 0)) - - # convert MIME message to string - fp = StringIO() - gen = Generator(fp, mangle_from_=False) - gen.flatten(msg) - msg = fp.getvalue() - - # send email - try: - timeout = 600 # set timeout to 5mins - - org_stderr = sys.stderr - sys.stderr = StderrLogger() - - if use_ssl == 2: - mailserver = smtplib.SMTP_SSL(settings["mail_server"], settings["mail_port"], timeout) - else: - mailserver = smtplib.SMTP(settings["mail_server"], settings["mail_port"], timeout) - mailserver.set_debuglevel(1) - - if use_ssl == 1: - mailserver.starttls() - - if settings["mail_password"]: - mailserver.login(str(settings["mail_login"]), str(settings["mail_password"])) - mailserver.sendmail(settings["mail_from"], kindle_mail, msg) - mailserver.quit() - - smtplib.stderr = org_stderr - - except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as ex: - app.logger.error(traceback.print_exc()) - return _("Failed to send mail: %s" % str(ex)) - - return None - - -def send_test_mail(kindle_mail): +def send_test_mail(kindle_mail, user_name): msg = MIMEMultipart() msg['Subject'] = _(u'Calibre-web test email') text = _(u'This email has been sent via calibre web.') msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8')) - return send_raw_email(kindle_mail, msg) + global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name) + return # send_raw_email(kindle_mail, msg) -def send_mail(book_id, kindle_mail, calibrepath): +def send_mail(book_id, kindle_mail, calibrepath, user_id): """Send email with attachments""" # create MIME message msg = MIMEMultipart() @@ -179,8 +129,8 @@ def send_mail(book_id, kindle_mail, calibrepath): msg.attach(get_attachment(formats['pdf'])) else: return _("Could not find any formats suitable for sending by email") - - return send_raw_email(kindle_mail, msg) + global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id) + return None # send_raw_email(kindle_mail, msg) def get_attachment(file_path): diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 01a1fa4a..fc589cc7 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -67,6 +67,9 @@ {% endif %} {% endif %} + {% if g.user.role_admin() %} +
    • + {% endif %} {% if g.user.role_admin() %}
    • {% endif %} @@ -222,6 +225,7 @@ {% block modal %}{% endblock %} + diff --git a/cps/templates/tasks.html b/cps/templates/tasks.html new file mode 100644 index 00000000..a7d622c6 --- /dev/null +++ b/cps/templates/tasks.html @@ -0,0 +1,48 @@ +{% extends "layout.html" %} +{% block header %} + +{% endblock %} +{% block body %} +
    +

    {{_('Tasks list')}}

    + + + + {% if g.user.role_admin() %} + + {% endif %} + + + + + + + +
    {{_('User')}}{{_('Task')}}{{_('Status')}}{{_('Progress')}}{{_('Runtime')}}{{_('Starttime')}}
    + +
    +{% endblock %} +{% block js %} + + + +{% endblock %} diff --git a/cps/ub.py b/cps/ub.py index bf28aaaa..09c765da 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -10,7 +10,6 @@ import sys import os import logging from werkzeug.security import generate_password_hash -from flask_babel import gettext as _ import json import datetime from binascii import hexlify diff --git a/cps/web.py b/cps/web.py index 4a93b343..0e304b56 100755 --- a/cps/web.py +++ b/cps/web.py @@ -912,13 +912,41 @@ def get_metadata_calibre_companion(uuid): else: return "" +@app.route("/ajax/emailstat") +@login_required +def get_email_status_json(): + answer=list() + tasks=helper.global_eMailThread.get_taskstatus() + if not current_user.role_admin(): + for task in tasks: + if task['user'] == current_user.nickname: + if task['formStarttime']: + task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale()) + task['formStarttime'] = "" + else: + if 'starttime' not in task: + task['starttime'] = "" + answer.append(task) + else: + for task in tasks: + if task['formStarttime']: + task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale()) + task['formStarttime'] = "" + else: + if 'starttime' not in task: + task['starttime'] = "" + answer = tasks + js=json.dumps(answer) + response = make_response(js) + response.headers["Content-Type"] = "application/json; charset=utf-8" + return response + @app.route("/get_authors_json", methods=['GET', 'POST']) @login_required_if_no_ano def get_authors_json(): if request.method == "GET": query = request.args.get('q') - # entries = db.session.execute("select name from authors where name like '%" + query + "%'") entries = db.session.query(db.Authors).filter(db.Authors.name.ilike("%" + query + "%")).all() json_dumps = json.dumps([dict(name=r.name) for r in entries]) return json_dumps @@ -929,10 +957,7 @@ def get_authors_json(): def get_tags_json(): if request.method == "GET": query = request.args.get('q') - # entries = db.session.execute("select name from tags where name like '%" + query + "%'") entries = db.session.query(db.Tags).filter(db.Tags.name.ilike("%" + query + "%")).all() - # for x in entries: - # alfa = dict(name=x.name) json_dumps = json.dumps([dict(name=r.name) for r in entries]) return json_dumps @@ -1421,6 +1446,35 @@ def bookmark(book_id, book_format): ub.session.commit() return "", 201 +@app.route("/tasks") +@login_required +def get_tasks_status(): + # if current user admin, show all email, otherwise only own emails + answer=list() + tasks=helper.global_eMailThread.get_taskstatus() + if not current_user.role_admin(): + for task in tasks: + if task['user'] == current_user.nickname: + if task['formStarttime']: + task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale()) + task['formStarttime'] = "" + else: + if 'starttime' not in task: + task['starttime'] = "" + answer.append(task) + else: + for task in tasks: + if task['formStarttime']: + task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale()) + task['formStarttime'] = "" + else: + if 'starttime' not in task: + task['starttime'] = "" + answer = tasks + + # foreach row format row + return render_title_template('tasks.html', entries=answer, title=_(u"Tasks")) + @app.route("/admin") @login_required @@ -2147,9 +2201,9 @@ def send_to_kindle(book_id): if settings.get("mail_server", "mail.example.com") == "mail.example.com": flash(_(u"Please configure the SMTP mail settings first..."), category="error") elif current_user.kindle_mail: - result = helper.send_mail(book_id, current_user.kindle_mail, config.config_calibre_dir) + result = helper.send_mail(book_id, current_user.kindle_mail, config.config_calibre_dir, current_user.nickname) if result is None: - flash(_(u"Book successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail), + flash(_(u"Book successfully queued for sending to %(kindlemail)s", kindlemail=current_user.kindle_mail), category="success") ub.update_download(book_id, int(current_user.id)) else: @@ -2851,7 +2905,7 @@ def edit_mailsettings(): flash(e, category="error") if "test" in to_save and to_save["test"]: if current_user.kindle_mail: - result = helper.send_test_mail(current_user.kindle_mail) + result = helper.send_test_mail(current_user.kindle_mail, current_user.nickname) if result is None: flash(_(u"Test E-Mail successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail), category="success")