NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Converting ebooks in background

Browse Source 
additional sorting of tasklist according to date and runtime
codecosmetics
This commit is contained in:
Ozzie Isaacs 2018-08-12 09:29:57 +02:00
parent 11b798a01c
commit 7be328c535
16 changed files with 825 additions and 737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
cps/asyncmail.py
View File

@ -1,241 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import smtplib
import threading
from datetime import datetime
import logging
import time
import socket
import sys
from email.generator import Generator
import web
from flask_babel import gettext as _
import re
try:
from StringIO import StringIO
except ImportError as e:
from io import StringIO
chunksize = 8192
STAT_WAITING = 0
STAT_FAIL = 1
STAT_STARTED = 2
STAT_FINISH_SUCCESS = 3
class email(smtplib.SMTP):
transferSize = 0
progress = 0
def __init__(self, *args, **kwargs):
smtplib.SMTP.__init__(self, *args, **kwargs)
def data(self, msg):
self.transferSize = len(msg)
(code, resp) = smtplib.SMTP.data(self, msg)
self.progress = 0
return (code, resp)
def send(self, str):
"""Send `str' to the server."""
if self.debuglevel > 0:
print('send:', repr(str), file=sys.stderr)
if hasattr(self, 'sock') and self.sock:
try:
if self.transferSize:
lock=threading.Lock()
lock.acquire()
self.transferSize = len(str)
lock.release()
for i in range(0, self.transferSize, chunksize):
self.sock.send(str[i:i+chunksize])
lock.acquire()
self.progress = i
lock.release()
else:
self.sock.sendall(str)
except socket.error:
self.close()
raise smtplib.SMTPServerDisconnected('Server not connected')
else:
raise smtplib.SMTPServerDisconnected('please run connect() first')
def getTransferStatus(self):
if self.transferSize:
lock2 = threading.Lock()
lock2.acquire()
value = round(float(self.progress) / float(self.transferSize),2)*100
lock2.release()
return str(value) + ' %'
else:
return "100 %"
class email_SSL(email):
def __init__(self, *args, **kwargs):
smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
class EMailThread(threading.Thread):
def __init__(self):
self._stopevent = threading.Event()
threading.Thread.__init__(self)
self.status = 0
self.current = 0
self.last = 0
self.queue=list()
self.UIqueue = list()
self.asyncSMTP=None
def run(self):
while not self._stopevent.isSet():
doLock = threading.Lock()
doLock.acquire()
if self.current != self.last:
doLock.release()
self.send_raw_email()
self.current += 1
else:
doLock.release()
time.sleep(1)
def stop(self):
self._stopevent.set()
def get_send_status(self):
if self.asyncSMTP:
return self.asyncSMTP.getTransferStatus()
else:
return "0 %"
def delete_completed_tasks(self):
# muss gelockt werden
for index, task in reversed(list(enumerate(self.UIqueue))):
if task['progress'] == "100 %":
# delete tasks
self.queue.pop(index)
self.UIqueue.pop(index)
# if we are deleting entries before the current index, adjust the index
# if self.current >= index:
self.current -= 1
self.last = len(self.queue)
def get_taskstatus(self):
if self.current < len(self.queue):
if self.queue[self.current]['status'] == STAT_STARTED:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def add_email(self, data, settings, recipient, user_name, type):
# if more than 50 entries in the list, clean the list
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.queue.append({'data':data, 'settings':settings, 'recipent':recipient, 'starttime': 0,
'status': STAT_WAITING})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': type,
'runtime': '0 s', 'status': _('Waiting') })
# access issue
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
obj=self.queue[self.current]
# settings = ub.get_mail_settings()
obj['data']['From'] = obj['settings']["mail_from"]
obj['data']['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(obj['data'])
obj['data'] = fp.getvalue()
# send email
try:
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
#org_stderr2 = smtplib.stderr
sys.stderr = StderrLogger()
#smtplib.stderr = StderrLogger()
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
# link to logginglevel
if web.ub.config.config_log_level != logging.DEBUG:
self.asyncSMTP.set_debuglevel(0)
else:
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], obj['data'])
self.asyncSMTP.quit()
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
sys.stderr = org_stderr
#smtplib.stderr = org_stderr2
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
web.app.logger.error(e)
return None
def _formatRuntime(self, runtime):
val = re.split('\:|\.', str(runtime))[0:3]
erg = list()
for v in val:
if int(v) > 0:
erg.append(v)
retVal = (':'.join(erg)).lstrip('0') + ' s'
if retVal == ' s':
retVal = '0 s'
return retVal
class StderrLogger(object):
buffer = ''
def __init__(self):
self.logger = web.app.logger
def write(self, message):
if message == '\n':
self.logger.debug(self.buffer)
print(self.buffer)
self.buffer = ''
else:
self.buffer += message

104
cps/converter.py
View File

@ -1,19 +1,12 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os import os
import sys
import subprocess import subprocess
import ub import ub
import db
import re import re
import web
from flask_babel import gettext as _ from flask_babel import gettext as _
RET_FAIL = 0
RET_SUCCESS = 1
def versionKindle(): def versionKindle():
versions = _(u'not installed') versions = _(u'not installed')
if os.path.exists(ub.config.config_converterpath): if os.path.exists(ub.config.config_converterpath):
@ -46,97 +39,6 @@ def versionCalibre():
return {'Calibre converter' : versions} return {'Calibre converter' : versions}
def convert_kindlegen(file_path, book):
error_message = None
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
try:
p = subprocess.Popen((ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
web.app.logger.info("convert_kindlegen: " + error_message)
web.app.logger.info(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
if ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
return error_message, RET_FAIL
def convert_calibre(file_path, book):
error_message = None
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found", converterpath=ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
try:
command = ("\""+ub.config.config_converterpath + "\" \"" + file_path + u".epub\" \""
+ file_path + u".mobi\" " + ub.config.config_calibre).encode(sys.getfilesystemencoding())
p = subprocess.Popen(command,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0 :
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
if ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
return error_message, RET_FAIL
def versioncheck(): def versioncheck():
if ub.config.config_ebookconverter == 1: if ub.config.config_ebookconverter == 1:
return versionKindle() return versionKindle()
@ -145,9 +47,3 @@ def versioncheck():
else: else:
return {'ebook_converter':''} return {'ebook_converter':''}
def convert_mobi(file_path, book):
if ub.config.config_ebookconverter == 2:
return convert_calibre(file_path, book)
else:
return convert_kindlegen(file_path, book)

9
cps/db.py
View File

@ -321,7 +321,7 @@ def setup_db():
try: try:
if not os.path.exists(dbpath): if not os.path.exists(dbpath):
raise raise
engine = create_engine('sqlite:///' + dbpath, echo=False, isolation_level="SERIALIZABLE") engine = create_engine('sqlite:///' + dbpath, echo=False, isolation_level="SERIALIZABLE", connect_args={'check_same_thread': False})
conn = engine.connect() conn = engine.connect()
except Exception: except Exception:
content = ub.session.query(ub.Settings).first() content = ub.session.query(ub.Settings).first()
@ -381,8 +381,9 @@ def setup_db():
secondary=books_custom_column_links[cc_id[0]], secondary=books_custom_column_links[cc_id[0]],
backref='books')) backref='books'))
# Base.metadata.create_all(engine)
Session = sessionmaker() Session = scoped_session(sessionmaker(autocommit=False,
Session.configure(bind=engine) autoflush=False,
bind=engine))
session = Session() session = Session()
return True return True

20
cps/gdriveutils.py
View File

@ -5,6 +5,7 @@ try:
from apiclient import errors from apiclient import errors
except ImportError: except ImportError:
pass pass
import os import os
from ub import config from ub import config
import cli import cli
@ -179,9 +180,8 @@ def getEbooksFolderId(drive=None):
gDriveId = GdriveId() gDriveId = GdriveId()
try: try:
gDriveId.gdrive_id = getEbooksFolder(drive)['id'] gDriveId.gdrive_id = getEbooksFolder(drive)['id']
except: except Exception:
pass web.app.logger.error('Error gDrive, root ID not found')
# ToDo Path not exisiting
gDriveId.path = '/' gDriveId.path = '/'
session.merge(gDriveId) session.merge(gDriveId)
session.commit() session.commit()
@ -282,19 +282,6 @@ def moveGdriveFolderRemote(origin_file, target_folder):
# drive.auth.service.files().delete(fileId=previous_parents).execute() # drive.auth.service.files().delete(fileId=previous_parents).execute()
#def downloadFile(path, filename, output):
# f = getFileFromEbooksFolder(path, filename)
# return f.GetContentFile(output)
# ToDo: Check purpose Parameter f ??, purpose of function ?
def backupCalibreDbAndOptionalDownload(drive):
drive = getDrive(drive)
metaDataFile = "'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId()
fileList = drive.ListFile({'q': metaDataFile}).GetList()
#databaseFile = fileList[0]
#if f:
# databaseFile.GetContentFile(f)
def copyToDrive(drive, uploadFile, createRoot, replaceFiles, def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
ignoreFiles=None, ignoreFiles=None,
@ -447,7 +434,6 @@ def deleteDatabaseOnChange():
session.commit() session.commit()
def updateGdriveCalibreFromLocal(): def updateGdriveCalibreFromLocal():
# backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive)
copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True) copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True)
for x in os.listdir(config.config_calibre_dir): for x in os.listdir(config.config_calibre_dir):
if os.path.isdir(os.path.join(config.config_calibre_dir, x)): if os.path.isdir(os.path.join(config.config_calibre_dir, x)):

142
cps/helper.py
View File

@ -8,28 +8,12 @@ import logging
from tempfile import gettempdir from tempfile import gettempdir
import sys import sys
import os import os
import traceback
import re import re
import unicodedata import unicodedata
from io import BytesIO from io import BytesIO
import converter import worker
import asyncmail
import time import time
try:
from StringIO import StringIO
from email.MIMEBase import MIMEBase
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
except ImportError as e:
from io import StringIO
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email import encoders
from email.utils import formatdate
from email.utils import make_msgid
from flask import send_from_directory, make_response, redirect, abort from flask import send_from_directory, make_response, redirect, abort
from flask_babel import gettext as _ from flask_babel import gettext as _
import threading import threading
@ -51,29 +35,25 @@ except ImportError:
# Global variables # Global variables
updater_thread = None updater_thread = None
global_eMailThread = asyncmail.EMailThread() global_WorkerThread = worker.WorkerThread()
global_eMailThread.start() global_WorkerThread.start()
RET_SUCCESS = 1
RET_FAIL = 0
def update_download(book_id, user_id): def update_download(book_id, user_id):
check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id == check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id ==
book_id).first() book_id).first()
if not check: if not check:
new_download = ub.Downloads(user_id=user_id, book_id=book_id) new_download = ub.Downloads(user_id=user_id, book_id=book_id)
ub.session.add(new_download) ub.session.add(new_download)
ub.session.commit() ub.session.commit()
def make_mobi(book_id, calibrepath): def make_mobi(book_id, calibrepath, user_id, kindle_mail):
book = db.session.query(db.Books).filter(db.Books.id == book_id).first() book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first() data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
if not data: if not data:
error_message = _(u"epub format not found for book id: %(book)d", book=book_id) error_message = _(u"epub format not found for book id: %(book)d", book=book_id)
app.logger.error("make_mobi: " + error_message) app.logger.error("make_mobi: " + error_message)
return error_message, RET_FAIL return error_message
if ub.config.config_use_google_drive: if ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(book.path, data.name + u".epub") df = gd.getFileFromEbooksFolder(book.path, data.name + u".epub")
if df: if df:
@ -82,120 +62,59 @@ def make_mobi(book_id, calibrepath):
os.makedirs(os.path.join(calibrepath, book.path)) os.makedirs(os.path.join(calibrepath, book.path))
df.GetContentFile(datafile) df.GetContentFile(datafile)
else: else:
error_message = "make_mobi: epub not found on gdrive: %s.epub" % data.name error_message = (u"make_mobi: epub not found on gdrive: %s.epub" % data.name)
return error_message, RET_FAIL return error_message
# else:
file_path = os.path.join(calibrepath, book.path, data.name) file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + u".epub"): if os.path.exists(file_path + u".epub"):
# convert book, and upload in case of google drive # append converter to queue
res = converter.convert_mobi(file_path, book) global_WorkerThread.add_convert(file_path, book.id, user_id, _(u"Convert: %s" % book.title), ub.get_mail_settings(),
if ub.config.config_use_google_drive: kindle_mail)
gd.updateGdriveCalibreFromLocal() return None
# time.sleep(10)
return res
else: else:
error_message = "make_mobi: epub not found: %s.epub" % file_path error_message = (u"make_mobi: epub not found: %s.epub" % file_path)
return error_message, RET_FAIL return error_message
def send_test_mail(kindle_mail, user_name): def send_test_mail(kindle_mail, user_name):
msg = MIMEMultipart() global_WorkerThread.add_email(_(u'Calibre-web test email'),None, None, ub.get_mail_settings(),
msg['Subject'] = _(u'Calibre-web test email') kindle_mail, user_name, _(u"Test E-Mail"))
text = _(u'This email has been sent via calibre web.') return
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name, _('Test E-Mail'))
return # send_raw_email(kindle_mail, msg)
# Files are processed in the following order/priority:
# 1: If Mobi file is exisiting, it's directly send to kindle email,
# 2: If Epub file is exisiting, it's converted and send to kindle email
# 3: If Pdf file is exisiting, it's directly send to kindle email,
def send_mail(book_id, kindle_mail, calibrepath, user_id): def send_mail(book_id, kindle_mail, calibrepath, user_id):
"""Send email with attachments""" """Send email with attachments"""
# create MIME message
result= None
msg = MIMEMultipart()
msg['Subject'] = _(u'Send to Kindle')
msg['Message-Id'] = make_msgid('calibre-web')
msg['Date'] = formatdate(localtime=True)
text = _(u'This email has been sent via calibre web.')
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
book = db.session.query(db.Books).filter(db.Books.id == book_id).first() book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).all() data = db.session.query(db.Data).filter(db.Data.book == book.id).all()
formats = {} formats = {}
index = 0 for entry in data:
for indx,entry in enumerate(data):
if entry.format == "MOBI": if entry.format == "MOBI":
formats["mobi"] = entry.name + ".mobi" formats["mobi"] = entry.name + ".mobi"
if entry.format == "EPUB": if entry.format == "EPUB":
formats["epub"] = entry.name + ".epub" formats["epub"] = entry.name + ".epub"
index = indx
if entry.format == "PDF": if entry.format == "PDF":
formats["pdf"] = entry.name + ".pdf" formats["pdf"] = entry.name + ".pdf"
if len(formats) == 0: if len(formats) == 0:
return _("Could not find any formats suitable for sending by email") return _(u"Could not find any formats suitable for sending by email")
if 'mobi' in formats: if 'mobi' in formats:
result = get_attachment(calibrepath, book.path, formats['mobi']) result = formats['mobi']
if result:
msg.attach(result)
elif 'epub' in formats: elif 'epub' in formats:
# returns filename if sucess, otherwise errormessage # returns None if sucess, otherwise errormessage
data, resultCode = make_mobi(book.id, calibrepath) return make_mobi(book.id, calibrepath, user_id, kindle_mail)
if resultCode == RET_SUCCESS:
result = get_attachment(calibrepath, book.path, os.path.basename(data))
if result:
msg.attach(result)
else:
app.logger.error(data)
return data
elif 'pdf' in formats: elif 'pdf' in formats:
result = get_attachment(calibrepath, book.path, formats['pdf']) result = formats['pdf'] # worker.get_attachment()
if result:
msg.attach(result)
else: else:
return _("Could not find any formats suitable for sending by email") return _(u"Could not find any formats suitable for sending by email")
if result: if result:
global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id, _(u"E-Mail: %s" % book.title)) global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, ub.get_mail_settings(),
return None # send_raw_email(kindle_mail, msg) kindle_mail, user_id, _(u"E-Mail: %s" % book.title))
else: else:
return _('The requested file could not be read. Maybe wrong permissions?') return _(u"The requested file could not be read. Maybe wrong permissions?")
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to
def get_attachment(calibrepath, bookpath, filename):
"""Get file as MIMEBase message"""
if ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(bookpath, filename)
if df:
datafile = os.path.join(calibrepath, bookpath, filename)
if not os.path.exists(os.path.join(calibrepath, bookpath)):
os.makedirs(os.path.join(calibrepath, bookpath))
df.GetContentFile(datafile)
else:
return None
file_ = open(datafile, 'rb')
data = file_.read()
file_.close()
os.remove(datafile)
else:
try:
file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
data = file_.read()
file_.close()
except IOError:
traceback.print_exc()
app.logger.error = u'The requested file could not be read. Maybe wrong permissions?'
return None
attachment = MIMEBase('application', 'octet-stream')
attachment.set_payload(data)
encoders.encode_base64(attachment)
attachment.add_header('Content-Disposition', 'attachment',
filename=filename)
return attachment
def get_valid_filename(value, replace_whitespace=True): def get_valid_filename(value, replace_whitespace=True):
@ -225,7 +144,6 @@ def get_valid_filename(value, replace_whitespace=True):
value = value[:128] value = value[:128]
if not value: if not value:
raise ValueError("Filename cannot be empty") raise ValueError("Filename cannot be empty")
return value return value

11
cps/server.py
View File

@ -44,8 +44,8 @@ class server:
web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...') web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args) self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
self.wsgiserver.serve_forever() self.wsgiserver.serve_forever()
except: except Exception:
pass web.app.logger.info("Unknown error while starting gevent")
def startServer(self): def startServer(self):
if gevent_present: if gevent_present:
@ -70,7 +70,7 @@ class server:
if self.restart == True: if self.restart == True:
web.app.logger.info("Performing restart of Calibre-web") web.app.logger.info("Performing restart of Calibre-web")
web.helper.global_eMailThread.stop() web.helper.global_WorkerThread.stop()
if os.name == 'nt': if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""] arguments = ["\"" + sys.executable + "\""]
for e in sys.argv: for e in sys.argv:
@ -80,7 +80,7 @@ class server:
os.execl(sys.executable, sys.executable, *sys.argv) os.execl(sys.executable, sys.executable, *sys.argv)
else: else:
web.app.logger.info("Performing shutdown of Calibre-web") web.app.logger.info("Performing shutdown of Calibre-web")
web.helper.global_eMailThread.stop() web.helper.global_WorkerThread.stop()
sys.exit(0) sys.exit(0)
def setRestartTyp(self,starttyp): def setRestartTyp(self,starttyp):
@ -92,7 +92,8 @@ class server:
else: else:
self.wsgiserver.add_callback(self.wsgiserver.stop) self.wsgiserver.add_callback(self.wsgiserver.stop)
def getNameVersion(self): @staticmethod
def getNameVersion():
if gevent_present: if gevent_present:
return {'Gevent':'v'+geventVersion} return {'Gevent':'v'+geventVersion}
else: else:

13
cps/static/js/archive.js
View File

@ -143,13 +143,12 @@ bitjs.archive = bitjs.archive || {};
* Progress event. * Progress event.
*/ */
bitjs.archive.UnarchiveProgressEvent = function( bitjs.archive.UnarchiveProgressEvent = function(
currentFilename, currentFilename,
currentFileNumber, currentFileNumber,
currentBytesUnarchivedInFile, currentBytesUnarchivedInFile,
currentBytesUnarchived, currentBytesUnarchived,
totalUncompressedBytesInArchive, totalUncompressedBytesInArchive,
totalFilesInArchive) totalFilesInArchive) {
{
bitjs.base(this, bitjs.archive.UnarchiveEvent.Type.PROGRESS); bitjs.base(this, bitjs.archive.UnarchiveEvent.Type.PROGRESS);
this.currentFilename = currentFilename; this.currentFilename = currentFilename;

2
cps/static/js/edit_books.js
View File

@ -80,7 +80,7 @@ function prefixedSource(prefix, query, cb, bhAdapter) {
function getPath() { function getPath() {
var jsFileLocation = $("script[src*=edit_books]").attr("src"); // the js file path var jsFileLocation = $("script[src*=edit_books]").attr("src"); // the js file path
return jsFileLocation.substr(0,jsFileLocation.search("/static/js/edit_books.js")); // the js folder path return jsFileLocation.substr(0, jsFileLocation.search("/static/js/edit_books.js")); // the js folder path
} }
var authors = new Bloodhound({ var authors = new Bloodhound({

5
cps/static/js/io.js
View File

@ -121,7 +121,7 @@ bitjs.io = bitjs.io || {};
* @return {number} The peeked bits, as an unsigned number. * @return {number} The peeked bits, as an unsigned number.
*/ */
bitjs.io.BitStream.prototype.peekBitsRtl = function(n, movePointers) { bitjs.io.BitStream.prototype.peekBitsRtl = function(n, movePointers) {
if (n <= 0 || typeof n != typeof 1) { if (n <= 0 || typeof n !== typeof 1) {
return 0; return 0;
} }
@ -150,8 +150,7 @@ bitjs.io = bitjs.io || {};
bytePtr++; bytePtr++;
bitPtr = 0; bitPtr = 0;
n -= numBitsLeftInThisByte; n -= numBitsLeftInThisByte;
} } else {
else {
result <<= n; result <<= n;
result |= ((bytes[bytePtr] & (BITMASK[n] << (8 - n - bitPtr))) >> (8 - n - bitPtr)); result |= ((bytes[bytePtr] & (BITMASK[n] << (8 - n - bitPtr))) >> (8 - n - bitPtr));

106
cps/static/js/unrar.js
View File

@ -42,10 +42,10 @@ var postProgress = function() {
// shows a byte value as its hex representation // shows a byte value as its hex representation
var nibble = "0123456789ABCDEF"; var nibble = "0123456789ABCDEF";
var byteValueToHexString = function(num) { var byteValueToHexString = function(num) {
return nibble[num>>4] + nibble[num & 0xF]; return nibble[num >> 4] + nibble[num & 0xF];
}; };
var twoByteValueToHexString = function(num) { var twoByteValueToHexString = function(num) {
return nibble[(num>>12) & 0xF] + nibble[(num>>8) & 0xF] + nibble[(num>>4) & 0xF] + nibble[num & 0xF]; return nibble[(num >> 12) & 0xF] + nibble[(num >> 8) & 0xF] + nibble[(num >> 4) & 0xF] + nibble[num & 0xF];
}; };
@ -146,7 +146,7 @@ var RarVolumeHeader = function(bstream) {
} else { } else {
this.HighPackSize = 0; this.HighPackSize = 0;
this.HighUnpSize = 0; this.HighUnpSize = 0;
if (this.unpackedSize == 0xffffffff) { if (this.unpackedSize === 0xffffffff) {
this.HighUnpSize = 0x7fffffff; this.HighUnpSize = 0x7fffffff;
this.unpackedSize = 0xffffffff; this.unpackedSize = 0xffffffff;
} }
@ -178,9 +178,10 @@ var RarVolumeHeader = function(bstream) {
// this is adapted straight out of arcread.cpp, Archive::ReadHeader() // this is adapted straight out of arcread.cpp, Archive::ReadHeader()
for (var I = 0; I < 4; ++I) { for (var I = 0; I < 4; ++I) {
var rmode = extTimeFlags >> ((3 - I) * 4); var rmode = extTimeFlags >> ((3 - I) * 4);
if ((rmode & 8)==0) if ((rmode & 8) === 0) {
continue; continue;
if (I!=0) { }
if (I !== 0) {
bstream.readBits(16); bstream.readBits(16);
} }
var count = (rmode & 3); var count = (rmode & 3);
@ -209,13 +210,12 @@ var RarVolumeHeader = function(bstream) {
}; };
var BLOCK_LZ = 0; var BLOCK_LZ = 0;
// BLOCK_PPM = 1;
var rLDecode = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224], var rLDecode = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224],
rLBits = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5], rLBits = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5],
rDBitLengthCounts = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 14, 0, 12], rDBitLengthCounts = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 14, 0, 12],
rSDDecode = [0, 4, 8, 16, 32, 64, 128, 192], rSDDecode = [0, 4, 8, 16, 32, 64, 128, 192],
rSDBits = [2,2,3, 4, 5, 6, 6, 6]; rSDBits = [2, 2, 3, 4, 5, 6, 6, 6];
var rDDecode = [0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, var rDDecode = [0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32,
48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072,
@ -236,7 +236,7 @@ var rNC = 299,
rBC = 20, rBC = 20,
rHUFF_TABLE_SIZE = (rNC + rDC + rRC + rLDC); rHUFF_TABLE_SIZE = (rNC + rDC + rRC + rLDC);
var UnpBlockType = BLOCK_LZ; //var UnpBlockType = BLOCK_LZ;
var UnpOldTable = new Array(rHUFF_TABLE_SIZE); var UnpOldTable = new Array(rHUFF_TABLE_SIZE);
var BD = { //bitdecode var BD = { //bitdecode
@ -281,26 +281,26 @@ function RarReadTables(bstream) {
} }
if (!bstream.readBits(1)) { //discard old table if (!bstream.readBits(1)) { //discard old table
for (var i = UnpOldTable.length; i--;) UnpOldTable[i] = 0; var i;
for (i = UnpOldTable.length; i--;) UnpOldTable[i] = 0;
} }
// read in bit lengths // read in bit lengths
for (var I = 0; I < rBC; ++I) { for (var I = 0; I < rBC; ++I) {
var Length = bstream.readBits(4); var Length = bstream.readBits(4);
if (Length == 15) { if (Length === 15) {
var ZeroCount = bstream.readBits(4); var ZeroCount = bstream.readBits(4);
if (ZeroCount == 0) { if (ZeroCount === 0) {
BitLength[I] = 15; BitLength[I] = 15;
} } else {
else {
ZeroCount += 2; ZeroCount += 2;
while (ZeroCount-- > 0 && I < rBC) while (ZeroCount-- > 0 && I < rBC) {
BitLength[I++] = 0; BitLength[I++] = 0;
}
--I; --I;
} }
} } else {
else {
BitLength[I] = Length; BitLength[I] = Length;
} }
} }
@ -311,20 +311,20 @@ function RarReadTables(bstream) {
var TableSize = rHUFF_TABLE_SIZE; var TableSize = rHUFF_TABLE_SIZE;
//console.log(DecodeLen, DecodePos, DecodeNum); //console.log(DecodeLen, DecodePos, DecodeNum);
for (var i = 0; i < TableSize;) { for (i = 0; i < TableSize;) {
var num = RarDecodeNumber(bstream, BD); var num = RarDecodeNumber(bstream, BD);
if (num < 16) { if (num < 16) {
Table[i] = (num + UnpOldTable[i]) & 0xf; Table[i] = (num + UnpOldTable[i]) & 0xf;
i++; i++;
} else if(num < 18) { } else if (num < 18) {
var N = (num == 16) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11); var N = (num === 16) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11);
while (N-- > 0 && i < TableSize) { while (N-- > 0 && i < TableSize) {
Table[i] = Table[i - 1]; Table[i] = Table[i - 1];
i++; i++;
} }
} else { } else {
var N = (num == 18) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11); var N = (num === 18) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11);
while (N-- > 0 && i < TableSize) { while (N-- > 0 && i < TableSize) {
Table[i++] = 0; Table[i++] = 0;
@ -337,7 +337,7 @@ function RarReadTables(bstream) {
RarMakeDecodeTables(Table, rNC + rDC, LDD, rLDC); RarMakeDecodeTables(Table, rNC + rDC, LDD, rLDC);
RarMakeDecodeTables(Table, rNC + rDC + rLDC, RD, rRC); RarMakeDecodeTables(Table, rNC + rDC + rLDC, RD, rRC);
for (var i = UnpOldTable.length; i--;) { for (i = UnpOldTable.length; i--;) {
UnpOldTable[i] = Table[i]; UnpOldTable[i] = Table[i];
} }
return true; return true;
@ -349,20 +349,20 @@ function RarDecodeNumber(bstream, dec) {
var bitField = bstream.getBits() & 0xfffe; var bitField = bstream.getBits() & 0xfffe;
//some sort of rolled out binary search //some sort of rolled out binary search
var bits = ((bitField < DecodeLen[8])? var bits = ((bitField < DecodeLen[8])?
((bitField < DecodeLen[4])? ((bitField < DecodeLen[4]) ?
((bitField < DecodeLen[2])? ((bitField < DecodeLen[2]) ?
((bitField < DecodeLen[1])?1:2) ((bitField < DecodeLen[1]) ? 1 : 2)
:((bitField < DecodeLen[3])?3:4)) : ((bitField < DecodeLen[3]) ? 3 : 4))
:(bitField < DecodeLen[6])? : (bitField < DecodeLen[6])?
((bitField < DecodeLen[5])?5:6) ((bitField < DecodeLen[5]) ? 5 : 6)
:((bitField < DecodeLen[7])?7:8)) :((bitField < DecodeLen[7]) ? 7 : 8))
:((bitField < DecodeLen[12])? : ((bitField < DecodeLen[12]) ?
((bitField < DecodeLen[10])? ((bitField < DecodeLen[10]) ?
((bitField < DecodeLen[9])?9:10) ((bitField < DecodeLen[9]) ? 9 : 10)
:((bitField < DecodeLen[11])?11:12)) :((bitField < DecodeLen[11]) ? 11 : 12))
:(bitField < DecodeLen[14])? : (bitField < DecodeLen[14]) ?
((bitField < DecodeLen[13])?13:14) ((bitField < DecodeLen[13]) ? 13 : 14)
:15)); : 15));
bstream.readBits(bits); bstream.readBits(bits);
var N = DecodePos[bits] + ((bitField - DecodeLen[bits -1]) >>> (16 - bits)); var N = DecodePos[bits] + ((bitField - DecodeLen[bits -1]) >>> (16 - bits));
@ -433,14 +433,14 @@ function Unpack20(bstream, Solid) {
RarCopyString(Length, Distance); RarCopyString(Length, Distance);
continue; continue;
} }
if (num == 269) { if (num === 269) {
RarReadTables20(bstream); RarReadTables20(bstream);
RarUpdateProgress() RarUpdateProgress()
continue; continue;
} }
if (num == 256) { if (num === 256) {
lastDist = rOldDist[oldDistPtr++ & 3] = lastDist; lastDist = rOldDist[oldDistPtr++ & 3] = lastDist;
RarCopyString(lastLength, lastDist); RarCopyString(lastLength, lastDist);
continue; continue;
@ -511,14 +511,14 @@ function RarReadTables20(bstream) {
if (num < 16) { if (num < 16) {
Table[I] = num + UnpOldTable20[I] & 0xf; Table[I] = num + UnpOldTable20[I] & 0xf;
I++; I++;
} else if(num == 16) { } else if(num === 16) {
N = bstream.readBits(2) + 3; N = bstream.readBits(2) + 3;
while (N-- > 0 && I < TableSize) { while (N-- > 0 && I < TableSize) {
Table[I] = Table[I - 1]; Table[I] = Table[I - 1];
I++; I++;
} }
} else { } else {
if (num == 17) { if (num === 17) {
N = bstream.readBits(3) + 3; N = bstream.readBits(3) + 3;
} else { } else {
N = bstream.readBits(7) + 11; N = bstream.readBits(7) + 11;
@ -595,7 +595,7 @@ function Unpack29(bstream, Solid) {
Distance += prevLowDist; Distance += prevLowDist;
} else { } else {
var LowDist = RarDecodeNumber(bstream, LDD); var LowDist = RarDecodeNumber(bstream, LDD);
if (LowDist == 16) { if (LowDist === 16) {
lowDistRepCount = rLOW_DIST_REP_COUNT - 1; lowDistRepCount = rLOW_DIST_REP_COUNT - 1;
Distance += prevLowDist; Distance += prevLowDist;
} else { } else {
@ -618,16 +618,16 @@ function Unpack29(bstream, Solid) {
RarCopyString(Length, Distance); RarCopyString(Length, Distance);
continue; continue;
} }
if (num == 256) { if (num === 256) {
if (!RarReadEndOfBlock(bstream)) break; if (!RarReadEndOfBlock(bstream)) break;
continue; continue;
} }
if (num == 257) { if (num === 257) {
//console.log("READVMCODE"); //console.log("READVMCODE");
if (!RarReadVMCode(bstream)) break; if (!RarReadVMCode(bstream)) break;
continue; continue;
} }
if (num == 258) { if (num === 258) {
if (lastLength != 0) { if (lastLength != 0) {
RarCopyString(lastLength, lastDist); RarCopyString(lastLength, lastDist);
} }
@ -684,9 +684,9 @@ function RarReadEndOfBlock(bstream) {
function RarReadVMCode(bstream) { function RarReadVMCode(bstream) {
var FirstByte = bstream.readBits(8); var FirstByte = bstream.readBits(8);
var Length = (FirstByte & 7) + 1; var Length = (FirstByte & 7) + 1;
if (Length == 7) { if (Length === 7) {
Length = bstream.readBits(8) + 7; Length = bstream.readBits(8) + 7;
} else if(Length == 8) { } else if(Length === 8) {
Length = bstream.readBits(16); Length = bstream.readBits(16);
} }
var vmCode = []; var vmCode = [];
@ -789,8 +789,8 @@ RarLocalFile.prototype.unrar = function() {
if (!this.header.flags.LHD_SPLIT_BEFORE) { if (!this.header.flags.LHD_SPLIT_BEFORE) {
// unstore file // unstore file
if (this.header.method == 0x30) { if (this.header.method === 0x30) {
info("Unstore "+this.filename); info("Unstore " + this.filename);
this.isValid = true; this.isValid = true;
currentBytesUnarchivedInFile += this.fileData.length; currentBytesUnarchivedInFile += this.fileData.length;
@ -820,10 +820,10 @@ var unrar = function(arrayBuffer) {
var bstream = new bitjs.io.BitStream(arrayBuffer, false /* rtl */); var bstream = new bitjs.io.BitStream(arrayBuffer, false /* rtl */);
var header = new RarVolumeHeader(bstream); var header = new RarVolumeHeader(bstream);
if (header.crc == 0x6152 && if (header.crc === 0x6152 &&
header.headType == 0x72 && header.headType === 0x72 &&
header.flags.value == 0x1A21 && header.flags.value === 0x1A21 &&
header.headSize == 7) header.headSize === 7)
{ {
info("Found RAR signature"); info("Found RAR signature");
@ -840,7 +840,7 @@ var unrar = function(arrayBuffer) {
if (localFile && localFile.isValid && localFile.header.packSize > 0) { if (localFile && localFile.isValid && localFile.header.packSize > 0) {
totalUncompressedBytesInArchive += localFile.header.unpackedSize; totalUncompressedBytesInArchive += localFile.header.unpackedSize;
localFiles.push(localFile); localFiles.push(localFile);
} else if (localFile.header.packSize == 0 && localFile.header.unpackedSize == 0) { } else if (localFile.header.packSize === 0 && localFile.header.unpackedSize === 0) {
localFile.isValid = true; localFile.isValid = true;
} }
} catch(err) { } catch(err) {

221
cps/static/js/untar.js
View File

@ -7,10 +7,11 @@
* *
* TAR format: http://www.gnu.org/software/automake/manual/tar/Standard.html * TAR format: http://www.gnu.org/software/automake/manual/tar/Standard.html
*/ */
/* global bitjs, importScripts, Uint8Array */
// This file expects to be invoked as a Worker (see onmessage below). // This file expects to be invoked as a Worker (see onmessage below).
importScripts('io.js'); importScripts("io.js");
importScripts('archive.js'); importScripts("archive.js");
// Progress variables. // Progress variables.
var currentFilename = ""; var currentFilename = "";
@ -22,147 +23,147 @@ var totalFilesInArchive = 0;
// Helper functions. // Helper functions.
var info = function(str) { var info = function(str) {
postMessage(new bitjs.archive.UnarchiveInfoEvent(str)); postMessage(new bitjs.archive.UnarchiveInfoEvent(str));
}; };
var err = function(str) { var err = function(str) {
postMessage(new bitjs.archive.UnarchiveErrorEvent(str)); postMessage(new bitjs.archive.UnarchiveErrorEvent(str));
}; };
var postProgress = function() { var postProgress = function() {
postMessage(new bitjs.archive.UnarchiveProgressEvent( postMessage(new bitjs.archive.UnarchiveProgressEvent(
currentFilename, currentFilename,
currentFileNumber, currentFileNumber,
currentBytesUnarchivedInFile, currentBytesUnarchivedInFile,
currentBytesUnarchived, currentBytesUnarchived,
totalUncompressedBytesInArchive, totalUncompressedBytesInArchive,
totalFilesInArchive)); totalFilesInArchive));
}; };
// Removes all characters from the first zero-byte in the string onwards. // Removes all characters from the first zero-byte in the string onwards.
var readCleanString = function(bstr, numBytes) { var readCleanString = function(bstr, numBytes) {
var str = bstr.readString(numBytes); var str = bstr.readString(numBytes);
var zIndex = str.indexOf(String.fromCharCode(0)); var zIndex = str.indexOf(String.fromCharCode(0));
return zIndex != -1 ? str.substr(0, zIndex) : str; return zIndex !== -1 ? str.substr(0, zIndex) : str;
}; };
// takes a ByteStream and parses out the local file information // takes a ByteStream and parses out the local file information
var TarLocalFile = function(bstream) { var TarLocalFile = function(bstream) {
this.isValid = false; this.isValid = false;
// Read in the header block // Read in the header block
this.name = readCleanString(bstream, 100); this.name = readCleanString(bstream, 100);
this.mode = readCleanString(bstream, 8); this.mode = readCleanString(bstream, 8);
this.uid = readCleanString(bstream, 8); this.uid = readCleanString(bstream, 8);
this.gid = readCleanString(bstream, 8); this.gid = readCleanString(bstream, 8);
this.size = parseInt(readCleanString(bstream, 12), 8); this.size = parseInt(readCleanString(bstream, 12), 8);
this.mtime = readCleanString(bstream, 12); this.mtime = readCleanString(bstream, 12);
this.chksum = readCleanString(bstream, 8); this.chksum = readCleanString(bstream, 8);
this.typeflag = readCleanString(bstream, 1); this.typeflag = readCleanString(bstream, 1);
this.linkname = readCleanString(bstream, 100); this.linkname = readCleanString(bstream, 100);
this.maybeMagic = readCleanString(bstream, 6); this.maybeMagic = readCleanString(bstream, 6);
if (this.maybeMagic == "ustar") { if (this.maybeMagic === "ustar") {
this.version = readCleanString(bstream, 2); this.version = readCleanString(bstream, 2);
this.uname = readCleanString(bstream, 32); this.uname = readCleanString(bstream, 32);
this.gname = readCleanString(bstream, 32); this.gname = readCleanString(bstream, 32);
this.devmajor = readCleanString(bstream, 8); this.devmajor = readCleanString(bstream, 8);
this.devminor = readCleanString(bstream, 8); this.devminor = readCleanString(bstream, 8);
this.prefix = readCleanString(bstream, 155); this.prefix = readCleanString(bstream, 155);
if (this.prefix.length) { if (this.prefix.length) {
this.name = this.prefix + this.name; this.name = this.prefix + this.name;
} }
bstream.readBytes(12); // 512 - 500 bstream.readBytes(12); // 512 - 500
} else { } else {
bstream.readBytes(255); // 512 - 257 bstream.readBytes(255); // 512 - 257
} }
// Done header, now rest of blocks are the file contents.
this.filename = this.name;
this.fileData = null;
info("Untarring file '" + this.filename + "'"); // Done header, now rest of blocks are the file contents.
info(" size = " + this.size); this.filename = this.name;
info(" typeflag = " + this.typeflag); this.fileData = null;
// A regular file. info("Untarring file '" + this.filename + "'");
if (this.typeflag == 0) { info(" size = " + this.size);
info(" This is a regular file."); info(" typeflag = " + this.typeflag);
var sizeInBytes = parseInt(this.size);
this.fileData = new Uint8Array(bstream.bytes.buffer, bstream.ptr, this.size);
if (this.name.length > 0 && this.size > 0 && this.fileData && this.fileData.buffer) {
this.isValid = true;
}
bstream.readBytes(this.size); // A regular file.
if (this.typeflag === 0) {
info(" This is a regular file.");
var sizeInBytes = parseInt(this.size);
this.fileData = new Uint8Array(bstream.bytes.buffer, bstream.ptr, this.size);
if (this.name.length > 0 && this.size > 0 && this.fileData && this.fileData.buffer) {
this.isValid = true;
}
// Round up to 512-byte blocks. bstream.readBytes(this.size);
var remaining = 512 - this.size % 512;
if (remaining > 0 && remaining < 512) { // Round up to 512-byte blocks.
bstream.readBytes(remaining); var remaining = 512 - this.size % 512;
} if (remaining > 0 && remaining < 512) {
} else if (this.typeflag == 5) { bstream.readBytes(remaining);
info(" This is a directory.") }
} } else if (this.typeflag === 5) {
info(" This is a directory.")
}
}; };
// Takes an ArrayBuffer of a tar file in // Takes an ArrayBuffer of a tar file in
// returns null on error // returns null on error
// returns an array of DecompressedFile objects on success // returns an array of DecompressedFile objects on success
var untar = function(arrayBuffer) { var untar = function(arrayBuffer) {
currentFilename = ""; currentFilename = "";
currentFileNumber = 0; currentFileNumber = 0;
currentBytesUnarchivedInFile = 0; currentBytesUnarchivedInFile = 0;
currentBytesUnarchived = 0; currentBytesUnarchived = 0;
totalUncompressedBytesInArchive = 0; totalUncompressedBytesInArchive = 0;
totalFilesInArchive = 0; totalFilesInArchive = 0;
postMessage(new bitjs.archive.UnarchiveStartEvent()); postMessage(new bitjs.archive.UnarchiveStartEvent());
var bstream = new bitjs.io.ByteStream(arrayBuffer); var bstream = new bitjs.io.ByteStream(arrayBuffer);
var localFiles = []; var localFiles = [];
// While we don't encounter an empty block, keep making TarLocalFiles. // While we don't encounter an empty block, keep making TarLocalFiles.
while (bstream.peekNumber(4) != 0) { while (bstream.peekNumber(4) !== 0) {
var oneLocalFile = new TarLocalFile(bstream); var oneLocalFile = new TarLocalFile(bstream);
if (oneLocalFile && oneLocalFile.isValid) { if (oneLocalFile && oneLocalFile.isValid) {
localFiles.push(oneLocalFile); localFiles.push(oneLocalFile);
totalUncompressedBytesInArchive += oneLocalFile.size; totalUncompressedBytesInArchive += oneLocalFile.size;
} }
} }
totalFilesInArchive = localFiles.length; totalFilesInArchive = localFiles.length;
// got all local files, now sort them // got all local files, now sort them
localFiles.sort(function(a,b) { localFiles.sort(function(a, b) {
var aname = a.filename.toLowerCase(); var aname = a.filename.toLowerCase();
var bname = b.filename.toLowerCase(); var bname = b.filename.toLowerCase();
return aname > bname ? 1 : -1; return aname > bname ? 1 : -1;
}); });
// report # files and total length
if (localFiles.length > 0) {
postProgress();
}
// now do the shipping of each file
for (var i = 0; i < localFiles.length; ++i) {
var localfile = localFiles[i];
info("Sending file '" + localfile.filename + "' up");
// update progress
currentFilename = localfile.filename;
currentFileNumber = i;
currentBytesUnarchivedInFile = localfile.size;
currentBytesUnarchived += localfile.size;
postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile));
postProgress();
}
// report # files and total length
if (localFiles.length > 0) {
postProgress(); postProgress();
}
// now do the shipping of each file postMessage(new bitjs.archive.UnarchiveFinishEvent());
for (var i = 0; i < localFiles.length; ++i) {
var localfile = localFiles[i];
info("Sending file '" + localfile.filename + "' up");
// update progress
currentFilename = localfile.filename;
currentFileNumber = i;
currentBytesUnarchivedInFile = localfile.size;
currentBytesUnarchived += localfile.size;
postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile));
postProgress();
}
postProgress();
postMessage(new bitjs.archive.UnarchiveFinishEvent());
}; };
// event.data.file has the ArrayBuffer. // event.data.file has the ArrayBuffer.
onmessage = function(event) { onmessage = function(event) {
var ab = event.data.file; var ab = event.data.file;
untar(ab); untar(ab);
}; };

102
cps/static/js/unzip.js
View File

@ -49,7 +49,7 @@ var zDigitalSignatureSignature = 0x05054b50;
// takes a ByteStream and parses out the local file information // takes a ByteStream and parses out the local file information
var ZipLocalFile = function(bstream) { var ZipLocalFile = function(bstream) {
if (typeof bstream !== typeof {} || !bstream.readNumber || typeof bstream.readNumber != typeof function() {} ) { if (typeof bstream !== typeof {} || !bstream.readNumber || typeof bstream.readNumber !== typeof function() {} ) {
return null; return null;
} }
@ -115,9 +115,8 @@ ZipLocalFile.prototype.unzip = function() {
info("ZIP v" + this.version + ", store only: " + this.filename + " (" + this.compressedSize + " bytes)"); info("ZIP v" + this.version + ", store only: " + this.filename + " (" + this.compressedSize + " bytes)");
currentBytesUnarchivedInFile = this.compressedSize; currentBytesUnarchivedInFile = this.compressedSize;
currentBytesUnarchived += this.compressedSize; currentBytesUnarchived += this.compressedSize;
} } else if (this.compressionMethod === 8) {
// version == 20, compression method == 8 (DEFLATE) // version == 20, compression method == 8 (DEFLATE)
else if (this.compressionMethod === 8) {
info("ZIP v2.0, DEFLATE: " + this.filename + " (" + this.compressedSize + " bytes)"); info("ZIP v2.0, DEFLATE: " + this.filename + " (" + this.compressedSize + " bytes)");
this.fileData = inflate(this.fileData, this.uncompressedSize); this.fileData = inflate(this.fileData, this.uncompressedSize);
} }
@ -245,6 +244,7 @@ var unzip = function(arrayBuffer) {
// each entry's index is its code and its value is a JavaScript object // each entry's index is its code and its value is a JavaScript object
// containing {length: 6, symbol: X} // containing {length: 6, symbol: X}
function getHuffmanCodes(bitLengths) { function getHuffmanCodes(bitLengths) {
var len;
// ensure bitLengths is an array containing at least one element // ensure bitLengths is an array containing at least one element
if (typeof bitLengths !== typeof [] || bitLengths.length < 1) { if (typeof bitLengths !== typeof [] || bitLengths.length < 1) {
err("Error! getHuffmanCodes() called with an invalid array"); err("Error! getHuffmanCodes() called with an invalid array");
@ -256,9 +256,10 @@ function getHuffmanCodes(bitLengths) {
blCount = [], blCount = [],
MAX_BITS = 1; MAX_BITS = 1;
// Step 1: count up how many codes of each length we have // Step 1: count up how many codes of each length we have
for (var i = 0; i < numLengths; ++i) { for (var i = 0; i < numLengths; ++i) {
var len = bitLengths[i]; len = bitLengths[i];
// test to ensure each bit length is a positive, non-zero number // test to ensure each bit length is a positive, non-zero number
if (typeof len !== typeof 1 || len < 0) { if (typeof len !== typeof 1 || len < 0) {
err("bitLengths contained an invalid number in getHuffmanCodes(): " + len + " of type " + (typeof len)); err("bitLengths contained an invalid number in getHuffmanCodes(): " + len + " of type " + (typeof len));
@ -276,17 +277,17 @@ function getHuffmanCodes(bitLengths) {
var nextCode = [], var nextCode = [],
code = 0; code = 0;
for (var bits = 1; bits <= MAX_BITS; ++bits) { for (var bits = 1; bits <= MAX_BITS; ++bits) {
var len = bits-1; len = bits - 1;
// ensure undefined lengths are zero // ensure undefined lengths are zero
if (blCount[len] == undefined) blCount[len] = 0; if (blCount[len] === undefined) blCount[len] = 0;
code = (code + blCount[bits-1]) << 1; code = (code + blCount[bits - 1]) << 1;
nextCode[bits] = code; nextCode[bits] = code;
} }
// Step 3: Assign numerical values to all codes // Step 3: Assign numerical values to all codes
var table = {}, tableLength = 0; var table = {}, tableLength = 0;
for (var n = 0; n < numLengths; ++n) { for (var n = 0; n < numLengths; ++n) {
var len = bitLengths[n]; len = bitLengths[n];
if (len !== 0) { if (len !== 0) {
table[nextCode[len]] = { length: len, symbol: n }; //, bitstring: binaryValueToString(nextCode[len],len) }; table[nextCode[len]] = { length: len, symbol: n }; //, bitstring: binaryValueToString(nextCode[len],len) };
tableLength++; tableLength++;
@ -318,13 +319,14 @@ function getHuffmanCodes(bitLengths) {
var fixedHCtoLiteral = null; var fixedHCtoLiteral = null;
var fixedHCtoDistance = null; var fixedHCtoDistance = null;
function getFixedLiteralTable() { function getFixedLiteralTable() {
var i;
// create once // create once
if (!fixedHCtoLiteral) { if (!fixedHCtoLiteral) {
var bitlengths = new Array(288); var bitlengths = new Array(288);
for (var i = 0; i <= 143; ++i) bitlengths[i] = 8; for (i = 0; i <= 143; ++i) bitlengths[i] = 8;
for (var i = 144; i <= 255; ++i) bitlengths[i] = 9; for (i = 144; i <= 255; ++i) bitlengths[i] = 9;
for (var i = 256; i <= 279; ++i) bitlengths[i] = 7; for (i = 256; i <= 279; ++i) bitlengths[i] = 7;
for (var i = 280; i <= 287; ++i) bitlengths[i] = 8; for (i = 280; i <= 287; ++i) bitlengths[i] = 8;
// get huffman code table // get huffman code table
fixedHCtoLiteral = getHuffmanCodes(bitlengths); fixedHCtoLiteral = getHuffmanCodes(bitlengths);
@ -355,11 +357,11 @@ function decodeSymbol(bstream, hcTable) {
for (;;) { for (;;) {
// read in next bit // read in next bit
var bit = bstream.readBits(1); var bit = bstream.readBits(1);
code = (code<<1) | bit; code = (code << 1) | bit;
++len; ++len;
// check against Huffman Code table and break if found // check against Huffman Code table and break if found
if (hcTable.hasOwnProperty(code) && hcTable[code].length == len) { if (hcTable.hasOwnProperty(code) && hcTable[code].length === len) {
break; break;
} }
@ -374,31 +376,31 @@ function decodeSymbol(bstream, hcTable) {
var CodeLengthCodeOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]; var CodeLengthCodeOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15];
/* /*
Extra Extra Extra Extra Extra Extra
Code Bits Length(s) Code Bits Lengths Code Bits Length(s) Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
---- ---- ------ ---- ---- ------- ---- ---- ------- ---- ---- ------ ---- ---- ------- ---- ---- -------
257 0 3 267 1 15,16 277 4 67-82 257 0 3 267 1 15,16 277 4 67-82
258 0 4 268 1 17,18 278 4 83-98 258 0 4 268 1 17,18 278 4 83-98
259 0 5 269 2 19-22 279 4 99-114 259 0 5 269 2 19-22 279 4 99-114
260 0 6 270 2 23-26 280 4 115-130 260 0 6 270 2 23-26 280 4 115-130
261 0 7 271 2 27-30 281 5 131-162 261 0 7 271 2 27-30 281 5 131-162
262 0 8 272 2 31-34 282 5 163-194 262 0 8 272 2 31-34 282 5 163-194
263 0 9 273 3 35-42 283 5 195-226 263 0 9 273 3 35-42 283 5 195-226
264 0 10 274 3 43-50 284 5 227-257 264 0 10 274 3 43-50 284 5 227-257
265 1 11,12 275 3 51-58 285 0 258 265 1 11,12 275 3 51-58 285 0 258
266 1 13,14 276 3 59-66 266 1 13,14 276 3 59-66
*/
*/
var LengthLookupTable = [ var LengthLookupTable = [
[0, 3], [0, 4], [0, 5], [0, 6], [0, 3], [0, 4], [0, 5], [0, 6],
[0, 7], [0, 8], [0, 9], [0, 10], [0, 7], [0, 8], [0, 9], [0, 10],
[1, 11], [1, 13], [1, 15], [1, 17], [1, 11], [1, 13], [1, 15], [1, 17],
[2, 19], [2, 23], [2, 27], [2, 31], [2, 19], [2, 23], [2, 27], [2, 31],
[3, 35], [3, 43], [3, 51], [3, 59], [3, 35], [3, 43], [3, 51], [3, 59],
[4, 67], [4, 83], [4, 99], [4, 115], [4, 67], [4, 83], [4, 99], [4, 115],
[5, 131], [5, 163], [5, 195], [5, 227], [5, 131], [5, 163], [5, 195], [5, 227],
[0, 258] [0, 258]
]; ];
/* /*
Extra Extra Extra Extra Extra Extra
@ -448,10 +450,10 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
stream, and copy length bytes from this stream, and copy length bytes from this
position to the output stream. position to the output stream.
*/ */
var numSymbols = 0, blockSize = 0; var blockSize = 0;
for (;;) { for (;;) {
var symbol = decodeSymbol(bstream, hcLiteralTable); var symbol = decodeSymbol(bstream, hcLiteralTable);
++numSymbols; // ++numSymbols;
if (symbol < 256) { if (symbol < 256) {
// copy literal byte to output // copy literal byte to output
buffer.insertByte(symbol); buffer.insertByte(symbol);
@ -463,7 +465,7 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
break; break;
} }
else { else {
var lengthLookup = LengthLookupTable[symbol-257], var lengthLookup = LengthLookupTable[symbol - 257],
length = lengthLookup[1] + bstream.readBits(lengthLookup[0]), length = lengthLookup[1] + bstream.readBits(lengthLookup[0]),
distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)], distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)],
distance = distLookup[1] + bstream.readBits(distLookup[0]); distance = distLookup[1] + bstream.readBits(distLookup[0]);
@ -481,13 +483,13 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
// loop for each character // loop for each character
var ch = buffer.ptr - distance; var ch = buffer.ptr - distance;
blockSize += length; blockSize += length;
if(length > distance) { if (length > distance) {
var data = buffer.data; var data = buffer.data;
while (length--) { while (length--) {
buffer.insertByte(data[ch++]); buffer.insertByte(data[ch++]);
} }
} else { } else {
buffer.insertBytes(buffer.data.subarray(ch, ch + length)) buffer.insertBytes(buffer.data.subarray(ch, ch + length));
} }
} // length-distance pair } // length-distance pair
@ -506,7 +508,7 @@ function inflate(compressedData, numDecompressedBytes) {
compressedData.byteOffset, compressedData.byteOffset,
compressedData.byteLength); compressedData.byteLength);
var buffer = new bitjs.io.ByteBuffer(numDecompressedBytes); var buffer = new bitjs.io.ByteBuffer(numDecompressedBytes);
var numBlocks = 0; //var numBlocks = 0;
var blockSize = 0; var blockSize = 0;
// block format: http://tools.ietf.org/html/rfc1951#page-9 // block format: http://tools.ietf.org/html/rfc1951#page-9
@ -514,9 +516,9 @@ function inflate(compressedData, numDecompressedBytes) {
var bFinal = bstream.readBits(1); var bFinal = bstream.readBits(1);
var bType = bstream.readBits(2); var bType = bstream.readBits(2);
blockSize = 0; blockSize = 0;
++numBlocks; // ++numBlocks;
// no compression // no compression
if (bType == 0) { if (bType === 0) {
// skip remaining bits in this byte // skip remaining bits in this byte
while (bstream.bitPtr != 0) bstream.readBits(1); while (bstream.bitPtr != 0) bstream.readBits(1);
var len = bstream.readBits(16), var len = bstream.readBits(16),

8
cps/templates/list.html
View File

@ -2,15 +2,15 @@
{% block body %} {% block body %}
<h1>{{title}}</h1> <h1>{{title}}</h1>
<div class="container"> <div class="container">
<div class="col-sm-6"> <div class="col-xs-12 col-sm-6">
{% for entry in entries %} {% for entry in entries %}
{% if loop.index0 == (loop.length/2)|int and loop.length > 20 %} {% if loop.index0 == (loop.length/2)|int and loop.length > 20 %}
</div> </div>
<div class="col-sm-6"> <div class="col-xs-12 col-sm-6">
{% endif %} {% endif %}
<div class="row"> <div class="row">
<div class="col-xs-1" align="left"><span class="badge">{{entry.count}}</span></div> <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
<div class="col-xs-6"><a id="list_{{loop.index0}}" href="{{url_for(folder, book_id=entry[0].id )}}">{{entry[0].name}}</a></div> <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for(folder, book_id=entry[0].id )}}">{{entry[0].name}}</a></div>
</div> </div>
{% endfor %} {% endfor %}
</div> </div>

23
cps/templates/tasks.html
View File

@ -5,17 +5,19 @@
{% block body %} {% block body %}
<div class="discover"> <div class="discover">
<h2>{{_('Tasks list')}}</h2> <h2>{{_('Tasks list')}}</h2>
<table class="table table-no-bordered" id="table" data-url="{{'/ajax/emailstat'}}"> <table class="table table-no-bordered" id="table" data-url="{{'/ajax/emailstat'}}" data-sort-name="starttime" data-sort-order="asc">
<thead> <thead>
<tr> <tr>
{% if g.user.role_admin() %} {% if g.user.role_admin() %}
<th data-field="user">{{_('User')}}</th> <th data-halign="right" data-align="right" data-field="user" data-sortable="true">{{_('User')}}</th>
{% endif %} {% endif %}
<th data-field="type">{{_('Task')}}</th> <th data-halign="right" data-align="right" data-field="type" data-sortable="true">{{_('Task')}}</th>
<th data-field="status">{{_('Status')}}</th> <th data-halign="right" data-align="right" data-field="status" data-sortable="true">{{_('Status')}}</th>
<th data-field="progress">{{_('Progress')}}</th> <th data-halign="right" data-align="right" data-field="progress" data-sortable="true" data-sorter="elementSorter">{{_('Progress')}}</th>
<th data-field="runtime">{{_('Runtime')}}</th> <th data-halign="right" data-align="right" data-field="runtime" data-sortable="true" data-sort-name="rt">{{_('Runtime')}}</th>
<th data-field="starttime">{{_('Starttime')}}</th> <th data-halign="right" data-align="right" data-field="starttime" data-sortable="true" data-sort-name="id">{{_('Starttime')}}</th>
<th data-field="id" data-visible="false"></th>
<th data-field="rt" data-visible="false"></th>
</tr> </tr>
</thead> </thead>
</table> </table>
@ -43,6 +45,13 @@
} }
}); });
}, 1000); }, 1000);
function elementSorter(a, b) {
a = +a.slice(0, -2);
b = +b.slice(0, -2);
if (a > b) return 1;
if (a < b) return -1;
return 0;
}
</script> </script>
{% endblock %} {% endblock %}

54
cps/web.py
View File

@ -29,14 +29,13 @@ import logging
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
from flask import (Flask, render_template, request, Response, redirect, from flask import (Flask, render_template, request, Response, redirect,
url_for, send_from_directory, make_response, g, flash, url_for, send_from_directory, make_response, g, flash,
abort, Markup, stream_with_context) abort, Markup)
from flask import __version__ as flaskVersion from flask import __version__ as flaskVersion
import cache_buster import cache_buster
import ub import ub
from ub import config from ub import config
import helper import helper
import os import os
import errno
from sqlalchemy.sql.expression import func from sqlalchemy.sql.expression import func
from sqlalchemy.sql.expression import false from sqlalchemy.sql.expression import false
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
@ -349,25 +348,20 @@ def remote_login_required(f):
def shortentitle_filter(s,nchar=20): def shortentitle_filter(s,nchar=20):
text = s.split() text = s.split()
res = "" # result res = "" # result
sum = 0 # overall length suml = 0 # overall length
for line in text: for line in text:
if sum >= 60: if suml >= 60:
res += '...' res += '...'
break break
# if word longer than 20 chars truncate line and append '...', otherwise add whole word to result # if word longer than 20 chars truncate line and append '...', otherwise add whole word to result
# string, and summarize total length to stop at 60 chars # string, and summarize total length to stop at 60 chars
if len(line) > nchar: if len(line) > nchar:
res += line[:(nchar-3)] + '[..] ' res += line[:(nchar-3)] + '[..] '
sum += nchar+3 suml += nchar+3
else: else:
res += line + ' ' res += line + ' '
sum += len(line) + 1 suml += len(line) + 1
return res.strip() return res.strip()
#if len(s) > 20:
# s = s.split(':', 1)[0]
# if len(s) > 20:
# s = textwrap.wrap(s, 20, break_long_words=True)[0] + ' ...'
#return s
@app.template_filter('mimetype') @app.template_filter('mimetype')
@ -784,7 +778,7 @@ def feed_series(book_id):
off = request.args.get("offset") off = request.args.get("offset")
if not off: if not off:
off = 0 off = 0
entries, random, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
db.Books, db.Books.series.any(db.Series.id == book_id), [db.Books.series_index]) db.Books, db.Books.series.any(db.Series.id == book_id), [db.Books.series_index])
xml = render_title_template('feed.xml', entries=entries, pagination=pagination) xml = render_title_template('feed.xml', entries=entries, pagination=pagination)
response = make_response(xml) response = make_response(xml)
@ -889,7 +883,7 @@ def get_metadata_calibre_companion(uuid):
@login_required @login_required
def get_email_status_json(): def get_email_status_json():
answer=list() answer=list()
tasks=helper.global_eMailThread.get_taskstatus() tasks=helper.global_WorkerThread.get_taskstatus()
if not current_user.role_admin(): if not current_user.role_admin():
for task in tasks: for task in tasks:
if task['user'] == current_user.nickname: if task['user'] == current_user.nickname:
@ -909,6 +903,32 @@ def get_email_status_json():
if 'starttime' not in task: if 'starttime' not in task:
task['starttime'] = "" task['starttime'] = ""
answer = tasks answer = tasks
'''answer.append({'user': 'Test', 'starttime': '07.3.2018 15:23', 'progress': " 0 %", 'type': 'E-Mail',
'runtime': '0 s', 'rt': 0, 'status': _('Waiting'),'id':1 })
answer.append({'user': 'Admin', 'starttime': '07.3.2018 15:33', 'progress': " 11 %", 'type': 'E-Mail',
'runtime': '2 s', 'rt':2, 'status': _('Waiting'),'id':2})
answer.append({'user': 'Nanny', 'starttime': '8.3.2018 15:23', 'progress': " 2 %", 'type': 'E-Mail',
'runtime': '32 s','rt':32, 'status': _('Waiting'),'id':3})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '7 s','rt':7, 'status': _('Waiting'),'id':4})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '22 s','rt':22, 'status': _('Waiting'),'id':5})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '17 s','rt':17, 'status': _('Waiting'),'id':6})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '72 s','rt':72, 'status': _('Waiting'),'id':7})
answer.append({'user': 'Guest', 'starttime': '19.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '1:07 s','rt':67, 'status': _('Waiting'),'id':8})
answer.append({'user': 'Guest', 'starttime': '18.2.2018 12:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '2:07 s','rt':127, 'status': _('Waiting'),'id':9})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '27 s','rt':27, 'status': _('Waiting'),'id':10})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 16:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '73 s','rt':73, 'status': _('Waiting'),'id':11})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '71 s','rt':71, 'status': _('Waiting'),'id':12})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 17:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '27 s','rt':27, 'status': _('Waiting'),'id':13})'''
js=json.dumps(answer) js=json.dumps(answer)
response = make_response(js) response = make_response(js)
response.headers["Content-Type"] = "application/json; charset=utf-8" response.headers["Content-Type"] = "application/json; charset=utf-8"
@ -1184,7 +1204,7 @@ def author(book_id, page):
gc = GoodreadsClient(config.config_goodreads_api_key, config.config_goodreads_api_secret) gc = GoodreadsClient(config.config_goodreads_api_key, config.config_goodreads_api_secret)
author_info = gc.find_author(author_name=name) author_info = gc.find_author(author_name=name)
other_books = get_unique_other_books(entries.all(), author_info.books) other_books = get_unique_other_books(entries.all(), author_info.books)
except: except Exception:
# Skip goodreads, if site is down/inaccessible # Skip goodreads, if site is down/inaccessible
app.logger.error('Goodreads website is down/inaccessible') app.logger.error('Goodreads website is down/inaccessible')
@ -1424,7 +1444,7 @@ def bookmark(book_id, book_format):
def get_tasks_status(): def get_tasks_status():
# if current user admin, show all email, otherwise only own emails # if current user admin, show all email, otherwise only own emails
answer=list() answer=list()
tasks=helper.global_eMailThread.get_taskstatus() tasks=helper.global_WorkerThread.get_taskstatus()
if not current_user.role_admin(): if not current_user.role_admin():
for task in tasks: for task in tasks:
if task['user'] == current_user.nickname: if task['user'] == current_user.nickname:
@ -1492,9 +1512,7 @@ def delete_book(book_id, book_format):
# delete book from Shelfs, Downloads, Read list # delete book from Shelfs, Downloads, Read list
ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete() ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete() ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete()
# ToDo check Downloads.book right
ub.delete_download(book_id) ub.delete_download(book_id)
# ub.session.query(ub.Downloads).filter(ub.Downloads.book_id == book_id).delete()
ub.session.commit() ub.session.commit()
# check if only this book links to: # check if only this book links to:
@ -2735,7 +2753,6 @@ def configuration_helper(origin):
gdriveError=gdriveError, goodreads=goodreads_support, gdriveError=gdriveError, goodreads=goodreads_support,
title=_(u"Basic Configuration"), page="config") title=_(u"Basic Configuration"), page="config")
if reboot_required: if reboot_required:
# db.engine.dispose() # ToDo verify correct
ub.session.close() ub.session.close()
ub.engine.dispose() ub.engine.dispose()
# stop Server # stop Server
@ -3066,7 +3083,6 @@ def edit_book(book_id):
if is_format: if is_format:
# Format entry already exists, no need to update the database # Format entry already exists, no need to update the database
app.logger.info('Bokk format already existing') app.logger.info('Bokk format already existing')
pass
else: else:
db_format = db.Data(book_id, file_ext.upper(), file_size, file_name) db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
db.session.add(db_format) db.session.add(db_format)

501
cps/worker.py Normal file
View File

@ -0,0 +1,501 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import smtplib
import threading
from datetime import datetime
import logging
import time
import socket
import sys
import os
from email.generator import Generator
import web
from flask_babel import gettext as _
# from babel.dates import format_datetime
import re
import gdriveutils as gd
import subprocess
try:
from StringIO import StringIO
from email.MIMEBase import MIMEBase
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
except ImportError:
from io import StringIO
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email import encoders
from email.utils import formatdate
from email.utils import make_msgid
chunksize = 8192
STAT_WAITING = 0
STAT_FAIL = 1
STAT_STARTED = 2
STAT_FINISH_SUCCESS = 3
TASK_EMAIL = 1
TASK_CONVERT = 2
RET_FAIL = 0
RET_SUCCESS = 1
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to
def get_attachment(bookpath, filename):
"""Get file as MIMEBase message"""
calibrepath = web.config.config_calibre_dir
if web.ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(bookpath, filename)
if df:
datafile = os.path.join(calibrepath, bookpath, filename)
if not os.path.exists(os.path.join(calibrepath, bookpath)):
os.makedirs(os.path.join(calibrepath, bookpath))
df.GetContentFile(datafile)
else:
return None
file_ = open(datafile, 'rb')
data = file_.read()
file_.close()
os.remove(datafile)
else:
try:
file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
data = file_.read()
file_.close()
except IOError:
web.app.logger.exception(e) # traceback.print_exc()
web.app.logger.error(u'The requested file could not be read. Maybe wrong permissions?')
return None
attachment = MIMEBase('application', 'octet-stream')
attachment.set_payload(data)
encoders.encode_base64(attachment)
attachment.add_header('Content-Disposition', 'attachment',
filename=filename)
return attachment
# Class for sending email with ability to get current progress
class email(smtplib.SMTP):
transferSize = 0
progress = 0
def __init__(self, *args, **kwargs):
smtplib.SMTP.__init__(self, *args, **kwargs)
def data(self, msg):
self.transferSize = len(msg)
(code, resp) = smtplib.SMTP.data(self, msg)
self.progress = 0
return (code, resp)
def send(self, strg):
"""Send `strg' to the server."""
if self.debuglevel > 0:
print('send:', repr(strg), file=sys.stderr)
if hasattr(self, 'sock') and self.sock:
try:
if self.transferSize:
lock=threading.Lock()
lock.acquire()
self.transferSize = len(strg)
lock.release()
for i in range(0, self.transferSize, chunksize):
self.sock.send(strg[i:i+chunksize])
lock.acquire()
self.progress = i
lock.release()
else:
self.sock.sendall(strg)
except socket.error:
self.close()
raise smtplib.SMTPServerDisconnected('Server not connected')
else:
raise smtplib.SMTPServerDisconnected('please run connect() first')
def getTransferStatus(self):
if self.transferSize:
lock2 = threading.Lock()
lock2.acquire()
value = round(float(self.progress) / float(self.transferSize),2)*100
lock2.release()
return str(value) + ' %'
else:
return "100 %"
# Class for sending ssl encrypted email with ability to get current progress
class email_SSL(email):
def __init__(self, *args, **kwargs):
smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
#Class for all worker tasks in the background
class WorkerThread(threading.Thread):
def __init__(self):
self._stopevent = threading.Event()
threading.Thread.__init__(self)
self.status = 0
self.current = 0
self.last = 0
self.queue = list()
self.UIqueue = list()
self.asyncSMTP=None
self.id = 0
# Main thread loop starting the different tasks
def run(self):
while not self._stopevent.isSet():
doLock = threading.Lock()
doLock.acquire()
if self.current != self.last:
doLock.release()
if self.queue[self.current]['typ'] == TASK_EMAIL:
self.send_raw_email()
if self.queue[self.current]['typ'] == TASK_CONVERT:
self.convert_mobi()
self.current += 1
else:
doLock.release()
time.sleep(1)
def stop(self):
self._stopevent.set()
def get_send_status(self):
if self.asyncSMTP:
return self.asyncSMTP.getTransferStatus()
else:
return "0 %"
def delete_completed_tasks(self):
for index, task in reversed(list(enumerate(self.UIqueue))):
if task['progress'] == "100 %":
# delete tasks
self.queue.pop(index)
self.UIqueue.pop(index)
# if we are deleting entries before the current index, adjust the index
self.current -= 1
self.last = len(self.queue)
def get_taskstatus(self):
if self.current < len(self.queue):
if self.queue[self.current]['status'] == STAT_STARTED:
if not self.queue[self.current]['typ'] == TASK_CONVERT:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def convert_mobi(self):
# convert book, and upload in case of google drive
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if web.ub.config.config_ebookconverter == 2:
filename = self.convert_calibre()
else:
filename = self.convert_kindlegen()
if web.ub.config.config_use_google_drive:
gd.updateGdriveCalibreFromLocal()
if(filename):
self.add_email(_(u'Send to Kindle'), self.queue[self.current]['path'], filename,
self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
self.UIqueue[self.current]['user'], _(u"E-Mail: %s" % self.queue[self.current]['title']))
def convert_kindlegen(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
if not os.path.exists(web.ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=web.ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return
try:
p = subprocess.Popen(
(web.ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
web.app.logger.info("convert_kindlegen: " + error_message)
web.app.logger.info(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
cur_book.data.append(new_format)
web.db.session.commit()
self.queue[self.current]['path'] = cur_book.path
self.queue[self.current]['title'] = cur_book.title
if web.ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return file_path + ".mobi" #, RET_SUCCESS
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
def convert_calibre(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
if not os.path.exists(web.ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found",
converterpath=web.ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return
try:
command = ("\"" + web.ub.config.config_converterpath + "\" \"" + file_path + u".epub\" \""
+ file_path + u".mobi\" " + web.ub.config.config_calibre).encode(sys.getfilesystemencoding())
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
progress = re.search("(\d+)%\s.*", nextline)
if progress:
self.UIqueue[self.current]['progress'] = progress.group(1) + '%'
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0:
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
cur_book.data.append(new_format)
web.db.session.commit()
self.queue[self.current]['path'] = cur_book.path
self.queue[self.current]['title'] = cur_book.title
if web.ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return file_path + ".mobi" # , RET_SUCCESS
else:
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
def add_convert(self, file_path, bookid, user_name, type, settings, kindle_mail):
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.id += 1
self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle':kindle_mail,
'status': STAT_WAITING, 'typ': TASK_CONVERT, 'settings':settings})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': type,
'runtime': '0 s', 'status': _('Waiting'),'id': self.id } )
self.id += 1
self.last=len(self.queue)
addLock.release()
def add_email(self, subject, filepath, attachment, settings, recipient, user_name, typ):
# if more than 20 entries in the list, clean the list
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.queue.append({'subject':subject, 'attachment':attachment, 'filepath':filepath,
'settings':settings, 'recipent':recipient, 'starttime': 0,
'status': STAT_WAITING, 'typ': TASK_EMAIL})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ,
'runtime': '0 s', 'status': _('Waiting'),'id': self.id })
self.id += 1
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
obj=self.queue[self.current]
# create MIME message
msg = MIMEMultipart()
msg['Subject'] = self.queue[self.current]['subject']
msg['Message-Id'] = make_msgid('calibre-web')
msg['Date'] = formatdate(localtime=True)
text = _(u'This email has been sent via calibre web.')
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
if obj['attachment']:
result = get_attachment(obj['filepath'], obj['attachment'])
if result:
msg.attach(result)
else:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return
msg['From'] = obj['settings']["mail_from"]
msg['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(msg)
msg = fp.getvalue()
# send email
try:
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
sys.stderr = StderrLogger()
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
# link to logginglevel
if web.ub.config.config_log_level != logging.DEBUG:
self.asyncSMTP.set_debuglevel(0)
else:
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], msg)
self.asyncSMTP.quit()
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
sys.stderr = org_stderr
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException):
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
web.app.logger.error(e)
# return None
def _formatRuntime(self, runtime):
self.UIqueue[self.current]['rt'] = runtime.total_seconds()
val = re.split('\:|\.', str(runtime))[0:3]
erg = list()
for v in val:
if int(v) > 0:
erg.append(v)
retVal = (':'.join(erg)).lstrip('0') + ' s'
if retVal == ' s':
retVal = '0 s'
return retVal
class StderrLogger(object):
buffer = ''
def __init__(self):
self.logger = web.app.logger
def write(self, message):
if message == '\n':
self.logger.debug(self.buffer)
print(self.buffer)
self.buffer = ''
else:
self.buffer += message