NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Converting ebooks in background

Browse Source 
additional sorting of tasklist according to date and runtime
codecosmetics
This commit is contained in:
Ozzie Isaacs 2018-08-12 09:29:57 +02:00
parent 11b798a01c
commit 7be328c535
16 changed files with 825 additions and 737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
cps/asyncmail.py
View File

@ -1,241 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import smtplib
import threading
from datetime import datetime
import logging
import time
import socket
import sys
from email.generator import Generator
import web
from flask_babel import gettext as _
import re
try:
from StringIO import StringIO
except ImportError as e:
from io import StringIO
chunksize = 8192
STAT_WAITING = 0
STAT_FAIL = 1
STAT_STARTED = 2
STAT_FINISH_SUCCESS = 3
class email(smtplib.SMTP):
transferSize = 0
progress = 0
def __init__(self, *args, **kwargs):
smtplib.SMTP.__init__(self, *args, **kwargs)
def data(self, msg):
self.transferSize = len(msg)
(code, resp) = smtplib.SMTP.data(self, msg)
self.progress = 0
return (code, resp)
def send(self, str):
"""Send `str' to the server."""
if self.debuglevel > 0:
print('send:', repr(str), file=sys.stderr)
if hasattr(self, 'sock') and self.sock:
try:
if self.transferSize:
lock=threading.Lock()
lock.acquire()
self.transferSize = len(str)
lock.release()
for i in range(0, self.transferSize, chunksize):
self.sock.send(str[i:i+chunksize])
lock.acquire()
self.progress = i
lock.release()
else:
self.sock.sendall(str)
except socket.error:
self.close()
raise smtplib.SMTPServerDisconnected('Server not connected')
else:
raise smtplib.SMTPServerDisconnected('please run connect() first')
def getTransferStatus(self):
if self.transferSize:
lock2 = threading.Lock()
lock2.acquire()
value = round(float(self.progress) / float(self.transferSize),2)*100
lock2.release()
return str(value) + ' %'
else:
return "100 %"
class email_SSL(email):
def __init__(self, *args, **kwargs):
smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
class EMailThread(threading.Thread):
def __init__(self):
self._stopevent = threading.Event()
threading.Thread.__init__(self)
self.status = 0
self.current = 0
self.last = 0
self.queue=list()
self.UIqueue = list()
self.asyncSMTP=None
def run(self):
while not self._stopevent.isSet():
doLock = threading.Lock()
doLock.acquire()
if self.current != self.last:
doLock.release()
self.send_raw_email()
self.current += 1
else:
doLock.release()
time.sleep(1)
def stop(self):
self._stopevent.set()
def get_send_status(self):
if self.asyncSMTP:
return self.asyncSMTP.getTransferStatus()
else:
return "0 %"
def delete_completed_tasks(self):
# muss gelockt werden
for index, task in reversed(list(enumerate(self.UIqueue))):
if task['progress'] == "100 %":
# delete tasks
self.queue.pop(index)
self.UIqueue.pop(index)
# if we are deleting entries before the current index, adjust the index
# if self.current >= index:
self.current -= 1
self.last = len(self.queue)
def get_taskstatus(self):
if self.current < len(self.queue):
if self.queue[self.current]['status'] == STAT_STARTED:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def add_email(self, data, settings, recipient, user_name, type):
# if more than 50 entries in the list, clean the list
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.queue.append({'data':data, 'settings':settings, 'recipent':recipient, 'starttime': 0,
'status': STAT_WAITING})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': type,
'runtime': '0 s', 'status': _('Waiting') })
# access issue
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
obj=self.queue[self.current]
# settings = ub.get_mail_settings()
obj['data']['From'] = obj['settings']["mail_from"]
obj['data']['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(obj['data'])
obj['data'] = fp.getvalue()
# send email
try:
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
#org_stderr2 = smtplib.stderr
sys.stderr = StderrLogger()
#smtplib.stderr = StderrLogger()
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
# link to logginglevel
if web.ub.config.config_log_level != logging.DEBUG:
self.asyncSMTP.set_debuglevel(0)
else:
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], obj['data'])
self.asyncSMTP.quit()
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
sys.stderr = org_stderr
#smtplib.stderr = org_stderr2
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
web.app.logger.error(e)
return None
def _formatRuntime(self, runtime):
val = re.split('\:|\.', str(runtime))[0:3]
erg = list()
for v in val:
if int(v) > 0:
erg.append(v)
retVal = (':'.join(erg)).lstrip('0') + ' s'
if retVal == ' s':
retVal = '0 s'
return retVal
class StderrLogger(object):
buffer = ''
def __init__(self):
self.logger = web.app.logger
def write(self, message):
if message == '\n':
self.logger.debug(self.buffer)
print(self.buffer)
self.buffer = ''
else:
self.buffer += message

104
cps/converter.py
View File

@ -1,19 +1,12 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import subprocess
import ub
import db
import re
import web
from flask_babel import gettext as _
RET_FAIL = 0
RET_SUCCESS = 1
def versionKindle():
versions = _(u'not installed')
if os.path.exists(ub.config.config_converterpath):
@ -46,97 +39,6 @@ def versionCalibre():
return {'Calibre converter' : versions}
def convert_kindlegen(file_path, book):
error_message = None
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
try:
p = subprocess.Popen((ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
web.app.logger.info("convert_kindlegen: " + error_message)
web.app.logger.info(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
if ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
return error_message, RET_FAIL
def convert_calibre(file_path, book):
error_message = None
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found", converterpath=ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
try:
command = ("\""+ub.config.config_converterpath + "\" \"" + file_path + u".epub\" \""
+ file_path + u".mobi\" " + ub.config.config_calibre).encode(sys.getfilesystemencoding())
p = subprocess.Popen(command,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0 :
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
if ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
return error_message, RET_FAIL
def versioncheck():
if ub.config.config_ebookconverter == 1:
return versionKindle()
@ -145,9 +47,3 @@ def versioncheck():
else:
return {'ebook_converter':''}
def convert_mobi(file_path, book):
if ub.config.config_ebookconverter == 2:
return convert_calibre(file_path, book)
else:
return convert_kindlegen(file_path, book)

9
cps/db.py
View File

@ -321,7 +321,7 @@ def setup_db():
try:
if not os.path.exists(dbpath):
raise
engine = create_engine('sqlite:///' + dbpath, echo=False, isolation_level="SERIALIZABLE")
engine = create_engine('sqlite:///' + dbpath, echo=False, isolation_level="SERIALIZABLE", connect_args={'check_same_thread': False})
conn = engine.connect()
except Exception:
content = ub.session.query(ub.Settings).first()
@ -381,8 +381,9 @@ def setup_db():
secondary=books_custom_column_links[cc_id[0]],
backref='books'))
# Base.metadata.create_all(engine)
Session = sessionmaker()
Session.configure(bind=engine)
Session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
session = Session()
return True

20
cps/gdriveutils.py
View File

@ -5,6 +5,7 @@ try:
from apiclient import errors
except ImportError:
pass
import os
from ub import config
import cli
@ -179,9 +180,8 @@ def getEbooksFolderId(drive=None):
gDriveId = GdriveId()
try:
gDriveId.gdrive_id = getEbooksFolder(drive)['id']
except:
pass
# ToDo Path not exisiting
except Exception:
web.app.logger.error('Error gDrive, root ID not found')
gDriveId.path = '/'
session.merge(gDriveId)
session.commit()
@ -282,19 +282,6 @@ def moveGdriveFolderRemote(origin_file, target_folder):
# drive.auth.service.files().delete(fileId=previous_parents).execute()
#def downloadFile(path, filename, output):
# f = getFileFromEbooksFolder(path, filename)
# return f.GetContentFile(output)
# ToDo: Check purpose Parameter f ??, purpose of function ?
def backupCalibreDbAndOptionalDownload(drive):
drive = getDrive(drive)
metaDataFile = "'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId()
fileList = drive.ListFile({'q': metaDataFile}).GetList()
#databaseFile = fileList[0]
#if f:
# databaseFile.GetContentFile(f)
def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
ignoreFiles=None,
@ -447,7 +434,6 @@ def deleteDatabaseOnChange():
session.commit()
def updateGdriveCalibreFromLocal():
# backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive)
copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True)
for x in os.listdir(config.config_calibre_dir):
if os.path.isdir(os.path.join(config.config_calibre_dir, x)):

144
cps/helper.py
View File

@ -8,28 +8,12 @@ import logging
from tempfile import gettempdir
import sys
import os
import traceback
import re
import unicodedata
from io import BytesIO
import converter
import asyncmail
import worker
import time
try:
from StringIO import StringIO
from email.MIMEBase import MIMEBase
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
except ImportError as e:
from io import StringIO
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email import encoders
from email.utils import formatdate
from email.utils import make_msgid
from flask import send_from_directory, make_response, redirect, abort
from flask_babel import gettext as _
import threading
@ -51,29 +35,25 @@ except ImportError:
# Global variables
updater_thread = None
global_eMailThread = asyncmail.EMailThread()
global_eMailThread.start()
RET_SUCCESS = 1
RET_FAIL = 0
global_WorkerThread = worker.WorkerThread()
global_WorkerThread.start()
def update_download(book_id, user_id):
check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id ==
book_id).first()
if not check:
new_download = ub.Downloads(user_id=user_id, book_id=book_id)
ub.session.add(new_download)
ub.session.commit()
def make_mobi(book_id, calibrepath):
def make_mobi(book_id, calibrepath, user_id, kindle_mail):
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
if not data:
error_message = _(u"epub format not found for book id: %(book)d", book=book_id)
app.logger.error("make_mobi: " + error_message)
return error_message, RET_FAIL
return error_message
if ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(book.path, data.name + u".epub")
if df:
@ -82,120 +62,59 @@ def make_mobi(book_id, calibrepath):
os.makedirs(os.path.join(calibrepath, book.path))
df.GetContentFile(datafile)
else:
error_message = "make_mobi: epub not found on gdrive: %s.epub" % data.name
return error_message, RET_FAIL
# else:
error_message = (u"make_mobi: epub not found on gdrive: %s.epub" % data.name)
return error_message
file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + u".epub"):
# convert book, and upload in case of google drive
res = converter.convert_mobi(file_path, book)
if ub.config.config_use_google_drive:
gd.updateGdriveCalibreFromLocal()
# time.sleep(10)
return res
# append converter to queue
global_WorkerThread.add_convert(file_path, book.id, user_id, _(u"Convert: %s" % book.title), ub.get_mail_settings(),
kindle_mail)
return None
else:
error_message = "make_mobi: epub not found: %s.epub" % file_path
return error_message, RET_FAIL
error_message = (u"make_mobi: epub not found: %s.epub" % file_path)
return error_message
def send_test_mail(kindle_mail, user_name):
msg = MIMEMultipart()
msg['Subject'] = _(u'Calibre-web test email')
text = _(u'This email has been sent via calibre web.')
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name, _('Test E-Mail'))
return # send_raw_email(kindle_mail, msg)
global_WorkerThread.add_email(_(u'Calibre-web test email'),None, None, ub.get_mail_settings(),
kindle_mail, user_name, _(u"Test E-Mail"))
return
# Files are processed in the following order/priority:
# 1: If Mobi file is exisiting, it's directly send to kindle email,
# 2: If Epub file is exisiting, it's converted and send to kindle email
# 3: If Pdf file is exisiting, it's directly send to kindle email,
def send_mail(book_id, kindle_mail, calibrepath, user_id):
"""Send email with attachments"""
# create MIME message
result= None
msg = MIMEMultipart()
msg['Subject'] = _(u'Send to Kindle')
msg['Message-Id'] = make_msgid('calibre-web')
msg['Date'] = formatdate(localtime=True)
text = _(u'This email has been sent via calibre web.')
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).all()
formats = {}
index = 0
for indx,entry in enumerate(data):
for entry in data:
if entry.format == "MOBI":
formats["mobi"] = entry.name + ".mobi"
if entry.format == "EPUB":
formats["epub"] = entry.name + ".epub"
index = indx
if entry.format == "PDF":
formats["pdf"] = entry.name + ".pdf"
if len(formats) == 0:
return _("Could not find any formats suitable for sending by email")
return _(u"Could not find any formats suitable for sending by email")
if 'mobi' in formats:
result = get_attachment(calibrepath, book.path, formats['mobi'])
if result:
msg.attach(result)
result = formats['mobi']
elif 'epub' in formats:
# returns filename if sucess, otherwise errormessage
data, resultCode = make_mobi(book.id, calibrepath)
if resultCode == RET_SUCCESS:
result = get_attachment(calibrepath, book.path, os.path.basename(data))
if result:
msg.attach(result)
else:
app.logger.error(data)
return data
# returns None if sucess, otherwise errormessage
return make_mobi(book.id, calibrepath, user_id, kindle_mail)
elif 'pdf' in formats:
result = get_attachment(calibrepath, book.path, formats['pdf'])
result = formats['pdf'] # worker.get_attachment()
else:
return _(u"Could not find any formats suitable for sending by email")
if result:
msg.attach(result)
global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, ub.get_mail_settings(),
kindle_mail, user_id, _(u"E-Mail: %s" % book.title))
else:
return _("Could not find any formats suitable for sending by email")
if result:
global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id, _(u"E-Mail: %s" % book.title))
return None # send_raw_email(kindle_mail, msg)
else:
return _('The requested file could not be read. Maybe wrong permissions?')
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to
def get_attachment(calibrepath, bookpath, filename):
"""Get file as MIMEBase message"""
if ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(bookpath, filename)
if df:
datafile = os.path.join(calibrepath, bookpath, filename)
if not os.path.exists(os.path.join(calibrepath, bookpath)):
os.makedirs(os.path.join(calibrepath, bookpath))
df.GetContentFile(datafile)
else:
return None
file_ = open(datafile, 'rb')
data = file_.read()
file_.close()
os.remove(datafile)
else:
try:
file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
data = file_.read()
file_.close()
except IOError:
traceback.print_exc()
app.logger.error = u'The requested file could not be read. Maybe wrong permissions?'
return None
attachment = MIMEBase('application', 'octet-stream')
attachment.set_payload(data)
encoders.encode_base64(attachment)
attachment.add_header('Content-Disposition', 'attachment',
filename=filename)
return attachment
return _(u"The requested file could not be read. Maybe wrong permissions?")
def get_valid_filename(value, replace_whitespace=True):
@ -225,7 +144,6 @@ def get_valid_filename(value, replace_whitespace=True):
value = value[:128]
if not value:
raise ValueError("Filename cannot be empty")
return value

11
cps/server.py
View File

@ -44,8 +44,8 @@ class server:
web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...')
self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args)
self.wsgiserver.serve_forever()
except:
pass
except Exception:
web.app.logger.info("Unknown error while starting gevent")
def startServer(self):
if gevent_present:
@ -70,7 +70,7 @@ class server:
if self.restart == True:
web.app.logger.info("Performing restart of Calibre-web")
web.helper.global_eMailThread.stop()
web.helper.global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""]
for e in sys.argv:
@ -80,7 +80,7 @@ class server:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
web.app.logger.info("Performing shutdown of Calibre-web")
web.helper.global_eMailThread.stop()
web.helper.global_WorkerThread.stop()
sys.exit(0)
def setRestartTyp(self,starttyp):
@ -92,7 +92,8 @@ class server:
else:
self.wsgiserver.add_callback(self.wsgiserver.stop)
def getNameVersion(self):
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent':'v'+geventVersion}
else:

3
cps/static/js/archive.js
View File

@ -148,8 +148,7 @@ bitjs.archive = bitjs.archive || {};
currentBytesUnarchivedInFile,
currentBytesUnarchived,
totalUncompressedBytesInArchive,
totalFilesInArchive)
{
totalFilesInArchive) {
bitjs.base(this, bitjs.archive.UnarchiveEvent.Type.PROGRESS);
this.currentFilename = currentFilename;

5
cps/static/js/io.js
View File

@ -121,7 +121,7 @@ bitjs.io = bitjs.io || {};
* @return {number} The peeked bits, as an unsigned number.
*/
bitjs.io.BitStream.prototype.peekBitsRtl = function(n, movePointers) {
if (n <= 0 || typeof n != typeof 1) {
if (n <= 0 || typeof n !== typeof 1) {
return 0;
}
@ -150,8 +150,7 @@ bitjs.io = bitjs.io || {};
bytePtr++;
bitPtr = 0;
n -= numBitsLeftInThisByte;
}
else {
} else {
result <<= n;
result |= ((bytes[bytePtr] & (BITMASK[n] << (8 - n - bitPtr))) >> (8 - n - bitPtr));

66
cps/static/js/unrar.js
View File

@ -146,7 +146,7 @@ var RarVolumeHeader = function(bstream) {
} else {
this.HighPackSize = 0;
this.HighUnpSize = 0;
if (this.unpackedSize == 0xffffffff) {
if (this.unpackedSize === 0xffffffff) {
this.HighUnpSize = 0x7fffffff;
this.unpackedSize = 0xffffffff;
}
@ -178,9 +178,10 @@ var RarVolumeHeader = function(bstream) {
// this is adapted straight out of arcread.cpp, Archive::ReadHeader()
for (var I = 0; I < 4; ++I) {
var rmode = extTimeFlags >> ((3 - I) * 4);
if ((rmode & 8)==0)
if ((rmode & 8) === 0) {
continue;
if (I!=0) {
}
if (I !== 0) {
bstream.readBits(16);
}
var count = (rmode & 3);
@ -209,7 +210,6 @@ var RarVolumeHeader = function(bstream) {
};
var BLOCK_LZ = 0;
// BLOCK_PPM = 1;
var rLDecode = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224],
rLBits = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5],
@ -236,7 +236,7 @@ var rNC = 299,
rBC = 20,
rHUFF_TABLE_SIZE = (rNC + rDC + rRC + rLDC);
var UnpBlockType = BLOCK_LZ;
//var UnpBlockType = BLOCK_LZ;
var UnpOldTable = new Array(rHUFF_TABLE_SIZE);
var BD = { //bitdecode
@ -281,26 +281,26 @@ function RarReadTables(bstream) {
}
if (!bstream.readBits(1)) { //discard old table
for (var i = UnpOldTable.length; i--;) UnpOldTable[i] = 0;
var i;
for (i = UnpOldTable.length; i--;) UnpOldTable[i] = 0;
}
// read in bit lengths
for (var I = 0; I < rBC; ++I) {
var Length = bstream.readBits(4);
if (Length == 15) {
if (Length === 15) {
var ZeroCount = bstream.readBits(4);
if (ZeroCount == 0) {
if (ZeroCount === 0) {
BitLength[I] = 15;
}
else {
} else {
ZeroCount += 2;
while (ZeroCount-- > 0 && I < rBC)
while (ZeroCount-- > 0 && I < rBC) {
BitLength[I++] = 0;
}
--I;
}
}
else {
} else {
BitLength[I] = Length;
}
}
@ -311,20 +311,20 @@ function RarReadTables(bstream) {
var TableSize = rHUFF_TABLE_SIZE;
//console.log(DecodeLen, DecodePos, DecodeNum);
for (var i = 0; i < TableSize;) {
for (i = 0; i < TableSize;) {
var num = RarDecodeNumber(bstream, BD);
if (num < 16) {
Table[i] = (num + UnpOldTable[i]) & 0xf;
i++;
} else if (num < 18) {
var N = (num == 16) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11);
var N = (num === 16) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11);
while (N-- > 0 && i < TableSize) {
Table[i] = Table[i - 1];
i++;
}
} else {
var N = (num == 18) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11);
var N = (num === 18) ? (bstream.readBits(3) + 3) : (bstream.readBits(7) + 11);
while (N-- > 0 && i < TableSize) {
Table[i++] = 0;
@ -337,7 +337,7 @@ function RarReadTables(bstream) {
RarMakeDecodeTables(Table, rNC + rDC, LDD, rLDC);
RarMakeDecodeTables(Table, rNC + rDC + rLDC, RD, rRC);
for (var i = UnpOldTable.length; i--;) {
for (i = UnpOldTable.length; i--;) {
UnpOldTable[i] = Table[i];
}
return true;
@ -433,14 +433,14 @@ function Unpack20(bstream, Solid) {
RarCopyString(Length, Distance);
continue;
}
if (num == 269) {
if (num === 269) {
RarReadTables20(bstream);
RarUpdateProgress()
continue;
}
if (num == 256) {
if (num === 256) {
lastDist = rOldDist[oldDistPtr++ & 3] = lastDist;
RarCopyString(lastLength, lastDist);
continue;
@ -511,14 +511,14 @@ function RarReadTables20(bstream) {
if (num < 16) {
Table[I] = num + UnpOldTable20[I] & 0xf;
I++;
} else if(num == 16) {
} else if(num === 16) {
N = bstream.readBits(2) + 3;
while (N-- > 0 && I < TableSize) {
Table[I] = Table[I - 1];
I++;
}
} else {
if (num == 17) {
if (num === 17) {
N = bstream.readBits(3) + 3;
} else {
N = bstream.readBits(7) + 11;
@ -595,7 +595,7 @@ function Unpack29(bstream, Solid) {
Distance += prevLowDist;
} else {
var LowDist = RarDecodeNumber(bstream, LDD);
if (LowDist == 16) {
if (LowDist === 16) {
lowDistRepCount = rLOW_DIST_REP_COUNT - 1;
Distance += prevLowDist;
} else {
@ -618,16 +618,16 @@ function Unpack29(bstream, Solid) {
RarCopyString(Length, Distance);
continue;
}
if (num == 256) {
if (num === 256) {
if (!RarReadEndOfBlock(bstream)) break;
continue;
}
if (num == 257) {
if (num === 257) {
//console.log("READVMCODE");
if (!RarReadVMCode(bstream)) break;
continue;
}
if (num == 258) {
if (num === 258) {
if (lastLength != 0) {
RarCopyString(lastLength, lastDist);
}
@ -684,9 +684,9 @@ function RarReadEndOfBlock(bstream) {
function RarReadVMCode(bstream) {
var FirstByte = bstream.readBits(8);
var Length = (FirstByte & 7) + 1;
if (Length == 7) {
if (Length === 7) {
Length = bstream.readBits(8) + 7;
} else if(Length == 8) {
} else if(Length === 8) {
Length = bstream.readBits(16);
}
var vmCode = [];
@ -789,7 +789,7 @@ RarLocalFile.prototype.unrar = function() {
if (!this.header.flags.LHD_SPLIT_BEFORE) {
// unstore file
if (this.header.method == 0x30) {
if (this.header.method === 0x30) {
info("Unstore " + this.filename);
this.isValid = true;
@ -820,10 +820,10 @@ var unrar = function(arrayBuffer) {
var bstream = new bitjs.io.BitStream(arrayBuffer, false /* rtl */);
var header = new RarVolumeHeader(bstream);
if (header.crc == 0x6152 &&
header.headType == 0x72 &&
header.flags.value == 0x1A21 &&
header.headSize == 7)
if (header.crc === 0x6152 &&
header.headType === 0x72 &&
header.flags.value === 0x1A21 &&
header.headSize === 7)
{
info("Found RAR signature");
@ -840,7 +840,7 @@ var unrar = function(arrayBuffer) {
if (localFile && localFile.isValid && localFile.header.packSize > 0) {
totalUncompressedBytesInArchive += localFile.header.unpackedSize;
localFiles.push(localFile);
} else if (localFile.header.packSize == 0 && localFile.header.unpackedSize == 0) {
} else if (localFile.header.packSize === 0 && localFile.header.unpackedSize === 0) {
localFile.isValid = true;
}
} catch(err) {

15
cps/static/js/untar.js
View File

@ -7,10 +7,11 @@
*
* TAR format: http://www.gnu.org/software/automake/manual/tar/Standard.html
*/
/* global bitjs, importScripts, Uint8Array */
// This file expects to be invoked as a Worker (see onmessage below).
importScripts('io.js');
importScripts('archive.js');
importScripts("io.js");
importScripts("archive.js");
// Progress variables.
var currentFilename = "";
@ -41,7 +42,7 @@ var postProgress = function() {
var readCleanString = function(bstr, numBytes) {
var str = bstr.readString(numBytes);
var zIndex = str.indexOf(String.fromCharCode(0));
return zIndex != -1 ? str.substr(0, zIndex) : str;
return zIndex !== -1 ? str.substr(0, zIndex) : str;
};
// takes a ByteStream and parses out the local file information
@ -60,7 +61,7 @@ var TarLocalFile = function(bstream) {
this.linkname = readCleanString(bstream, 100);
this.maybeMagic = readCleanString(bstream, 6);
if (this.maybeMagic == "ustar") {
if (this.maybeMagic === "ustar") {
this.version = readCleanString(bstream, 2);
this.uname = readCleanString(bstream, 32);
this.gname = readCleanString(bstream, 32);
@ -85,7 +86,7 @@ var TarLocalFile = function(bstream) {
info(" typeflag = " + this.typeflag);
// A regular file.
if (this.typeflag == 0) {
if (this.typeflag === 0) {
info(" This is a regular file.");
var sizeInBytes = parseInt(this.size);
this.fileData = new Uint8Array(bstream.bytes.buffer, bstream.ptr, this.size);
@ -100,7 +101,7 @@ var TarLocalFile = function(bstream) {
if (remaining > 0 && remaining < 512) {
bstream.readBytes(remaining);
}
} else if (this.typeflag == 5) {
} else if (this.typeflag === 5) {
info(" This is a directory.")
}
};
@ -121,7 +122,7 @@ var untar = function(arrayBuffer) {
var localFiles = [];
// While we don't encounter an empty block, keep making TarLocalFiles.
while (bstream.peekNumber(4) != 0) {
while (bstream.peekNumber(4) !== 0) {
var oneLocalFile = new TarLocalFile(bstream);
if (oneLocalFile && oneLocalFile.isValid) {
localFiles.push(oneLocalFile);

40
cps/static/js/unzip.js
View File

@ -49,7 +49,7 @@ var zDigitalSignatureSignature = 0x05054b50;
// takes a ByteStream and parses out the local file information
var ZipLocalFile = function(bstream) {
if (typeof bstream !== typeof {} || !bstream.readNumber || typeof bstream.readNumber != typeof function() {} ) {
if (typeof bstream !== typeof {} || !bstream.readNumber || typeof bstream.readNumber !== typeof function() {} ) {
return null;
}
@ -115,9 +115,8 @@ ZipLocalFile.prototype.unzip = function() {
info("ZIP v" + this.version + ", store only: " + this.filename + " (" + this.compressedSize + " bytes)");
currentBytesUnarchivedInFile = this.compressedSize;
currentBytesUnarchived += this.compressedSize;
}
} else if (this.compressionMethod === 8) {
// version == 20, compression method == 8 (DEFLATE)
else if (this.compressionMethod === 8) {
info("ZIP v2.0, DEFLATE: " + this.filename + " (" + this.compressedSize + " bytes)");
this.fileData = inflate(this.fileData, this.uncompressedSize);
}
@ -245,6 +244,7 @@ var unzip = function(arrayBuffer) {
// each entry's index is its code and its value is a JavaScript object
// containing {length: 6, symbol: X}
function getHuffmanCodes(bitLengths) {
var len;
// ensure bitLengths is an array containing at least one element
if (typeof bitLengths !== typeof [] || bitLengths.length < 1) {
err("Error! getHuffmanCodes() called with an invalid array");
@ -256,9 +256,10 @@ function getHuffmanCodes(bitLengths) {
blCount = [],
MAX_BITS = 1;
// Step 1: count up how many codes of each length we have
for (var i = 0; i < numLengths; ++i) {
var len = bitLengths[i];
len = bitLengths[i];
// test to ensure each bit length is a positive, non-zero number
if (typeof len !== typeof 1 || len < 0) {
err("bitLengths contained an invalid number in getHuffmanCodes(): " + len + " of type " + (typeof len));
@ -276,9 +277,9 @@ function getHuffmanCodes(bitLengths) {
var nextCode = [],
code = 0;
for (var bits = 1; bits <= MAX_BITS; ++bits) {
var len = bits-1;
len = bits - 1;
// ensure undefined lengths are zero
if (blCount[len] == undefined) blCount[len] = 0;
if (blCount[len] === undefined) blCount[len] = 0;
code = (code + blCount[bits - 1]) << 1;
nextCode[bits] = code;
}
@ -286,7 +287,7 @@ function getHuffmanCodes(bitLengths) {
// Step 3: Assign numerical values to all codes
var table = {}, tableLength = 0;
for (var n = 0; n < numLengths; ++n) {
var len = bitLengths[n];
len = bitLengths[n];
if (len !== 0) {
table[nextCode[len]] = { length: len, symbol: n }; //, bitstring: binaryValueToString(nextCode[len],len) };
tableLength++;
@ -318,13 +319,14 @@ function getHuffmanCodes(bitLengths) {
var fixedHCtoLiteral = null;
var fixedHCtoDistance = null;
function getFixedLiteralTable() {
var i;
// create once
if (!fixedHCtoLiteral) {
var bitlengths = new Array(288);
for (var i = 0; i <= 143; ++i) bitlengths[i] = 8;
for (var i = 144; i <= 255; ++i) bitlengths[i] = 9;
for (var i = 256; i <= 279; ++i) bitlengths[i] = 7;
for (var i = 280; i <= 287; ++i) bitlengths[i] = 8;
for (i = 0; i <= 143; ++i) bitlengths[i] = 8;
for (i = 144; i <= 255; ++i) bitlengths[i] = 9;
for (i = 256; i <= 279; ++i) bitlengths[i] = 7;
for (i = 280; i <= 287; ++i) bitlengths[i] = 8;
// get huffman code table
fixedHCtoLiteral = getHuffmanCodes(bitlengths);
@ -359,7 +361,7 @@ function decodeSymbol(bstream, hcTable) {
++len;
// check against Huffman Code table and break if found
if (hcTable.hasOwnProperty(code) && hcTable[code].length == len) {
if (hcTable.hasOwnProperty(code) && hcTable[code].length === len) {
break;
}
@ -388,8 +390,8 @@ var CodeLengthCodeOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2
264 0 10 274 3 43-50 284 5 227-257
265 1 11,12 275 3 51-58 285 0 258
266 1 13,14 276 3 59-66
*/
var LengthLookupTable = [
[0, 3], [0, 4], [0, 5], [0, 6],
[0, 7], [0, 8], [0, 9], [0, 10],
@ -448,10 +450,10 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
stream, and copy length bytes from this
position to the output stream.
*/
var numSymbols = 0, blockSize = 0;
var blockSize = 0;
for (;;) {
var symbol = decodeSymbol(bstream, hcLiteralTable);
++numSymbols;
// ++numSymbols;
if (symbol < 256) {
// copy literal byte to output
buffer.insertByte(symbol);
@ -487,7 +489,7 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
buffer.insertByte(data[ch++]);
}
} else {
buffer.insertBytes(buffer.data.subarray(ch, ch + length))
buffer.insertBytes(buffer.data.subarray(ch, ch + length));
}
} // length-distance pair
@ -506,7 +508,7 @@ function inflate(compressedData, numDecompressedBytes) {
compressedData.byteOffset,
compressedData.byteLength);
var buffer = new bitjs.io.ByteBuffer(numDecompressedBytes);
var numBlocks = 0;
//var numBlocks = 0;
var blockSize = 0;
// block format: http://tools.ietf.org/html/rfc1951#page-9
@ -514,9 +516,9 @@ function inflate(compressedData, numDecompressedBytes) {
var bFinal = bstream.readBits(1);
var bType = bstream.readBits(2);
blockSize = 0;
++numBlocks;
// ++numBlocks;
// no compression
if (bType == 0) {
if (bType === 0) {
// skip remaining bits in this byte
while (bstream.bitPtr != 0) bstream.readBits(1);
var len = bstream.readBits(16),

8
cps/templates/list.html
View File

@ -2,15 +2,15 @@
{% block body %}
<h1>{{title}}</h1>
<div class="container">
<div class="col-sm-6">
<div class="col-xs-12 col-sm-6">
{% for entry in entries %}
{% if loop.index0 == (loop.length/2)|int and loop.length > 20 %}
</div>
<div class="col-sm-6">
<div class="col-xs-12 col-sm-6">
{% endif %}
<div class="row">
<div class="col-xs-1" align="left"><span class="badge">{{entry.count}}</span></div>
<div class="col-xs-6"><a id="list_{{loop.index0}}" href="{{url_for(folder, book_id=entry[0].id )}}">{{entry[0].name}}</a></div>
<div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
<div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for(folder, book_id=entry[0].id )}}">{{entry[0].name}}</a></div>
</div>
{% endfor %}
</div>

23
cps/templates/tasks.html
View File

@ -5,17 +5,19 @@
{% block body %}
<div class="discover">
<h2>{{_('Tasks list')}}</h2>
<table class="table table-no-bordered" id="table" data-url="{{'/ajax/emailstat'}}">
<table class="table table-no-bordered" id="table" data-url="{{'/ajax/emailstat'}}" data-sort-name="starttime" data-sort-order="asc">
<thead>
<tr>
{% if g.user.role_admin() %}
<th data-field="user">{{_('User')}}</th>
<th data-halign="right" data-align="right" data-field="user" data-sortable="true">{{_('User')}}</th>
{% endif %}
<th data-field="type">{{_('Task')}}</th>
<th data-field="status">{{_('Status')}}</th>
<th data-field="progress">{{_('Progress')}}</th>
<th data-field="runtime">{{_('Runtime')}}</th>
<th data-field="starttime">{{_('Starttime')}}</th>
<th data-halign="right" data-align="right" data-field="type" data-sortable="true">{{_('Task')}}</th>
<th data-halign="right" data-align="right" data-field="status" data-sortable="true">{{_('Status')}}</th>
<th data-halign="right" data-align="right" data-field="progress" data-sortable="true" data-sorter="elementSorter">{{_('Progress')}}</th>
<th data-halign="right" data-align="right" data-field="runtime" data-sortable="true" data-sort-name="rt">{{_('Runtime')}}</th>
<th data-halign="right" data-align="right" data-field="starttime" data-sortable="true" data-sort-name="id">{{_('Starttime')}}</th>
<th data-field="id" data-visible="false"></th>
<th data-field="rt" data-visible="false"></th>
</tr>
</thead>
</table>
@ -43,6 +45,13 @@
}
});
}, 1000);
function elementSorter(a, b) {
a = +a.slice(0, -2);
b = +b.slice(0, -2);
if (a > b) return 1;
if (a < b) return -1;
return 0;
}
</script>
{% endblock %}

54
cps/web.py
View File

@ -29,14 +29,13 @@ import logging
from logging.handlers import RotatingFileHandler
from flask import (Flask, render_template, request, Response, redirect,
url_for, send_from_directory, make_response, g, flash,
abort, Markup, stream_with_context)
abort, Markup)
from flask import __version__ as flaskVersion
import cache_buster
import ub
from ub import config
import helper
import os
import errno
from sqlalchemy.sql.expression import func
from sqlalchemy.sql.expression import false
from sqlalchemy.exc import IntegrityError
@ -349,25 +348,20 @@ def remote_login_required(f):
def shortentitle_filter(s,nchar=20):
text = s.split()
res = "" # result
sum = 0 # overall length
suml = 0 # overall length
for line in text:
if sum >= 60:
if suml >= 60:
res += '...'
break
# if word longer than 20 chars truncate line and append '...', otherwise add whole word to result
# string, and summarize total length to stop at 60 chars
if len(line) > nchar:
res += line[:(nchar-3)] + '[..] '
sum += nchar+3
suml += nchar+3
else:
res += line + ' '
sum += len(line) + 1
suml += len(line) + 1
return res.strip()
#if len(s) > 20:
# s = s.split(':', 1)[0]
# if len(s) > 20:
# s = textwrap.wrap(s, 20, break_long_words=True)[0] + ' ...'
#return s
@app.template_filter('mimetype')
@ -784,7 +778,7 @@ def feed_series(book_id):
off = request.args.get("offset")
if not off:
off = 0
entries, random, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
db.Books, db.Books.series.any(db.Series.id == book_id), [db.Books.series_index])
xml = render_title_template('feed.xml', entries=entries, pagination=pagination)
response = make_response(xml)
@ -889,7 +883,7 @@ def get_metadata_calibre_companion(uuid):
@login_required
def get_email_status_json():
answer=list()
tasks=helper.global_eMailThread.get_taskstatus()
tasks=helper.global_WorkerThread.get_taskstatus()
if not current_user.role_admin():
for task in tasks:
if task['user'] == current_user.nickname:
@ -909,6 +903,32 @@ def get_email_status_json():
if 'starttime' not in task:
task['starttime'] = ""
answer = tasks
'''answer.append({'user': 'Test', 'starttime': '07.3.2018 15:23', 'progress': " 0 %", 'type': 'E-Mail',
'runtime': '0 s', 'rt': 0, 'status': _('Waiting'),'id':1 })
answer.append({'user': 'Admin', 'starttime': '07.3.2018 15:33', 'progress': " 11 %", 'type': 'E-Mail',
'runtime': '2 s', 'rt':2, 'status': _('Waiting'),'id':2})
answer.append({'user': 'Nanny', 'starttime': '8.3.2018 15:23', 'progress': " 2 %", 'type': 'E-Mail',
'runtime': '32 s','rt':32, 'status': _('Waiting'),'id':3})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '7 s','rt':7, 'status': _('Waiting'),'id':4})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '22 s','rt':22, 'status': _('Waiting'),'id':5})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '17 s','rt':17, 'status': _('Waiting'),'id':6})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '72 s','rt':72, 'status': _('Waiting'),'id':7})
answer.append({'user': 'Guest', 'starttime': '19.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '1:07 s','rt':67, 'status': _('Waiting'),'id':8})
answer.append({'user': 'Guest', 'starttime': '18.2.2018 12:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '2:07 s','rt':127, 'status': _('Waiting'),'id':9})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '27 s','rt':27, 'status': _('Waiting'),'id':10})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 16:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '73 s','rt':73, 'status': _('Waiting'),'id':11})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 14:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '71 s','rt':71, 'status': _('Waiting'),'id':12})
answer.append({'user': 'Guest', 'starttime': '09.3.2018 17:23', 'progress': " 44 %", 'type': 'E-Mail',
'runtime': '27 s','rt':27, 'status': _('Waiting'),'id':13})'''
js=json.dumps(answer)
response = make_response(js)
response.headers["Content-Type"] = "application/json; charset=utf-8"
@ -1184,7 +1204,7 @@ def author(book_id, page):
gc = GoodreadsClient(config.config_goodreads_api_key, config.config_goodreads_api_secret)
author_info = gc.find_author(author_name=name)
other_books = get_unique_other_books(entries.all(), author_info.books)
except:
except Exception:
# Skip goodreads, if site is down/inaccessible
app.logger.error('Goodreads website is down/inaccessible')
@ -1424,7 +1444,7 @@ def bookmark(book_id, book_format):
def get_tasks_status():
# if current user admin, show all email, otherwise only own emails
answer=list()
tasks=helper.global_eMailThread.get_taskstatus()
tasks=helper.global_WorkerThread.get_taskstatus()
if not current_user.role_admin():
for task in tasks:
if task['user'] == current_user.nickname:
@ -1492,9 +1512,7 @@ def delete_book(book_id, book_format):
# delete book from Shelfs, Downloads, Read list
ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete()
# ToDo check Downloads.book right
ub.delete_download(book_id)
# ub.session.query(ub.Downloads).filter(ub.Downloads.book_id == book_id).delete()
ub.session.commit()
# check if only this book links to:
@ -2735,7 +2753,6 @@ def configuration_helper(origin):
gdriveError=gdriveError, goodreads=goodreads_support,
title=_(u"Basic Configuration"), page="config")
if reboot_required:
# db.engine.dispose() # ToDo verify correct
ub.session.close()
ub.engine.dispose()
# stop Server
@ -3066,7 +3083,6 @@ def edit_book(book_id):
if is_format:
# Format entry already exists, no need to update the database
app.logger.info('Bokk format already existing')
pass
else:
db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
db.session.add(db_format)

501
cps/worker.py Normal file
View File

@ -0,0 +1,501 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import smtplib
import threading
from datetime import datetime
import logging
import time
import socket
import sys
import os
from email.generator import Generator
import web
from flask_babel import gettext as _
# from babel.dates import format_datetime
import re
import gdriveutils as gd
import subprocess
try:
from StringIO import StringIO
from email.MIMEBase import MIMEBase
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
except ImportError:
from io import StringIO
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email import encoders
from email.utils import formatdate
from email.utils import make_msgid
chunksize = 8192
STAT_WAITING = 0
STAT_FAIL = 1
STAT_STARTED = 2
STAT_FINISH_SUCCESS = 3
TASK_EMAIL = 1
TASK_CONVERT = 2
RET_FAIL = 0
RET_SUCCESS = 1
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to
def get_attachment(bookpath, filename):
"""Get file as MIMEBase message"""
calibrepath = web.config.config_calibre_dir
if web.ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(bookpath, filename)
if df:
datafile = os.path.join(calibrepath, bookpath, filename)
if not os.path.exists(os.path.join(calibrepath, bookpath)):
os.makedirs(os.path.join(calibrepath, bookpath))
df.GetContentFile(datafile)
else:
return None
file_ = open(datafile, 'rb')
data = file_.read()
file_.close()
os.remove(datafile)
else:
try:
file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
data = file_.read()
file_.close()
except IOError:
web.app.logger.exception(e) # traceback.print_exc()
web.app.logger.error(u'The requested file could not be read. Maybe wrong permissions?')
return None
attachment = MIMEBase('application', 'octet-stream')
attachment.set_payload(data)
encoders.encode_base64(attachment)
attachment.add_header('Content-Disposition', 'attachment',
filename=filename)
return attachment
# Class for sending email with ability to get current progress
class email(smtplib.SMTP):
transferSize = 0
progress = 0
def __init__(self, *args, **kwargs):
smtplib.SMTP.__init__(self, *args, **kwargs)
def data(self, msg):
self.transferSize = len(msg)
(code, resp) = smtplib.SMTP.data(self, msg)
self.progress = 0
return (code, resp)
def send(self, strg):
"""Send `strg' to the server."""
if self.debuglevel > 0:
print('send:', repr(strg), file=sys.stderr)
if hasattr(self, 'sock') and self.sock:
try:
if self.transferSize:
lock=threading.Lock()
lock.acquire()
self.transferSize = len(strg)
lock.release()
for i in range(0, self.transferSize, chunksize):
self.sock.send(strg[i:i+chunksize])
lock.acquire()
self.progress = i
lock.release()
else:
self.sock.sendall(strg)
except socket.error:
self.close()
raise smtplib.SMTPServerDisconnected('Server not connected')
else:
raise smtplib.SMTPServerDisconnected('please run connect() first')
def getTransferStatus(self):
if self.transferSize:
lock2 = threading.Lock()
lock2.acquire()
value = round(float(self.progress) / float(self.transferSize),2)*100
lock2.release()
return str(value) + ' %'
else:
return "100 %"
# Class for sending ssl encrypted email with ability to get current progress
class email_SSL(email):
def __init__(self, *args, **kwargs):
smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
#Class for all worker tasks in the background
class WorkerThread(threading.Thread):
def __init__(self):
self._stopevent = threading.Event()
threading.Thread.__init__(self)
self.status = 0
self.current = 0
self.last = 0
self.queue = list()
self.UIqueue = list()
self.asyncSMTP=None
self.id = 0
# Main thread loop starting the different tasks
def run(self):
while not self._stopevent.isSet():
doLock = threading.Lock()
doLock.acquire()
if self.current != self.last:
doLock.release()
if self.queue[self.current]['typ'] == TASK_EMAIL:
self.send_raw_email()
if self.queue[self.current]['typ'] == TASK_CONVERT:
self.convert_mobi()
self.current += 1
else:
doLock.release()
time.sleep(1)
def stop(self):
self._stopevent.set()
def get_send_status(self):
if self.asyncSMTP:
return self.asyncSMTP.getTransferStatus()
else:
return "0 %"
def delete_completed_tasks(self):
for index, task in reversed(list(enumerate(self.UIqueue))):
if task['progress'] == "100 %":
# delete tasks
self.queue.pop(index)
self.UIqueue.pop(index)
# if we are deleting entries before the current index, adjust the index
self.current -= 1
self.last = len(self.queue)
def get_taskstatus(self):
if self.current < len(self.queue):
if self.queue[self.current]['status'] == STAT_STARTED:
if not self.queue[self.current]['typ'] == TASK_CONVERT:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def convert_mobi(self):
# convert book, and upload in case of google drive
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if web.ub.config.config_ebookconverter == 2:
filename = self.convert_calibre()
else:
filename = self.convert_kindlegen()
if web.ub.config.config_use_google_drive:
gd.updateGdriveCalibreFromLocal()
if(filename):
self.add_email(_(u'Send to Kindle'), self.queue[self.current]['path'], filename,
self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
self.UIqueue[self.current]['user'], _(u"E-Mail: %s" % self.queue[self.current]['title']))
def convert_kindlegen(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
if not os.path.exists(web.ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=web.ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return
try:
p = subprocess.Popen(
(web.ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
web.app.logger.info("convert_kindlegen: " + error_message)
web.app.logger.info(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
cur_book.data.append(new_format)
web.db.session.commit()
self.queue[self.current]['path'] = cur_book.path
self.queue[self.current]['title'] = cur_book.title
if web.ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return file_path + ".mobi" #, RET_SUCCESS
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
def convert_calibre(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
if not os.path.exists(web.ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found",
converterpath=web.ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return
try:
command = ("\"" + web.ub.config.config_converterpath + "\" \"" + file_path + u".epub\" \""
+ file_path + u".mobi\" " + web.ub.config.config_calibre).encode(sys.getfilesystemencoding())
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
progress = re.search("(\d+)%\s.*", nextline)
if progress:
self.UIqueue[self.current]['progress'] = progress.group(1) + '%'
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0:
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
cur_book.data.append(new_format)
web.db.session.commit()
self.queue[self.current]['path'] = cur_book.path
self.queue[self.current]['title'] = cur_book.title
if web.ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return file_path + ".mobi" # , RET_SUCCESS
else:
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
def add_convert(self, file_path, bookid, user_name, type, settings, kindle_mail):
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.id += 1
self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle':kindle_mail,
'status': STAT_WAITING, 'typ': TASK_CONVERT, 'settings':settings})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': type,
'runtime': '0 s', 'status': _('Waiting'),'id': self.id } )
self.id += 1
self.last=len(self.queue)
addLock.release()
def add_email(self, subject, filepath, attachment, settings, recipient, user_name, typ):
# if more than 20 entries in the list, clean the list
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.queue.append({'subject':subject, 'attachment':attachment, 'filepath':filepath,
'settings':settings, 'recipent':recipient, 'starttime': 0,
'status': STAT_WAITING, 'typ': TASK_EMAIL})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ,
'runtime': '0 s', 'status': _('Waiting'),'id': self.id })
self.id += 1
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
obj=self.queue[self.current]
# create MIME message
msg = MIMEMultipart()
msg['Subject'] = self.queue[self.current]['subject']
msg['Message-Id'] = make_msgid('calibre-web')
msg['Date'] = formatdate(localtime=True)
text = _(u'This email has been sent via calibre web.')
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
if obj['attachment']:
result = get_attachment(obj['filepath'], obj['attachment'])
if result:
msg.attach(result)
else:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return
msg['From'] = obj['settings']["mail_from"]
msg['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(msg)
msg = fp.getvalue()
# send email
try:
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
sys.stderr = StderrLogger()
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
# link to logginglevel
if web.ub.config.config_log_level != logging.DEBUG:
self.asyncSMTP.set_debuglevel(0)
else:
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], msg)
self.asyncSMTP.quit()
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
sys.stderr = org_stderr
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException):
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
web.app.logger.error(e)
# return None
def _formatRuntime(self, runtime):
self.UIqueue[self.current]['rt'] = runtime.total_seconds()
val = re.split('\:|\.', str(runtime))[0:3]
erg = list()
for v in val:
if int(v) > 0:
erg.append(v)
retVal = (':'.join(erg)).lstrip('0') + ' s'
if retVal == ' s':
retVal = '0 s'
return retVal
class StderrLogger(object):
buffer = ''
def __init__(self):
self.logger = web.app.logger
def write(self, message):
if message == '\n':
self.logger.debug(self.buffer)
print(self.buffer)
self.buffer = ''
else:
self.buffer += message