From f79d549910ac2707a593c2df7f775a41ab34e6a0 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 30 Jun 2019 11:20:36 +0200 Subject: [PATCH] Fix duplicate user and email (now case insensitive #948) Fix sorting in comics (#950) Fix log error on Calibre converter error (#953) Fix long running tasks (#954) --- cps/helper.py | 30 +- cps/static/js/archive/archive.js | 64 ++++ cps/static/js/archive/unrar.js | 7 +- cps/static/js/archive/untar.js | 19 +- cps/static/js/archive/unzip.js | 69 +++-- cps/static/js/io.js | 483 ------------------------------- cps/web.py | 35 ++- cps/worker.py | 36 +-- 8 files changed, 184 insertions(+), 559 deletions(-) delete mode 100644 cps/static/js/io.js diff --git a/cps/helper.py b/cps/helper.py index 9b789f78..114eeccd 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -34,7 +34,8 @@ from flask import send_from_directory, make_response, redirect, abort from flask_babel import gettext as _ from flask_login import current_user from babel.dates import format_datetime -from datetime import datetime +from babel.units import format_unit +from datetime import datetime, timedelta import shutil import requests try: @@ -566,8 +567,33 @@ def json_serial(obj): if isinstance(obj, (datetime)): return obj.isoformat() + if isinstance(obj, (timedelta)): + return { + '__type__': 'timedelta', + 'days': obj.days, + 'seconds': obj.seconds, + 'microseconds': obj.microseconds, + } raise TypeError ("Type %s not serializable" % type(obj)) + +# helper function for displaying the runtime of tasks +def format_runtime(runtime): + retVal = "" + if runtime.days: + retVal = format_unit(runtime.days, 'duration-day', length="long", locale=web.get_locale()) + ', ' + mins, seconds = divmod(runtime.seconds, 60) + hours, minutes = divmod(mins, 60) + # ToDo: locale.number_symbols._data['timeSeparator'] -> localize time separator ? + if hours: + retVal += '{:d}:{:02d}:{:02d}s'.format(hours, minutes, seconds) + elif minutes: + retVal += '{:2d}:{:02d}s'.format(minutes, seconds) + else: + retVal += '{:2d}s'.format(seconds) + return retVal + + # helper function to apply localize status information in tasklist entries def render_task_status(tasklist): renderedtasklist=list() @@ -579,6 +605,8 @@ def render_task_status(tasklist): if 'starttime' not in task: task['starttime'] = "" + task['runtime'] = format_runtime(task['formRuntime']) + # localize the task status if isinstance( task['stat'], int ): if task['stat'] == worker.STAT_WAITING: diff --git a/cps/static/js/archive/archive.js b/cps/static/js/archive/archive.js index 331997d9..cfc7bd40 100644 --- a/cps/static/js/archive/archive.js +++ b/cps/static/js/archive/archive.js @@ -1,3 +1,67 @@ +/* alphanum.js (C) Brian Huisman + * Based on the Alphanum Algorithm by David Koelle + * The Alphanum Algorithm is discussed at http://www.DaveKoelle.com + * + * Distributed under same license as original + * + * Released under the MIT License - https://opensource.org/licenses/MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + /* ******************************************************************** + * Alphanum sort() function version - case insensitive + * - Slower, but easier to modify for arrays of objects which contain + * string properties + * + */ +function alphanumCase(a, b) { + function chunkify(t) { + var tz = new Array(); + var x = 0, y = -1, n = 0, i, j; + + while (i = (j = t.charAt(x++)).charCodeAt(0)) { + var m = (i == 46 || (i >=48 && i <= 57)); + if (m !== n) { + tz[++y] = ""; + n = m; + } + tz[y] += j; + } + return tz; + } + + var aa = chunkify(a.filename.toLowerCase()); + var bb = chunkify(b.filename.toLowerCase()); + + for (x = 0; aa[x] && bb[x]; x++) { + if (aa[x] !== bb[x]) { + var c = Number(aa[x]), d = Number(bb[x]); + if (c == aa[x] && d == bb[x]) { + return c - d; + } else return (aa[x] > bb[x]) ? 1 : -1; + } + } + return aa.length - bb.length; +} +// =========================================================================== + + /** * archive.js * diff --git a/cps/static/js/archive/unrar.js b/cps/static/js/archive/unrar.js index 89263b83..fadb791e 100644 --- a/cps/static/js/archive/unrar.js +++ b/cps/static/js/archive/unrar.js @@ -1332,12 +1332,7 @@ var unrar = function(arrayBuffer) { totalFilesInArchive = localFiles.length; // now we have all information but things are unpacked - // TODO: unpack - localFiles = localFiles.sort(function(a, b) { - var aname = a.filename.toLowerCase(); - var bname = b.filename.toLowerCase(); - return aname > bname ? 1 : -1; - }); + localFiles.sort(alphanumCase); info(localFiles.map(function(a) { return a.filename; diff --git a/cps/static/js/archive/untar.js b/cps/static/js/archive/untar.js index d9a1fdfd..cc1499ef 100644 --- a/cps/static/js/archive/untar.js +++ b/cps/static/js/archive/untar.js @@ -115,6 +115,7 @@ var TarLocalFile = function(bstream) { } }; + var untar = function(arrayBuffer) { postMessage(new bitjs.archive.UnarchiveStartEvent()); currentFilename = ""; @@ -127,14 +128,22 @@ var untar = function(arrayBuffer) { var bstream = new bitjs.io.ByteStream(arrayBuffer); postProgress(); - // While we don't encounter an empty block, keep making TarLocalFiles. + /* + // go through whole file, read header of each block and memorize, filepointer + */ while (bstream.peekNumber(4) !== 0) { - var oneLocalFile = new TarLocalFile(bstream); + var localFile = new TarLocalFile(bstream); + allLocalFiles.push(localFile); + postProgress(); + } + // got all local files, now sort them + allLocalFiles.sort(alphanumCase); + + allLocalFiles.forEach(function(oneLocalFile) { + // While we don't encounter an empty block, keep making TarLocalFiles. if (oneLocalFile && oneLocalFile.isValid) { // If we make it to this point and haven't thrown an error, we have successfully // read in the data for a local file, so we can update the actual bytestream. - - allLocalFiles.push(oneLocalFile); totalUncompressedBytesInArchive += oneLocalFile.size; // update progress @@ -145,7 +154,7 @@ var untar = function(arrayBuffer) { postMessage(new bitjs.archive.UnarchiveExtractEvent(oneLocalFile)); postProgress(); } - } + }); totalFilesInArchive = allLocalFiles.length; postProgress(); diff --git a/cps/static/js/archive/unzip.js b/cps/static/js/archive/unzip.js index f8de27f7..a4cec8d0 100644 --- a/cps/static/js/archive/unzip.js +++ b/cps/static/js/archive/unzip.js @@ -72,23 +72,10 @@ var ZipLocalFile = function(bstream) { this.filename = bstream.readString(this.fileNameLength); } - info("Zip Local File Header:"); - info(" version=" + this.version); - info(" general purpose=" + this.generalPurpose); - info(" compression method=" + this.compressionMethod); - info(" last mod file time=" + this.lastModFileTime); - info(" last mod file date=" + this.lastModFileDate); - info(" crc32=" + this.crc32); - info(" compressed size=" + this.compressedSize); - info(" uncompressed size=" + this.uncompressedSize); - info(" file name length=" + this.fileNameLength); - info(" extra field length=" + this.extraFieldLength); - info(" filename = '" + this.filename + "'"); - this.extraField = null; if (this.extraFieldLength > 0) { - this.extraField = bstream.readString(this.extraFieldLength); - info(" extra field=" + this.extraField); + this.extraField = bstream.readString(this.extraFieldLength); + info(" extra field=" + this.extraField); } // read in the compressed data @@ -107,6 +94,21 @@ var ZipLocalFile = function(bstream) { this.compressedSize = bstream.readNumber(4); this.uncompressedSize = bstream.readNumber(4); } + + // Now that we have all the bytes for this file, we can print out some information. + info("Zip Local File Header:"); + info(" version=" + this.version); + info(" general purpose=" + this.generalPurpose); + info(" compression method=" + this.compressionMethod); + info(" last mod file time=" + this.lastModFileTime); + info(" last mod file date=" + this.lastModFileDate); + info(" crc32=" + this.crc32); + info(" compressed size=" + this.compressedSize); + info(" uncompressed size=" + this.uncompressedSize); + info(" file name length=" + this.fileNameLength); + info(" extra field length=" + this.extraFieldLength); + info(" filename = '" + this.filename + "'"); + }; // determine what kind of compressed data we have and decompress @@ -132,6 +134,7 @@ ZipLocalFile.prototype.unzip = function() { // Takes an ArrayBuffer of a zip file in // returns null on error // returns an array of DecompressedFile objects on success +// ToDo This function differs var unzip = function(arrayBuffer) { postMessage(new bitjs.archive.UnarchiveStartEvent()); @@ -159,11 +162,7 @@ var unzip = function(arrayBuffer) { totalFilesInArchive = localFiles.length; // got all local files, now sort them - localFiles.sort(function(a, b) { - var aname = a.filename.toLowerCase(); - var bname = b.filename.toLowerCase(); - return aname > bname ? 1 : -1; - }); + localFiles.sort(alphanumCase); // archive extra data record if (bstream.peekNumber(4) === zArchiveExtraDataSignature) { @@ -253,9 +252,9 @@ function getHuffmanCodes(bitLengths) { } // Reference: http://tools.ietf.org/html/rfc1951#page-8 - var numLengths = bitLengths.length, - blCount = [], - MAX_BITS = 1; + var numLengths = bitLengths.length; + var blCount = []; + var MAX_BITS = 1; // Step 1: count up how many codes of each length we have for (var i = 0; i < numLengths; ++i) { @@ -274,8 +273,8 @@ function getHuffmanCodes(bitLengths) { } // Step 2: Find the numerical value of the smallest code for each code length - var nextCode = [], - code = 0; + var nextCode = []; + var code = 0; for (var bits = 1; bits <= MAX_BITS; ++bits) { var length2 = bits - 1; // ensure undefined lengths are zero @@ -285,8 +284,8 @@ function getHuffmanCodes(bitLengths) { } // Step 3: Assign numerical values to all codes - var table = {}, - tableLength = 0; + var table = {}; + var tableLength = 0; for (var n = 0; n < numLengths; ++n) { var len = bitLengths[n]; if (len !== 0) { @@ -353,7 +352,8 @@ function getFixedDistanceTable() { // extract one bit at a time until we find a matching Huffman Code // then return that symbol function decodeSymbol(bstream, hcTable) { - var code = 0, len = 0; + var code = 0; + var len = 0; // loop until we match for (;;) { @@ -364,7 +364,6 @@ function decodeSymbol(bstream, hcTable) { // check against Huffman Code table and break if found if (hcTable.hasOwnProperty(code) && hcTable[code].length === len) { - break; } if (len > hcTable.maxLength) { @@ -500,10 +499,10 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) { if (symbol === 256) { break; } else { - var lengthLookup = LengthLookupTable[symbol - 257], - length = lengthLookup[1] + bstream.readBits(lengthLookup[0]), - distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)], - distance = distLookup[1] + bstream.readBits(distLookup[0]); + var lengthLookup = LengthLookupTable[symbol - 257]; + var length = lengthLookup[1] + bstream.readBits(lengthLookup[0]); + var distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)]; + var distance = distLookup[1] + bstream.readBits(distLookup[0]); // now apply length and distance appropriately and copy to output @@ -634,8 +633,8 @@ function inflate(compressedData, numDecompressedBytes) { var distanceCodeLengths = literalCodeLengths.splice(numLiteralLengthCodes, numDistanceCodes); // now generate the true Huffman Code tables using these code lengths - var hcLiteralTable = getHuffmanCodes(literalCodeLengths), - hcDistanceTable = getHuffmanCodes(distanceCodeLengths); + var hcLiteralTable = getHuffmanCodes(literalCodeLengths); + var hcDistanceTable = getHuffmanCodes(distanceCodeLengths); blockSize = inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer); } else { // error diff --git a/cps/static/js/io.js b/cps/static/js/io.js deleted file mode 100644 index 292f5f95..00000000 --- a/cps/static/js/io.js +++ /dev/null @@ -1,483 +0,0 @@ -/* - * io.js - * - * Provides readers for bit/byte streams (reading) and a byte buffer (writing). - * - * Licensed under the MIT License - * - * Copyright(c) 2011 Google Inc. - * Copyright(c) 2011 antimatter15 - */ - -/* global bitjs, Uint8Array */ - -var bitjs = bitjs || {}; -bitjs.io = bitjs.io || {}; - -(function() { - - // mask for getting the Nth bit (zero-based) - bitjs.BIT = [ 0x01, 0x02, 0x04, 0x08, - 0x10, 0x20, 0x40, 0x80, - 0x100, 0x200, 0x400, 0x800, - 0x1000, 0x2000, 0x4000, 0x8000]; - - // mask for getting N number of bits (0-8) - var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ]; - - - /** - * This bit stream peeks and consumes bits out of a binary stream. - * - * @param {ArrayBuffer} ab An ArrayBuffer object or a Uint8Array. - * @param {boolean} rtl Whether the stream reads bits from the byte starting - * from bit 7 to 0 (true) or bit 0 to 7 (false). - * @param {Number} optOffset The offset into the ArrayBuffer - * @param {Number} optLength The length of this BitStream - */ - bitjs.io.BitStream = function(ab, rtl, optOffset, optLength) { - if (!ab || !ab.toString || ab.toString() !== "[object ArrayBuffer]") { - throw "Error! BitArray constructed with an invalid ArrayBuffer object"; - } - - var offset = optOffset || 0; - var length = optLength || ab.byteLength; - this.bytes = new Uint8Array(ab, offset, length); - this.bytePtr = 0; // tracks which byte we are on - this.bitPtr = 0; // tracks which bit we are on (can have values 0 through 7) - this.peekBits = rtl ? this.peekBitsRtl : this.peekBitsLtr; - }; - - - /** - * byte0 byte1 byte2 byte3 - * 7......0 | 7......0 | 7......0 | 7......0 - * - * The bit pointer starts at bit0 of byte0 and moves left until it reaches - * bit7 of byte0, then jumps to bit0 of byte1, etc. - * @param {number} n The number of bits to peek. - * @param {boolean=} movePointers Whether to move the pointer, defaults false. - * @return {number} The peeked bits, as an unsigned number. - */ - bitjs.io.BitStream.prototype.peekBitsLtr = function(n, movePointers) { - if (n <= 0 || typeof n !== typeof 1) { - return 0; - } - - var movePointers = movePointers || false; - var bytePtr = this.bytePtr; - var bitPtr = this.bitPtr; - var result = 0; - var bitsIn = 0; - var bytes = this.bytes; - - // keep going until we have no more bits left to peek at - // TODO: Consider putting all bits from bytes we will need into a variable and then - // shifting/masking it to just extract the bits we want. - // This could be considerably faster when reading more than 3 or 4 bits at a time. - while (n > 0) { - if (bytePtr >= bytes.length) { - throw "Error! Overflowed the bit stream! n=" + n + ", bytePtr=" + bytePtr + ", bytes.length=" + - bytes.length + ", bitPtr=" + bitPtr; - } - - var numBitsLeftInThisByte = (8 - bitPtr); - var mask; - if (n >= numBitsLeftInThisByte) { - mask = (BITMASK[numBitsLeftInThisByte] << bitPtr); - result |= (((bytes[bytePtr] & mask) >> bitPtr) << bitsIn); - - bytePtr++; - bitPtr = 0; - bitsIn += numBitsLeftInThisByte; - n -= numBitsLeftInThisByte; - } else { - mask = (BITMASK[n] << bitPtr); - result |= (((bytes[bytePtr] & mask) >> bitPtr) << bitsIn); - - bitPtr += n; - bitsIn += n; - n = 0; - } - } - - if (movePointers) { - this.bitPtr = bitPtr; - this.bytePtr = bytePtr; - } - - return result; - }; - - - /** - * byte0 byte1 byte2 byte3 - * 7......0 | 7......0 | 7......0 | 7......0 - * - * The bit pointer starts at bit7 of byte0 and moves right until it reaches - * bit0 of byte0, then goes to bit7 of byte1, etc. - * @param {number} n The number of bits to peek. - * @param {boolean=} movePointers Whether to move the pointer, defaults false. - * @return {number} The peeked bits, as an unsigned number. - */ - bitjs.io.BitStream.prototype.peekBitsRtl = function(n, movePointers) { - if (n <= 0 || typeof n !== typeof 1) { - return 0; - } - - var movePointers = movePointers || false; - var bytePtr = this.bytePtr; - var bitPtr = this.bitPtr; - var result = 0; - var bytes = this.bytes; - - // keep going until we have no more bits left to peek at - // TODO: Consider putting all bits from bytes we will need into a variable and then - // shifting/masking it to just extract the bits we want. - // This could be considerably faster when reading more than 3 or 4 bits at a time. - while (n > 0) { - - if (bytePtr >= bytes.length) { - throw "Error! Overflowed the bit stream! n=" + n + ", bytePtr=" + bytePtr + ", bytes.length=" + - bytes.length + ", bitPtr=" + bitPtr; - // return -1; - } - - var numBitsLeftInThisByte = (8 - bitPtr); - if (n >= numBitsLeftInThisByte) { - result <<= numBitsLeftInThisByte; - result |= (BITMASK[numBitsLeftInThisByte] & bytes[bytePtr]); - bytePtr++; - bitPtr = 0; - n -= numBitsLeftInThisByte; - } else { - result <<= n; - result |= ((bytes[bytePtr] & (BITMASK[n] << (8 - n - bitPtr))) >> (8 - n - bitPtr)); - - bitPtr += n; - n = 0; - } - } - - if (movePointers) { - this.bitPtr = bitPtr; - this.bytePtr = bytePtr; - } - - return result; - }; - - - /** - * Some voodoo magic. - */ - bitjs.io.BitStream.prototype.getBits = function() { - return (((((this.bytes[this.bytePtr] & 0xff) << 16) + - ((this.bytes[this.bytePtr + 1] & 0xff) << 8) + - ((this.bytes[this.bytePtr + 2] & 0xff))) >>> (8 - this.bitPtr)) & 0xffff); - }; - - - /** - * Reads n bits out of the stream, consuming them (moving the bit pointer). - * @param {number} n The number of bits to read. - * @return {number} The read bits, as an unsigned number. - */ - bitjs.io.BitStream.prototype.readBits = function(n) { - return this.peekBits(n, true); - }; - - - /** - * This returns n bytes as a sub-array, advancing the pointer if movePointers - * is true. Only use this for uncompressed blocks as this throws away remaining - * bits in the current byte. - * @param {number} n The number of bytes to peek. - * @param {boolean=} movePointers Whether to move the pointer, defaults false. - * @return {Uint8Array} The subarray. - */ - bitjs.io.BitStream.prototype.peekBytes = function(n, movePointers) { - if (n <= 0 || typeof n != typeof 1) { - return 0; - } - - // from http://tools.ietf.org/html/rfc1951#page-11 - // "Any bits of input up to the next byte boundary are ignored." - while (this.bitPtr !== 0) { - this.readBits(1); - } - - movePointers = movePointers || false; - var bytePtr = this.bytePtr; - // var bitPtr = this.bitPtr; - - var result = this.bytes.subarray(bytePtr, bytePtr + n); - - if (movePointers) { - this.bytePtr += n; - } - - return result; - }; - - - /** - * @param {number} n The number of bytes to read. - * @return {Uint8Array} The subarray. - */ - bitjs.io.BitStream.prototype.readBytes = function(n) { - return this.peekBytes(n, true); - }; - - - /** - * This object allows you to peek and consume bytes as numbers and strings - * out of an ArrayBuffer. In this buffer, everything must be byte-aligned. - * - * @param {ArrayBuffer} ab The ArrayBuffer object. - * @param {number=} optOffset The offset into the ArrayBuffer - * @param {number=} optLength The length of this BitStream - * @constructor - */ - bitjs.io.ByteStream = function(ab, optOffset, optLength) { - var offset = optOffset || 0; - var length = optLength || ab.byteLength; - this.bytes = new Uint8Array(ab, offset, length); - this.ptr = 0; - }; - - - /** - * Peeks at the next n bytes as an unsigned number but does not advance the - * pointer - * TODO: This apparently cannot read more than 4 bytes as a number? - * @param {number} n The number of bytes to peek at. - * @return {number} The n bytes interpreted as an unsigned number. - */ - bitjs.io.ByteStream.prototype.peekNumber = function(n) { - // TODO: return error if n would go past the end of the stream? - if (n <= 0 || typeof n !== typeof 1) { - return -1; - } - - var result = 0; - // read from last byte to first byte and roll them in - var curByte = this.ptr + n - 1; - while (curByte >= this.ptr) { - result <<= 8; - result |= this.bytes[curByte]; - --curByte; - } - return result; - }; - - - /** - * Returns the next n bytes as an unsigned number (or -1 on error) - * and advances the stream pointer n bytes. - * @param {number} n The number of bytes to read. - * @return {number} The n bytes interpreted as an unsigned number. - */ - bitjs.io.ByteStream.prototype.readNumber = function(n) { - var num = this.peekNumber( n ); - this.ptr += n; - return num; - }; - - - /** - * Returns the next n bytes as a signed number but does not advance the - * pointer. - * @param {number} n The number of bytes to read. - * @return {number} The bytes interpreted as a signed number. - */ - bitjs.io.ByteStream.prototype.peekSignedNumber = function(n) { - var num = this.peekNumber(n); - var HALF = Math.pow(2, (n * 8) - 1); - var FULL = HALF * 2; - - if (num >= HALF) num -= FULL; - - return num; - }; - - - /** - * Returns the next n bytes as a signed number and advances the stream pointer. - * @param {number} n The number of bytes to read. - * @return {number} The bytes interpreted as a signed number. - */ - bitjs.io.ByteStream.prototype.readSignedNumber = function(n) { - var num = this.peekSignedNumber(n); - this.ptr += n; - return num; - }; - - - /** - * This returns n bytes as a sub-array, advancing the pointer if movePointers - * is true. - * @param {number} n The number of bytes to read. - * @param {boolean} movePointers Whether to move the pointers. - * @return {Uint8Array} The subarray. - */ - bitjs.io.ByteStream.prototype.peekBytes = function(n, movePointers) { - if (n <= 0 || typeof n != typeof 1) { - return null; - } - - var result = this.bytes.subarray(this.ptr, this.ptr + n); - - if (movePointers) { - this.ptr += n; - } - - return result; - }; - - - /** - * Reads the next n bytes as a sub-array. - * @param {number} n The number of bytes to read. - * @return {Uint8Array} The subarray. - */ - bitjs.io.ByteStream.prototype.readBytes = function(n) { - return this.peekBytes(n, true); - }; - - - /** - * Peeks at the next n bytes as a string but does not advance the pointer. - * @param {number} n The number of bytes to peek at. - * @return {string} The next n bytes as a string. - */ - bitjs.io.ByteStream.prototype.peekString = function(n) { - if (n <= 0 || typeof n != typeof 1) { - return ""; - } - - var result = ""; - for (var p = this.ptr, end = this.ptr + n; p < end; ++p) { - result += String.fromCharCode(this.bytes[p]); - } - return result; - }; - - - /** - * Returns the next n bytes as an ASCII string and advances the stream pointer - * n bytes. - * @param {number} n The number of bytes to read. - * @return {string} The next n bytes as a string. - */ - bitjs.io.ByteStream.prototype.readString = function(n) { - var strToReturn = this.peekString(n); - this.ptr += n; - return strToReturn; - }; - - - /** - * A write-only Byte buffer which uses a Uint8 Typed Array as a backing store. - * @param {number} numBytes The number of bytes to allocate. - * @constructor - */ - bitjs.io.ByteBuffer = function(numBytes) { - if (typeof numBytes !== typeof 1 || numBytes <= 0) { - throw "Error! ByteBuffer initialized with '" + numBytes + "'"; - } - this.data = new Uint8Array(numBytes); - this.ptr = 0; - }; - - - /** - * @param {number} b The byte to insert. - */ - bitjs.io.ByteBuffer.prototype.insertByte = function(b) { - // TODO: throw if byte is invalid? - this.data[this.ptr++] = b; - }; - - - /** - * @param {Array.|Uint8Array|Int8Array} bytes The bytes to insert. - */ - bitjs.io.ByteBuffer.prototype.insertBytes = function(bytes) { - // TODO: throw if bytes is invalid? - this.data.set(bytes, this.ptr); - this.ptr += bytes.length; - }; - - - /** - * Writes an unsigned number into the next n bytes. If the number is too large - * to fit into n bytes or is negative, an error is thrown. - * @param {number} num The unsigned number to write. - * @param {number} numBytes The number of bytes to write the number into. - */ - bitjs.io.ByteBuffer.prototype.writeNumber = function(num, numBytes) { - if (numBytes < 1) { - throw "Trying to write into too few bytes: " + numBytes; - } - if (num < 0) { - throw "Trying to write a negative number (" + num + - ") as an unsigned number to an ArrayBuffer"; - } - if (num > (Math.pow(2, numBytes * 8) - 1)) { - throw "Trying to write " + num + " into only " + numBytes + " bytes"; - } - - // Roll 8-bits at a time into an array of bytes. - var bytes = []; - while (numBytes-- > 0) { - var eightBits = num & 255; - bytes.push(eightBits); - num >>= 8; - } - - this.insertBytes(bytes); - }; - - - /** - * Writes a signed number into the next n bytes. If the number is too large - * to fit into n bytes, an error is thrown. - * @param {number} num The signed number to write. - * @param {number} numBytes The number of bytes to write the number into. - */ - bitjs.io.ByteBuffer.prototype.writeSignedNumber = function(num, numBytes) { - if (numBytes < 1) { - throw "Trying to write into too few bytes: " + numBytes; - } - - var HALF = Math.pow(2, (numBytes * 8) - 1); - if (num >= HALF || num < -HALF) { - throw "Trying to write " + num + " into only " + numBytes + " bytes"; - } - - // Roll 8-bits at a time into an array of bytes. - var bytes = []; - while (numBytes-- > 0) { - var eightBits = num & 255; - bytes.push(eightBits); - num >>= 8; - } - - this.insertBytes(bytes); - }; - - - /** - * @param {string} str The ASCII string to write. - */ - bitjs.io.ByteBuffer.prototype.writeASCIIString = function(str) { - for (var i = 0; i < str.length; ++i) { - var curByte = str.charCodeAt(i); - if (curByte < 0 || curByte > 255) { - throw "Trying to write a non-ASCII string!"; - } - this.insertByte(curByte); - } - }; -})(); diff --git a/cps/web.py b/cps/web.py index 1a3849dd..3b19148c 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3171,13 +3171,22 @@ def new_user(): return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, title=_(u"Add new user")) content.password = generate_password_hash(to_save["password"]) - content.nickname = to_save["nickname"] - if config.config_public_reg and not check_valid_domain(to_save["email"]): - flash(_(u"E-mail is not from valid domain"), category="error") - return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, - title=_(u"Add new user")) + existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower())\ + .first() + existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower())\ + .first() + if not existing_user and not existing_email: + content.nickname = to_save["nickname"] + if config.config_public_reg and not check_valid_domain(to_save["email"]): + flash(_(u"E-mail is not from valid domain"), category="error") + return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, + title=_(u"Add new user")) + else: + content.email = to_save["email"] else: - content.email = to_save["email"] + flash(_(u"Found an existing account for this e-mail address or nickname."), category="error") + return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, + languages=languages, title=_(u"Add new user"), page="newuser") try: ub.session.add(content) ub.session.commit() @@ -3362,14 +3371,24 @@ def edit_user(user_id): if "locale" in to_save and to_save["locale"]: content.locale = to_save["locale"] if to_save["email"] and to_save["email"] != content.email: - content.email = to_save["email"] + existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \ + .first() + if not existing_email: + content.email = to_save["email"] + else: + flash(_(u"Found an existing account for this e-mail address."), category="error") + return render_title_template("user_edit.html", translations=translations, languages=languages, + new_user=0, content=content, downloads=downloads, + title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser") + if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail: content.kindle_mail = to_save["kindle_mail"] try: ub.session.commit() flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success") - except IntegrityError: + except IntegrityError as e: ub.session.rollback() + print(e) flash(_(u"An unknown error occured."), category="error") return render_title_template("user_edit.html", translations=translations, languages=languages, new_user=0, content=content, downloads=downloads, title=_(u"Edit User %(nick)s", diff --git a/cps/worker.py b/cps/worker.py index 2cc2b337..02f46ce6 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -20,7 +20,7 @@ from __future__ import print_function import smtplib import threading -from datetime import datetime +from datetime import datetime, timedelta import logging import time import socket @@ -221,8 +221,10 @@ class WorkerThread(threading.Thread): if self.UIqueue[self.current]['stat'] == STAT_STARTED: if self.queue[self.current]['taskType'] == TASK_EMAIL: self.UIqueue[self.current]['progress'] = self.get_send_status() - self.UIqueue[self.current]['runtime'] = self._formatRuntime( - datetime.now() - self.queue[self.current]['starttime']) + self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime'] + self.UIqueue[self.current]['rt'] = self.UIqueue[self.current]['formRuntime'].days*24*60 \ + + self.UIqueue[self.current]['formRuntime'].seconds \ + + self.UIqueue[self.current]['formRuntime'].microseconds return self.UIqueue def _convert_any_format(self): @@ -259,7 +261,8 @@ class WorkerThread(threading.Thread): self._handleSuccess() return file_path + format_new_ext else: - web.app.logger.info("Book id %d - target format of %s does not exist. Moving forward with convert.", bookid, format_new_ext) + web.app.logger.info("Book id %d - target format of %s does not exist. Moving forward with convert.", + bookid, format_new_ext) # check if converter-executable is existing if not os.path.exists(web.ub.config.config_converterpath): @@ -300,7 +303,7 @@ class WorkerThread(threading.Thread): if sys.version_info < (3, 0): command = [x.encode(sys.getfilesystemencoding()) for x in command] - p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) except OSError as e: self._handleError(_(u"Ebook-converter failed: %(error)s", error=e)) return @@ -328,6 +331,11 @@ class WorkerThread(threading.Thread): # process returncode check = p.returncode + calibre_traceback = p.stderr.readlines() + for ele in calibre_traceback: + web.app.logger.debug(ele.strip('\n')) + if not ele.startswith('Traceback') and not ele.startswith(' File'): + error_message = "Calibre failed with error: %s" % ele.strip('\n') # kindlegen returncodes # 0 = Info(prcgen):I1036: Mobi file built successfully @@ -481,31 +489,17 @@ class WorkerThread(threading.Thread): self._handleError(u'Error sending email: ' + e.strerror) return None - def _formatRuntime(self, runtime): - self.UIqueue[self.current]['rt'] = runtime.total_seconds() - val = re.split('\:|\.', str(runtime))[0:3] - erg = list() - for v in val: - if int(v) > 0: - erg.append(v) - retVal = (':'.join(erg)).lstrip('0') + ' s' - if retVal == ' s': - retVal = '0 s' - return retVal - def _handleError(self, error_message): web.app.logger.error(error_message) self.UIqueue[self.current]['stat'] = STAT_FAIL self.UIqueue[self.current]['progress'] = "100 %" - self.UIqueue[self.current]['runtime'] = self._formatRuntime( - datetime.now() - self.queue[self.current]['starttime']) + self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime'] self.UIqueue[self.current]['message'] = error_message def _handleSuccess(self): self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS self.UIqueue[self.current]['progress'] = "100 %" - self.UIqueue[self.current]['runtime'] = self._formatRuntime( - datetime.now() - self.queue[self.current]['starttime']) + self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime'] # Enable logging of smtp lib debug output