From 047af4b27d39352dbc136ffb2b36c104fdcfe3ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 9 Nov 2016 15:14:01 -0500 Subject: [PATCH] use human-readable sizes more broadly there was a function to "abbreviate" sizes, but it was somewhat unclear and incomplete. reuse the sizeof_fmt_* set of functions from the borg backup project (MIT licensed) to implement a more complete and flexible display that will scale up to the Yottabyte and beyond. it also supports non-IEC units (like "kibibyte", AKA 1024 bytes) if you fancy that stuff. this is a workaround for #91: it allows users to better see the size of the file that will be transfered. *some* places are still kept in bytes, most notably when receive fails to receive all bytes ("got %d bytes, wanted %d") because we may want more clarity there. text transfers also use the "bytes" suffix (instead of "B") because it will commonly not reach beyond the KiB range. note that the test suite only covers decimal (non-IEC) prefix, but it is assumed to be sufficient to be considered correct. --- src/wormhole/cli/cmd_receive.py | 14 +++++++------- src/wormhole/cli/cmd_send.py | 11 ++++++----- src/wormhole/server/cmd_usage.py | 32 ++++--------------------------- src/wormhole/test/test_scripts.py | 17 +++++++++------- src/wormhole/test/test_util.py | 28 +++++++++++++++++++++++++++ src/wormhole/util.py | 19 ++++++++++++++++++ 6 files changed, 74 insertions(+), 47 deletions(-) diff --git a/src/wormhole/cli/cmd_receive.py b/src/wormhole/cli/cmd_receive.py index 89caba2..a2cd89d 100644 --- a/src/wormhole/cli/cmd_receive.py +++ b/src/wormhole/cli/cmd_receive.py @@ -7,7 +7,7 @@ from twisted.python import log from ..wormhole import wormhole from ..transit import TransitReceiver from ..errors import TransferError, WormholeClosedError -from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr +from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr, sizeof_fmt_iec APPID = u"lothar.com/wormhole/text-or-file-xfer" @@ -194,8 +194,8 @@ class TwistedReceiver: file_data["filename"]) self.xfersize = file_data["filesize"] - self._msg(u"Receiving file (%d bytes) into: %s" % - (self.xfersize, os.path.basename(self.abs_destname))) + self._msg(u"Receiving file (%s) into: %s" % + (sizeof_fmt_iec(self.xfersize), os.path.basename(self.abs_destname))) self._ask_permission() tmp_destname = self.abs_destname + ".tmp" return open(tmp_destname, "wb") @@ -210,10 +210,10 @@ class TwistedReceiver: file_data["dirname"]) self.xfersize = file_data["zipsize"] - self._msg(u"Receiving directory (%d bytes) into: %s/" % - (self.xfersize, os.path.basename(self.abs_destname))) - self._msg(u"%d files, %d bytes (uncompressed)" % - (file_data["numfiles"], file_data["numbytes"])) + self._msg(u"Receiving directory (%s) into: %s/" % + (sizeof_fmt_iec(self.xfersize), os.path.basename(self.abs_destname))) + self._msg(u"%d files, %s (uncompressed)" % + (file_data["numfiles"], sizeof_fmt_iec(file_data["numbytes"]))) self._ask_permission() return tempfile.SpooledTemporaryFile() diff --git a/src/wormhole/cli/cmd_send.py b/src/wormhole/cli/cmd_send.py index a610b36..de6d304 100644 --- a/src/wormhole/cli/cmd_send.py +++ b/src/wormhole/cli/cmd_send.py @@ -8,7 +8,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue from ..errors import TransferError, WormholeClosedError from ..wormhole import wormhole from ..transit import TransitSender -from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr +from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr, sizeof_fmt_iec APPID = u"lothar.com/wormhole/text-or-file-xfer" @@ -167,7 +167,7 @@ class Sender: text = six.moves.input("Text to send: ") if text is not None: - print(u"Sending text message (%d bytes)" % len(text), + print(u"Sending text message (%s)" % sizeof_fmt_iec(len(text), suffix='bytes'), file=args.stdout) offer = { "message": text } fd_to_send = None @@ -187,7 +187,8 @@ class Sender: "filename": basename, "filesize": filesize, } - print(u"Sending %d byte file named '%s'" % (filesize, basename), + print(u"Sending %s file named '%s'" + % (sizeof_fmt_iec(filesize), basename), file=args.stdout) fd_to_send = open(what, "rb") return offer, fd_to_send @@ -222,8 +223,8 @@ class Sender: "numbytes": num_bytes, "numfiles": num_files, } - print(u"Sending directory (%d bytes compressed) named '%s'" - % (filesize, basename), file=args.stdout) + print(u"Sending directory (%s compressed) named '%s'" + % (sizeof_fmt_iec(filesize), basename), file=args.stdout) return offer, fd_to_send raise TypeError("'%s' is neither file nor directory" % args.what) diff --git a/src/wormhole/server/cmd_usage.py b/src/wormhole/server/cmd_usage.py index c314313..6b2f9ad 100644 --- a/src/wormhole/server/cmd_usage.py +++ b/src/wormhole/server/cmd_usage.py @@ -3,6 +3,7 @@ import os, time, json from collections import defaultdict import click from .database import get_db +from ..util import sizeof_fmt_iec def abbrev(t): if t is None: @@ -13,31 +14,6 @@ def abbrev(t): return "%.1fms" % (t*1e3) return "%.1fus" % (t*1e6) -def abbreviate_space(s, SI=True): - if s is None: - return "-" - if SI: - U = 1000.0 - isuffix = "B" - else: - U = 1024.0 - isuffix = "iB" - def r(count, suffix): - return "%.2f %s%s" % (count, suffix, isuffix) - - if s < 1024: # 1000-1023 get emitted as bytes, even in SI mode - return "%d B" % s - if s < U*U: - return r(s/U, "k") - if s < U*U*U: - return r(s/(U*U), "M") - if s < U*U*U*U: - return r(s/(U*U*U), "G") - if s < U*U*U*U*U: - return r(s/(U*U*U*U), "T") - if s < U*U*U*U*U*U: - return r(s/(U*U*U*U*U), "P") - return r(s/(U*U*U*U*U*U), "E") def print_event(event): event_type, started, result, total_bytes, waiting_time, total_time = event @@ -49,7 +25,7 @@ def print_event(event): abbrev(total_time), abbrev(waiting_time), abbrev(followthrough), - abbreviate_space(total_bytes), + sizeof_fmt_iec(total_bytes), time.ctime(started), )) @@ -108,8 +84,8 @@ def show_usage(args): print(" %d events in %s (%.2f per hour)" % (total, abbrev(elapsed), (3600 * total / elapsed))) rate = total_transit_bytes / elapsed - print(" %s total bytes, %sps" % (abbreviate_space(total_transit_bytes), - abbreviate_space(rate))) + print(" %s total bytes, %sps" % (sizeof_fmt_iec(total_transit_bytes), + sizeof_fmt_iec(rate))) print("", ", ".join(["%s=%d (%d%%)" % (k, counters[k], (100.0 * counters[k] / total)) for k in sorted(counters) diff --git a/src/wormhole/test/test_scripts.py b/src/wormhole/test/test_scripts.py index 3a45b0d..ecf0dee 100644 --- a/src/wormhole/test/test_scripts.py +++ b/src/wormhole/test/test_scripts.py @@ -9,6 +9,7 @@ from .. import __version__ from .common import ServerBase, config from ..cli import cmd_send, cmd_receive from ..errors import TransferError, WrongPasswordError, WelcomeError +from ..util import sizeof_fmt_iec def build_offer(args): @@ -376,8 +377,9 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase): NL=NL) self.failUnlessEqual(send_stdout, expected) elif mode == "file": - self.failUnlessIn("Sending {bytes:d} byte file named '{name}'{NL}" - .format(bytes=len(message), name=send_filename, + self.failUnlessIn("Sending {size:s} file named '{name}'{NL}" + .format(size=sizeof_fmt_iec(len(message)), + name=send_filename, NL=NL), send_stdout) self.failUnlessIn("On the other computer, please run: " "wormhole receive{NL}" @@ -402,8 +404,8 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase): if mode == "text": self.failUnlessEqual(receive_stdout, message+NL) elif mode == "file": - self.failUnlessIn("Receiving file ({bytes:d} bytes) into: {name}" - .format(bytes=len(message), + self.failUnlessIn("Receiving file ({size:s}) into: {name}" + .format(size=sizeof_fmt_iec(len(message)), name=receive_filename), receive_stdout) self.failUnlessIn("Received file written to ", receive_stdout) fn = os.path.join(receive_dir, receive_filename) @@ -411,7 +413,7 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase): with open(fn, "r") as f: self.failUnlessEqual(f.read(), message) elif mode == "directory": - want = (r"Receiving directory \(\d+ bytes\) into: {name}/" + want = (r"Receiving directory \(\d+ \w+\) into: {name}/" .format(name=receive_dirname)) self.failUnless(re.search(want, receive_stdout), (want, receive_stdout)) @@ -511,8 +513,9 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase): (receive_stdout, receive_stderr)) # check sender - self.failUnlessIn("Sending {bytes:d} byte file named '{name}'{NL}" - .format(bytes=len(message), name=send_filename, + self.failUnlessIn("Sending {size:s} file named '{name}'{NL}" + .format(size=sizeof_fmt_iec(len(message)), + name=send_filename, NL=NL), send_stdout) self.failUnlessIn("On the other computer, please run: " "wormhole receive{NL}" diff --git a/src/wormhole/test/test_util.py b/src/wormhole/test/test_util.py index fb58adc..7e377a6 100644 --- a/src/wormhole/test/test_util.py +++ b/src/wormhole/test/test_util.py @@ -38,3 +38,31 @@ class Utils(unittest.TestCase): d = util.bytes_to_dict(b) self.assertIsInstance(d, dict) self.assertEqual(d, {"a": "b", "c": 2}) + + def test_size_fmt_decimal(self): + """test the size formatting routines""" + si_size_map = { + 0: '0 B', # no rounding necessary for those + 1: '1 B', + 142: '142 B', + 999: '999 B', + 1000: '1.00 kB', # rounding starts here + 1001: '1.00 kB', # should be rounded away + 1234: '1.23 kB', # should be rounded down + 1235: '1.24 kB', # should be rounded up + 1010: '1.01 kB', # rounded down as well + 999990000: '999.99 MB', # rounded down + 999990001: '999.99 MB', # rounded down + 999995000: '1.00 GB', # rounded up to next unit + 10**6: '1.00 MB', # and all the remaining units, megabytes + 10**9: '1.00 GB', # gigabytes + 10**12: '1.00 TB', # terabytes + 10**15: '1.00 PB', # petabytes + 10**18: '1.00 EB', # exabytes + 10**21: '1.00 ZB', # zottabytes + 10**24: '1.00 YB', # yottabytes + -1: '-1 B', # negative value + -1010: '-1.01 kB', # negative value with rounding + } + for size, fmt in si_size_map.items(): + self.assertEqual(util.sizeof_fmt_decimal(size), fmt) diff --git a/src/wormhole/util.py b/src/wormhole/util.py index 967243e..a34005f 100644 --- a/src/wormhole/util.py +++ b/src/wormhole/util.py @@ -24,3 +24,22 @@ def bytes_to_dict(b): d = json.loads(b.decode("utf-8")) assert isinstance(d, dict) return d + + +def sizeof_fmt(num, suffix='B', units=None, power=None, sep=' ', precision=2): + for unit in units[:-1]: + if abs(round(num, precision)) < power: + if isinstance(num, int): + return "{}{}{}{}".format(num, sep, unit, suffix) + else: + return "{:3.{}f}{}{}{}".format(num, precision, sep, unit, suffix) + num /= float(power) + return "{:.{}f}{}{}{}".format(num, precision, sep, units[-1], suffix) + + +def sizeof_fmt_iec(num, suffix='B', sep=' ', precision=2): + return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'], power=1024) + + +def sizeof_fmt_decimal(num, suffix='B', sep=' ', precision=2): + return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], power=1000)