open.source/magic-wormhole
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use human-readable sizes more broadly

Browse Source 
there was a function to "abbreviate" sizes, but it was somewhat
unclear and incomplete. reuse the sizeof_fmt_* set of functions from
the borg backup project (MIT licensed) to implement a more complete
and flexible display that will scale up to the Yottabyte and
beyond. it also supports non-IEC units (like "kibibyte", AKA 1024
bytes) if you fancy that stuff.

this is a workaround for #91: it allows users to better see the size
of the file that will be transfered.

*some* places are still kept in bytes, most notably when receive fails
to receive all bytes ("got %d bytes, wanted %d") because we may want
more clarity there.

text transfers also use the "bytes" suffix (instead of "B") because it
will commonly not reach beyond the KiB range.

note that the test suite only covers decimal (non-IEC) prefix, but it
is assumed to be sufficient to be considered correct.
This commit is contained in:
Antoine Beaupré 2016-11-09 15:14:01 -05:00
parent 007d76c145
commit 047af4b27d
6 changed files with 74 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/wormhole/cli/cmd_receive.py
View File

@ -7,7 +7,7 @@ from twisted.python import log
from ..wormhole import wormhole
from ..transit import TransitReceiver
from ..errors import TransferError, WormholeClosedError
from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr
from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr, sizeof_fmt_iec
APPID = u"lothar.com/wormhole/text-or-file-xfer"
@ -194,8 +194,8 @@ class TwistedReceiver:
file_data["filename"])
self.xfersize = file_data["filesize"]
self._msg(u"Receiving file (%d bytes) into: %s" %
(self.xfersize, os.path.basename(self.abs_destname)))
self._msg(u"Receiving file (%s) into: %s" %
(sizeof_fmt_iec(self.xfersize), os.path.basename(self.abs_destname)))
self._ask_permission()
tmp_destname = self.abs_destname + ".tmp"
return open(tmp_destname, "wb")
@ -210,10 +210,10 @@ class TwistedReceiver:
file_data["dirname"])
self.xfersize = file_data["zipsize"]
self._msg(u"Receiving directory (%d bytes) into: %s/" %
(self.xfersize, os.path.basename(self.abs_destname)))
self._msg(u"%d files, %d bytes (uncompressed)" %
(file_data["numfiles"], file_data["numbytes"]))
self._msg(u"Receiving directory (%s) into: %s/" %
(sizeof_fmt_iec(self.xfersize), os.path.basename(self.abs_destname)))
self._msg(u"%d files, %s (uncompressed)" %
(file_data["numfiles"], sizeof_fmt_iec(file_data["numbytes"])))
self._ask_permission()
return tempfile.SpooledTemporaryFile()

11
src/wormhole/cli/cmd_send.py
View File

@ -8,7 +8,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue
from ..errors import TransferError, WormholeClosedError
from ..wormhole import wormhole
from ..transit import TransitSender
from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr
from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr, sizeof_fmt_iec
APPID = u"lothar.com/wormhole/text-or-file-xfer"
@ -167,7 +167,7 @@ class Sender:
text = six.moves.input("Text to send: ")
if text is not None:
print(u"Sending text message (%d bytes)" % len(text),
print(u"Sending text message (%s)" % sizeof_fmt_iec(len(text), suffix='bytes'),
file=args.stdout)
offer = { "message": text }
fd_to_send = None
@ -187,7 +187,8 @@ class Sender:
"filename": basename,
"filesize": filesize,
}
print(u"Sending %d byte file named '%s'" % (filesize, basename),
print(u"Sending %s file named '%s'"
% (sizeof_fmt_iec(filesize), basename),
file=args.stdout)
fd_to_send = open(what, "rb")
return offer, fd_to_send
@ -222,8 +223,8 @@ class Sender:
"numbytes": num_bytes,
"numfiles": num_files,
}
print(u"Sending directory (%d bytes compressed) named '%s'"
% (filesize, basename), file=args.stdout)
print(u"Sending directory (%s compressed) named '%s'"
% (sizeof_fmt_iec(filesize), basename), file=args.stdout)
return offer, fd_to_send
raise TypeError("'%s' is neither file nor directory" % args.what)

32
src/wormhole/server/cmd_usage.py
View File

@ -3,6 +3,7 @@ import os, time, json
from collections import defaultdict
import click
from .database import get_db
from ..util import sizeof_fmt_iec
def abbrev(t):
if t is None:
@ -13,31 +14,6 @@ def abbrev(t):
return "%.1fms" % (t*1e3)
return "%.1fus" % (t*1e6)
def abbreviate_space(s, SI=True):
if s is None:
return "-"
if SI:
U = 1000.0
isuffix = "B"
else:
U = 1024.0
isuffix = "iB"
def r(count, suffix):
return "%.2f %s%s" % (count, suffix, isuffix)
if s < 1024: # 1000-1023 get emitted as bytes, even in SI mode
return "%d B" % s
if s < U*U:
return r(s/U, "k")
if s < U*U*U:
return r(s/(U*U), "M")
if s < U*U*U*U:
return r(s/(U*U*U), "G")
if s < U*U*U*U*U:
return r(s/(U*U*U*U), "T")
if s < U*U*U*U*U*U:
return r(s/(U*U*U*U*U), "P")
return r(s/(U*U*U*U*U*U), "E")
def print_event(event):
event_type, started, result, total_bytes, waiting_time, total_time = event
@ -49,7 +25,7 @@ def print_event(event):
abbrev(total_time),
abbrev(waiting_time),
abbrev(followthrough),
abbreviate_space(total_bytes),
sizeof_fmt_iec(total_bytes),
time.ctime(started),
))
@ -108,8 +84,8 @@ def show_usage(args):
print(" %d events in %s (%.2f per hour)" % (total, abbrev(elapsed),
(3600 * total / elapsed)))
rate = total_transit_bytes / elapsed
print(" %s total bytes, %sps" % (abbreviate_space(total_transit_bytes),
abbreviate_space(rate)))
print(" %s total bytes, %sps" % (sizeof_fmt_iec(total_transit_bytes),
sizeof_fmt_iec(rate)))
print("", ", ".join(["%s=%d (%d%%)" %
(k, counters[k], (100.0 * counters[k] / total))
for k in sorted(counters)

17
src/wormhole/test/test_scripts.py
View File

@ -9,6 +9,7 @@ from .. import __version__
from .common import ServerBase, config
from ..cli import cmd_send, cmd_receive
from ..errors import TransferError, WrongPasswordError, WelcomeError
from ..util import sizeof_fmt_iec
def build_offer(args):
@ -376,8 +377,9 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
NL=NL)
self.failUnlessEqual(send_stdout, expected)
elif mode == "file":
self.failUnlessIn("Sending {bytes:d} byte file named '{name}'{NL}"
.format(bytes=len(message), name=send_filename,
self.failUnlessIn("Sending {size:s} file named '{name}'{NL}"
.format(size=sizeof_fmt_iec(len(message)),
name=send_filename,
NL=NL), send_stdout)
self.failUnlessIn("On the other computer, please run: "
"wormhole receive{NL}"
@ -402,8 +404,8 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
if mode == "text":
self.failUnlessEqual(receive_stdout, message+NL)
elif mode == "file":
self.failUnlessIn("Receiving file ({bytes:d} bytes) into: {name}"
.format(bytes=len(message),
self.failUnlessIn("Receiving file ({size:s}) into: {name}"
.format(size=sizeof_fmt_iec(len(message)),
name=receive_filename), receive_stdout)
self.failUnlessIn("Received file written to ", receive_stdout)
fn = os.path.join(receive_dir, receive_filename)
@ -411,7 +413,7 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
with open(fn, "r") as f:
self.failUnlessEqual(f.read(), message)
elif mode == "directory":
want = (r"Receiving directory \(\d+ bytes\) into: {name}/"
want = (r"Receiving directory \(\d+ \w+\) into: {name}/"
.format(name=receive_dirname))
self.failUnless(re.search(want, receive_stdout),
(want, receive_stdout))
@ -511,8 +513,9 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
(receive_stdout, receive_stderr))
# check sender
self.failUnlessIn("Sending {bytes:d} byte file named '{name}'{NL}"
.format(bytes=len(message), name=send_filename,
self.failUnlessIn("Sending {size:s} file named '{name}'{NL}"
.format(size=sizeof_fmt_iec(len(message)),
name=send_filename,
NL=NL), send_stdout)
self.failUnlessIn("On the other computer, please run: "
"wormhole receive{NL}"

28
src/wormhole/test/test_util.py
View File

@ -38,3 +38,31 @@ class Utils(unittest.TestCase):
d = util.bytes_to_dict(b)
self.assertIsInstance(d, dict)
self.assertEqual(d, {"a": "b", "c": 2})
def test_size_fmt_decimal(self):
"""test the size formatting routines"""
si_size_map = {
0: '0 B', # no rounding necessary for those
1: '1 B',
142: '142 B',
999: '999 B',
1000: '1.00 kB', # rounding starts here
1001: '1.00 kB', # should be rounded away
1234: '1.23 kB', # should be rounded down
1235: '1.24 kB', # should be rounded up
1010: '1.01 kB', # rounded down as well
999990000: '999.99 MB', # rounded down
999990001: '999.99 MB', # rounded down
999995000: '1.00 GB', # rounded up to next unit
10**6: '1.00 MB', # and all the remaining units, megabytes
10**9: '1.00 GB', # gigabytes
10**12: '1.00 TB', # terabytes
10**15: '1.00 PB', # petabytes
10**18: '1.00 EB', # exabytes
10**21: '1.00 ZB', # zottabytes
10**24: '1.00 YB', # yottabytes
-1: '-1 B', # negative value
-1010: '-1.01 kB', # negative value with rounding
}
for size, fmt in si_size_map.items():
self.assertEqual(util.sizeof_fmt_decimal(size), fmt)

19
src/wormhole/util.py
View File

@ -24,3 +24,22 @@ def bytes_to_dict(b):
d = json.loads(b.decode("utf-8"))
assert isinstance(d, dict)
return d
def sizeof_fmt(num, suffix='B', units=None, power=None, sep=' ', precision=2):
for unit in units[:-1]:
if abs(round(num, precision)) < power:
if isinstance(num, int):
return "{}{}{}{}".format(num, sep, unit, suffix)
else:
return "{:3.{}f}{}{}{}".format(num, precision, sep, unit, suffix)
num /= float(power)
return "{:.{}f}{}{}{}".format(num, precision, sep, units[-1], suffix)
def sizeof_fmt_iec(num, suffix='B', sep=' ', precision=2):
return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'], power=1024)
def sizeof_fmt_decimal(num, suffix='B', sep=' ', precision=2):
return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], power=1000)