use human-readable sizes more broadly

there was a function to "abbreviate" sizes, but it was somewhat
unclear and incomplete. reuse the sizeof_fmt_* set of functions from
the borg backup project (MIT licensed) to implement a more complete
and flexible display that will scale up to the Yottabyte and
beyond. it also supports non-IEC units (like "kibibyte", AKA 1024
bytes) if you fancy that stuff.

this is a workaround for #91: it allows users to better see the size
of the file that will be transfered.

*some* places are still kept in bytes, most notably when receive fails
to receive all bytes ("got %d bytes, wanted %d") because we may want
more clarity there.

text transfers also use the "bytes" suffix (instead of "B") because it
will commonly not reach beyond the KiB range.

note that the test suite only covers decimal (non-IEC) prefix, but it
is assumed to be sufficient to be considered correct.

src/wormhole/cli/cmd_receive.py

@@ -7,7 +7,7 @@ from twisted.python import log
 from ..wormhole import wormhole
 from ..transit import TransitReceiver
 from ..errors import TransferError, WormholeClosedError
-from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr
+from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr, sizeof_fmt_iec
 
 APPID = u"lothar.com/wormhole/text-or-file-xfer"
 
@@ -194,8 +194,8 @@ class TwistedReceiver:
                                                   file_data["filename"])
         self.xfersize = file_data["filesize"]
 
-        self._msg(u"Receiving file (%d bytes) into: %s" %
+        self._msg(u"Receiving file (%s) into: %s" %
-                  (self.xfersize, os.path.basename(self.abs_destname)))
+                  (sizeof_fmt_iec(self.xfersize), os.path.basename(self.abs_destname)))
         self._ask_permission()
         tmp_destname = self.abs_destname + ".tmp"
         return open(tmp_destname, "wb")
@@ -210,10 +210,10 @@ class TwistedReceiver:
                                                   file_data["dirname"])
         self.xfersize = file_data["zipsize"]
 
-        self._msg(u"Receiving directory (%d bytes) into: %s/" %
+        self._msg(u"Receiving directory (%s) into: %s/" %
-                  (self.xfersize, os.path.basename(self.abs_destname)))
+                  (sizeof_fmt_iec(self.xfersize), os.path.basename(self.abs_destname)))
-        self._msg(u"%d files, %d bytes (uncompressed)" %
+        self._msg(u"%d files, %s (uncompressed)" %
-                  (file_data["numfiles"], file_data["numbytes"]))
+                  (file_data["numfiles"], sizeof_fmt_iec(file_data["numbytes"])))
         self._ask_permission()
         return tempfile.SpooledTemporaryFile()
 

src/wormhole/cli/cmd_send.py

@@ -8,7 +8,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue
 from ..errors import TransferError, WormholeClosedError
 from ..wormhole import wormhole
 from ..transit import TransitSender
-from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr
+from ..util import dict_to_bytes, bytes_to_dict, bytes_to_hexstr, sizeof_fmt_iec
 
 APPID = u"lothar.com/wormhole/text-or-file-xfer"
 
@@ -167,7 +167,7 @@ class Sender:
             text = six.moves.input("Text to send: ")
 
         if text is not None:
-            print(u"Sending text message (%d bytes)" % len(text),
+            print(u"Sending text message (%s)" % sizeof_fmt_iec(len(text), suffix='bytes'),
                   file=args.stdout)
             offer = { "message": text }
             fd_to_send = None
@@ -187,7 +187,8 @@ class Sender:
                 "filename": basename,
                 "filesize": filesize,
                 }
-            print(u"Sending %d byte file named '%s'" % (filesize, basename),
+            print(u"Sending %s file named '%s'"
+                  % (sizeof_fmt_iec(filesize), basename),
                   file=args.stdout)
             fd_to_send = open(what, "rb")
             return offer, fd_to_send
@@ -222,8 +223,8 @@ class Sender:
                 "numbytes": num_bytes,
                 "numfiles": num_files,
                 }
-            print(u"Sending directory (%d bytes compressed) named '%s'"
+            print(u"Sending directory (%s compressed) named '%s'"
-                  % (filesize, basename), file=args.stdout)
+                  % (sizeof_fmt_iec(filesize), basename), file=args.stdout)
             return offer, fd_to_send
 
         raise TypeError("'%s' is neither file nor directory" % args.what)

src/wormhole/server/cmd_usage.py

@@ -3,6 +3,7 @@ import os, time, json
 from collections import defaultdict
 import click
 from .database import get_db
+from ..util import sizeof_fmt_iec
 
 def abbrev(t):
     if t is None:
@@ -13,31 +14,6 @@ def abbrev(t):
         return "%.1fms" % (t*1e3)
     return "%.1fus" % (t*1e6)
 
-def abbreviate_space(s, SI=True):
-    if s is None:
-        return "-"
-    if SI:
-        U = 1000.0
-        isuffix = "B"
-    else:
-        U = 1024.0
-        isuffix = "iB"
-    def r(count, suffix):
-        return "%.2f %s%s" % (count, suffix, isuffix)
-
-    if s < 1024: # 1000-1023 get emitted as bytes, even in SI mode
-        return "%d B" % s
-    if s < U*U:
-        return r(s/U, "k")
-    if s < U*U*U:
-        return r(s/(U*U), "M")
-    if s < U*U*U*U:
-        return r(s/(U*U*U), "G")
-    if s < U*U*U*U*U:
-        return r(s/(U*U*U*U), "T")
-    if s < U*U*U*U*U*U:
-        return r(s/(U*U*U*U*U), "P")
-    return r(s/(U*U*U*U*U*U), "E")
 
 def print_event(event):
     event_type, started, result, total_bytes, waiting_time, total_time = event
@@ -49,7 +25,7 @@ def print_event(event):
            abbrev(total_time),
            abbrev(waiting_time),
            abbrev(followthrough),
-           abbreviate_space(total_bytes),
+           sizeof_fmt_iec(total_bytes),
            time.ctime(started),
           ))
 
@@ -108,8 +84,8 @@ def show_usage(args):
         print(" %d events in %s (%.2f per hour)" % (total, abbrev(elapsed),
                                                     (3600 * total / elapsed)))
         rate = total_transit_bytes / elapsed
-        print(" %s total bytes, %sps" % (abbreviate_space(total_transit_bytes),
+        print(" %s total bytes, %sps" % (sizeof_fmt_iec(total_transit_bytes),
-                                         abbreviate_space(rate)))
+                                         sizeof_fmt_iec(rate)))
         print("", ", ".join(["%s=%d (%d%%)" %
                              (k, counters[k], (100.0 * counters[k] / total))
                              for k in sorted(counters)

src/wormhole/test/test_scripts.py

@@ -9,6 +9,7 @@ from .. import __version__
 from .common import ServerBase, config
 from ..cli import cmd_send, cmd_receive
 from ..errors import TransferError, WrongPasswordError, WelcomeError
+from ..util import sizeof_fmt_iec
 
 
 def build_offer(args):
@@ -376,8 +377,9 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
                                                         NL=NL)
             self.failUnlessEqual(send_stdout, expected)
         elif mode == "file":
-            self.failUnlessIn("Sending {bytes:d} byte file named '{name}'{NL}"
+            self.failUnlessIn("Sending {size:s} file named '{name}'{NL}"
-                              .format(bytes=len(message), name=send_filename,
+                              .format(size=sizeof_fmt_iec(len(message)),
+                                      name=send_filename,
                                       NL=NL), send_stdout)
             self.failUnlessIn("On the other computer, please run: "
                               "wormhole receive{NL}"
@@ -402,8 +404,8 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
         if mode == "text":
             self.failUnlessEqual(receive_stdout, message+NL)
         elif mode == "file":
-            self.failUnlessIn("Receiving file ({bytes:d} bytes) into: {name}"
+            self.failUnlessIn("Receiving file ({size:s}) into: {name}"
-                              .format(bytes=len(message),
+                              .format(size=sizeof_fmt_iec(len(message)),
                                       name=receive_filename), receive_stdout)
             self.failUnlessIn("Received file written to ", receive_stdout)
             fn = os.path.join(receive_dir, receive_filename)
@@ -411,7 +413,7 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
             with open(fn, "r") as f:
                 self.failUnlessEqual(f.read(), message)
         elif mode == "directory":
-            want = (r"Receiving directory \(\d+ bytes\) into: {name}/"
+            want = (r"Receiving directory \(\d+ \w+\) into: {name}/"
                     .format(name=receive_dirname))
             self.failUnless(re.search(want, receive_stdout),
                             (want, receive_stdout))
@@ -511,8 +513,9 @@ class PregeneratedCode(ServerBase, ScriptsBase, unittest.TestCase):
                              (receive_stdout, receive_stderr))
 
         # check sender
-        self.failUnlessIn("Sending {bytes:d} byte file named '{name}'{NL}"
+        self.failUnlessIn("Sending {size:s} file named '{name}'{NL}"
-                          .format(bytes=len(message), name=send_filename,
+                          .format(size=sizeof_fmt_iec(len(message)),
+                                  name=send_filename,
                                   NL=NL), send_stdout)
         self.failUnlessIn("On the other computer, please run: "
                           "wormhole receive{NL}"

src/wormhole/test/test_util.py

@@ -38,3 +38,31 @@ class Utils(unittest.TestCase):
         d = util.bytes_to_dict(b)
         self.assertIsInstance(d, dict)
         self.assertEqual(d, {"a": "b", "c": 2})
+
+    def test_size_fmt_decimal(self):
+        """test the size formatting routines"""
+        si_size_map = {
+            0: '0 B',  # no rounding necessary for those
+            1: '1 B',
+            142: '142 B',
+            999: '999 B',
+            1000: '1.00 kB',  # rounding starts here
+            1001: '1.00 kB',  # should be rounded away
+            1234: '1.23 kB',  # should be rounded down
+            1235: '1.24 kB',  # should be rounded up
+            1010: '1.01 kB',  # rounded down as well
+            999990000: '999.99 MB',  # rounded down
+            999990001: '999.99 MB',  # rounded down
+            999995000: '1.00 GB',  # rounded up to next unit
+            10**6: '1.00 MB',  # and all the remaining units, megabytes
+            10**9: '1.00 GB',  # gigabytes
+            10**12: '1.00 TB',  # terabytes
+            10**15: '1.00 PB',  # petabytes
+            10**18: '1.00 EB',  # exabytes
+            10**21: '1.00 ZB',  # zottabytes
+            10**24: '1.00 YB',  # yottabytes
+            -1: '-1 B',  # negative value
+            -1010: '-1.01 kB',  # negative value with rounding
+        }
+        for size, fmt in si_size_map.items():
+            self.assertEqual(util.sizeof_fmt_decimal(size), fmt)

src/wormhole/util.py

@@ -24,3 +24,22 @@ def bytes_to_dict(b):
     d = json.loads(b.decode("utf-8"))
     assert isinstance(d, dict)
     return d
+
+
+def sizeof_fmt(num, suffix='B', units=None, power=None, sep=' ', precision=2):
+    for unit in units[:-1]:
+        if abs(round(num, precision)) < power:
+            if isinstance(num, int):
+                return "{}{}{}{}".format(num, sep, unit, suffix)
+            else:
+                return "{:3.{}f}{}{}{}".format(num, precision, sep, unit, suffix)
+        num /= float(power)
+    return "{:.{}f}{}{}{}".format(num, precision, sep, units[-1], suffix)
+
+
+def sizeof_fmt_iec(num, suffix='B', sep=' ', precision=2):
+    return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'], power=1024)
+
+
+def sizeof_fmt_decimal(num, suffix='B', sep=' ', precision=2):
+    return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], power=1000)