tests.ServerBase: print message when threadpool is not idle

The previous commits improve test failures by dropping relay connections
at shutdown, and flunking a test quickly when one client fails but the
other one hangs.

If that doesn't work (say, some client has a time.sleep(), or other
stall that isn't affected by the relay shutdown), we'll be left with an
active thread holding that hanging client.

This patch adds a check to wormhole.test.common.ServerBase.tearDown that
looks for active threads, waits a second (after stopService), then
checks the threadpool again. If the threadpool is empty, everything is
fine. If not, it prints a message (to stdout) to inform the impatient
user why the test is probably hanging.

src/wormhole/test/common.py

@@ -1,4 +1,6 @@
 from twisted.application import service
+from twisted.internet import reactor, defer
+from twisted.python import log
 from ..twisted.util import allocate_ports
 from ..servers.server import RelayServer
 from .. import __version__
@@ -22,4 +24,31 @@ class ServerBase:
         return d
 
     def tearDown(self):
+        # Unit tests that spawn a (blocking) client in a thread might still
+        # have threads running at this point, if one is stuck waiting for a
+        # message from a companion which has exited with an error. Our
+        # relay's .stopService() drops all connections, which ought to
+        # encourage those threads to terminate soon. If they don't, print a
+        # warning to ease debugging.
+        tp = reactor.getThreadPool()
+        if not tp.working:
             return self.sp.stopService()
+        # disconnect all callers
+        d = defer.maybeDeferred(self.sp.stopService)
+        wait_d = defer.Deferred()
+        # wait a second, then check to see if it worked
+        reactor.callLater(1.0, wait_d.callback, None)
+        def _later(res):
+            if len(tp.working):
+                log.msg("wormhole.test.common.ServerBase.tearDown:"
+                        " I was unable to convince all threads to exit.")
+                tp.dumpStats()
+                print("tearDown warning: threads are still active")
+                print("This test will probably hang until one of the"
+                      " clients gives up of their own accord.")
+            else:
+                log.msg("wormhole.test.common.ServerBase.tearDown:"
+                        " I convinced all threads to exit.")
+            return d
+        wait_d.addCallback(_later)
+        return wait_d