finish py3/async support, needs Twisted >= 15.5.0

The latest Twisted fixes the web.Agent code we need for proper async
support. There's still a daemonization bug that prevents 'wormhole
server start' from succeeding (it hangs).

.travis.yml

@@ -6,7 +6,7 @@ python:
   - "3.4"
   - "3.5"
 install:
-  - pip install . Twisted pyflakes
+  - pip install . "Twisted>=15.5.0" pyflakes
 script:
   - pyflakes src
   - trial wormhole

README.md

@@ -137,9 +137,9 @@ using a key derived from the PAKE phase. See
 This library is released under the MIT license, see LICENSE for details.
 
 This library is compatible with python2.7, 3.3, 3.4, and 3.5 . It is probably
-compatible with py2.6, but the latest Twisted (15.5.0) is not. The async
+compatible with py2.6, but the latest Twisted (15.5.0) is not. The
-support does not yet work with py3, but will in the future once Twisted
+(daemonizing) 'wormhole server start' command does not yet work with py3, but
-itself is finished being ported.
+will in the future once Twisted itself is finished being ported.
 
 This package depends upon the SPAKE2, pynacl, requests, and argparse
 libraries. To run a relay server, use the async support, or run the unit

setup.py

@@ -22,6 +22,8 @@ setup(name="magic-wormhole",
                     ["wormhole = wormhole.scripts.runner:entry"]},
       install_requires=["spake2==0.3", "pynacl", "requests", "argparse",
                         "six"],
+      # for Twisted support, we want Twisted>=15.5.0. Older Twisteds don't
+      # provide sufficient python3 compatibility.
       test_suite="wormhole.test",
       cmdclass=commands,
       )

src/wormhole/test/test_interop.py

@@ -1,5 +1,4 @@
 from __future__ import print_function
-import sys
 from twisted.trial import unittest
 from twisted.internet.defer import gatherResults
 from twisted.internet.threads import deferToThread
@@ -56,9 +55,3 @@ class Basic(ServerBase, unittest.TestCase):
             return self.doBoth([bw.close], tw.close())
         d.addCallback(_done)
         return d
-
-if sys.version_info[0] >= 3:
-    Basic.skip = "twisted is not yet sufficiently ported to py3"
-    # as of 15.4.0, Twisted is still missing:
-    # * web.client.Agent (for all non-EventSource POSTs in transcribe.py)
-    # * python.logfile (to allow daemonization of 'wormhole server')

src/wormhole/test/test_server.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, json
+import json
 import requests
 from six.moves.urllib_parse import urlencode
 from twisted.trial import unittest
@@ -14,7 +14,7 @@ from ..twisted.eventsource_twisted import EventSource
 class Reachable(ServerBase, unittest.TestCase):
 
     def test_getPage(self):
-        # client.getPage requires str/unicode URL, returns bytes
+        # client.getPage requires bytes URL, returns bytes
         url = self.relayurl.replace("wormhole-relay/", "").encode("ascii")
         d = getPage(url)
         def _got(res):
@@ -23,14 +23,9 @@ class Reachable(ServerBase, unittest.TestCase):
         return d
 
     def test_agent(self):
-        # client.Agent is not yet ported: it wants URLs to be both unicode
-        # and bytes at the same time.
-        # https://twistedmatrix.com/trac/ticket/7407
-        if sys.version_info[0] > 2:
-            raise unittest.SkipTest("twisted.web.client.Agent does not yet support py3")
         url = self.relayurl.replace("wormhole-relay/", "").encode("ascii")
         agent = Agent(reactor)
-        d = agent.request("GET", url)
+        d = agent.request(b"GET", url)
         def _check(resp):
             self.failUnlessEqual(resp.code, 200)
             return readBody(resp)
@@ -281,9 +276,6 @@ class API(ServerBase, unittest.TestCase):
         return self._do_watch("watch")
 
     def _do_watch(self, endpoint_name):
-        if sys.version_info[0] >= 3:
-            raise unittest.SkipTest("twisted vs py3")
-
         d = self.post("allocate", {"appid": "app1", "side": "abc"})
         def _allocated(data):
             self.cid = data["channelid"]

src/wormhole/test/test_twisted.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, json
+import json
 from twisted.trial import unittest
 from twisted.internet.defer import gatherResults, succeed
 from ..twisted.transcribe import (Wormhole, UsageError, ChannelManager,
@@ -389,7 +389,7 @@ class Basic(ServerBase, unittest.TestCase):
         d.addCallback(_done)
         return d
 
-data1 = u"""\
+data1 = b"""\
 event: welcome
 data: one and a
 data: two
@@ -420,10 +420,9 @@ class EventSourceClient(unittest.TestCase):
                               (u"e2", u"four"),
                               ])
 
-if sys.version_info[0] >= 3:
+# new py3 support in 15.5.0: web.client.Agent, w.c.downloadPage, twistd
-    Channel.skip = "twisted is not yet sufficiently ported to py3"
+
-    Basic.skip = "twisted is not yet sufficiently ported to py3"
+# However trying 'wormhole server start' with py3/twisted-15.5.0 throws an
-    EventSourceClient.skip = "twisted is not yet sufficiently ported to py3"
+# error in t.i._twistd_unix.UnixApplicationRunner.postApplication, it calls
-    # as of 15.4.0, Twisted is still missing:
+# os.write with str, not bytes. This file does not cover that test (testing
-    # * web.client.Agent (for all non-EventSource POSTs in transcribe.py)
+# daemonization is hard).
-    # * python.logfile (to allow daemonization of 'wormhole server')

src/wormhole/twisted/eventsource_twisted.py

@@ -15,7 +15,7 @@ from ..util.eventual import eventually
 
 class EventSourceParser(basic.LineOnlyReceiver):
     # http://www.w3.org/TR/eventsource/
-    delimiter = "\n"
+    delimiter = b"\n"
 
     def __init__(self, handler):
         self.current_field = None
@@ -97,8 +97,8 @@ class EventSource: # TODO: service.Service
         assert not self.started, "single-use"
         self.started = True
         assert self.url
-        d = self.agent.request("GET", self.url.encode("utf-8"),
+        d = self.agent.request(b"GET", self.url.encode("utf-8"),
-                               Headers({"accept": ["text/event-stream"]}))
+                               Headers({b"accept": [b"text/event-stream"]}))
         d.addCallback(self._connected)
         return d
 

src/wormhole/twisted/transcribe.py

@@ -45,7 +45,7 @@ class DataProducer:
 def post_json(agent, url, request_body):
     # POST a JSON body to a URL, parsing the response as JSON
     data = json.dumps(request_body).encode("utf-8")
-    d = agent.request("POST", url.encode("utf-8"),
+    d = agent.request(b"POST", url.encode("utf-8"),
                       bodyProducer=DataProducer(data))
     def _check_error(resp):
         if resp.code != 200:
@@ -53,19 +53,19 @@ def post_json(agent, url, request_body):
         return resp
     d.addCallback(_check_error)
     d.addCallback(web_client.readBody)
-    d.addCallback(lambda data: json.loads(data))
+    d.addCallback(lambda data: json.loads(data.decode("utf-8")))
     return d
 
 def get_json(agent, url):
     # GET from a URL, parsing the response as JSON
-    d = agent.request("GET", url.encode("utf-8"))
+    d = agent.request(b"GET", url.encode("utf-8"))
     def _check_error(resp):
         if resp.code != 200:
             raise web_error.Error(resp.code, resp.phrase)
         return resp
     d.addCallback(_check_error)
     d.addCallback(web_client.readBody)
-    d.addCallback(lambda data: json.loads(data))
+    d.addCallback(lambda data: json.loads(data.decode("utf-8")))
     return d
 
 class Channel:
@@ -100,6 +100,7 @@ class Channel:
         if not isinstance(phase, type(u"")): raise TypeError(type(phase))
         if not isinstance(msg, type(b"")): raise TypeError(type(msg))
         self._sent_messages.add( (phase,msg) )
+        assert isinstance(self._side, type(u"")), type(self._side)
         payload = {"appid": self._appid,
                    "channelid": self._channelid,
                    "side": self._side,
@@ -321,8 +322,8 @@ class Wormhole:
             "relay_url": self._relay_url,
             "code": self.code,
             "side": self._side,
-            "spake2": json.loads(self.sp.serialize()),
+            "spake2": json.loads(self.sp.serialize().decode("ascii")),
-            "msg1": self.msg1.encode("hex"),
+            "msg1": hexlify(self.msg1).decode("ascii"),
         }
         return json.dumps(data)
 
@@ -330,10 +331,11 @@ class Wormhole:
     def from_serialized(klass, data):
         d = json.loads(data)
         self = klass(d["appid"], d["relay_url"])
-        self._set_side(d["side"].encode("ascii"))
+        self._set_side(d["side"])
         self._set_code_and_channelid(d["code"])
-        self.sp = SPAKE2_Symmetric.from_serialized(json.dumps(d["spake2"]))
+        sp_data = json.dumps(d["spake2"]).encode("ascii")
-        self.msg1 = d["msg1"].decode("hex")
+        self.sp = SPAKE2_Symmetric.from_serialized(sp_data)
+        self.msg1 = unhexlify(d["msg1"].encode("ascii"))
         return self
 
     def derive_key(self, purpose, length=SecretBox.KEY_SIZE):