From 7aa55e6b65668afd32ccc364b6c311493f7f2e69 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 25 May 2016 00:11:17 -0700 Subject: [PATCH] INCOMPATIBILITY: deliver hints as JSON, not strings The file-send protocol now sends a "hints-v1" key in the "transit" message, which contains a list of JSON data structures that describe the connection hints (a mixture of direct, tor, and relay hints, for now). Previously the direct/tor and relay hints were sent in different keys, and all were sent as strings like "tcp:hostname:1234" which had to be parsed by the recipient. The new structures include a version string, to make it easier to add new types in the future. Transit logs+ignores hints it cannot understand. --- src/wormhole/cli/cmd_receive.py | 20 ++-- src/wormhole/cli/cmd_send.py | 12 +- src/wormhole/test/test_transit_twisted.py | 94 ++++++++------- src/wormhole/transit.py | 138 +++++++++++++++------- 4 files changed, 158 insertions(+), 106 deletions(-) diff --git a/src/wormhole/cli/cmd_receive.py b/src/wormhole/cli/cmd_receive.py index e860958..dd2d244 100644 --- a/src/wormhole/cli/cmd_receive.py +++ b/src/wormhole/cli/cmd_receive.py @@ -130,15 +130,15 @@ class TwistedReceiver: self._msg(u"Verifier %s." % verifier_hex) @inlineCallbacks - def _parse_transit(self, sender_hints, w): + def _parse_transit(self, sender_transit, w): if self._transit_receiver: # TODO: accept multiple messages, add the additional hints to the # existing TransitReceiver return - yield self._build_transit(w, sender_hints) + yield self._build_transit(w, sender_transit) @inlineCallbacks - def _build_transit(self, w, sender_hints): + def _build_transit(self, w, sender_transit): tr = TransitReceiver(self.args.transit_helper, no_listen=self.args.no_listen, tor_manager=self._tor_manager, @@ -148,16 +148,10 @@ class TwistedReceiver: transit_key = w.derive_key(APPID+u"/transit-key", tr.TRANSIT_KEY_LENGTH) tr.set_transit_key(transit_key) - tr.add_their_direct_hints(sender_hints["direct_connection_hints"]) - tr.add_their_relay_hints(sender_hints["relay_connection_hints"]) - - direct_hints = yield tr.get_direct_hints() - relay_hints = yield tr.get_relay_hints() - receiver_hints = { - "direct_connection_hints": direct_hints, - "relay_connection_hints": relay_hints, - } - self._send_data({u"transit": receiver_hints}, w) + tr.add_connection_hints(sender_transit.get("hints-v1", [])) + receiver_hints = yield tr.get_connection_hints() + receiver_transit = {"hints-v1": receiver_hints} + self._send_data({u"transit": receiver_transit}, w) # TODO: send more hints as the TransitReceiver produces them @inlineCallbacks diff --git a/src/wormhole/cli/cmd_send.py b/src/wormhole/cli/cmd_send.py index dbffad2..58ed8de 100644 --- a/src/wormhole/cli/cmd_send.py +++ b/src/wormhole/cli/cmd_send.py @@ -106,11 +106,8 @@ class Sender: self._transit_sender = ts # for now, send this before the main offer - direct_hints = yield ts.get_direct_hints() - sender_hints = {"relay_connection_hints": ts.get_relay_hints(), - "direct_connection_hints": direct_hints, - } - self._send_data({u"transit": sender_hints}, w) + hints = yield ts.get_connection_hints() + self._send_data({u"transit": {"hints-v1": hints}}, w) # TODO: move this down below w.get() transit_key = w.derive_key(APPID+"/transit-key", @@ -146,10 +143,9 @@ class Sender: if not recognized: log.msg("unrecognized message %r" % (them_d,)) - def _handle_transit(self, receiver_hints): + def _handle_transit(self, receiver_transit): ts = self._transit_sender - ts.add_their_direct_hints(receiver_hints["direct_connection_hints"]) - ts.add_their_relay_hints(receiver_hints["relay_connection_hints"]) + ts.add_connection_hints(receiver_transit.get("hints-v1", [])) def _build_offer(self): offer = {} diff --git a/src/wormhole/test/test_transit_twisted.py b/src/wormhole/test/test_transit_twisted.py index bbc33a8..4cacc0a 100644 --- a/src/wormhole/test/test_transit_twisted.py +++ b/src/wormhole/test/test_transit_twisted.py @@ -130,31 +130,38 @@ class Misc(unittest.TestCase): class Hints(unittest.TestCase): def test_endpoint_from_hint(self): c = transit.Common(u"") - ep = c._endpoint_from_hint("tcp:localhost:1234") + ep = c._endpoint_from_hint(transit.DirectTCPV1Hint("localhost", 1234)) self.assertIsInstance(ep, endpoints.HostnameEndpoint) ep = c._endpoint_from_hint("unknown:stuff:yowza:pivlor") self.assertEqual(ep, None) - ep = c._endpoint_from_hint("tooshort") - self.assertEqual(ep, None) class Basic(unittest.TestCase): + @inlineCallbacks def test_relay_hints(self): - URL = u"RELAYURL" - c = transit.Common(URL) - self.assertEqual(c.get_relay_hints(), [URL]) + URL = u"tcp:host:1234" + c = transit.Common(URL, no_listen=True) + hints = yield c.get_connection_hints() + self.assertEqual(hints, [{"type": "relay-v1", + "hints": [{"type": "direct-tcp-v1", + "hostname": u"host", + "port": 1234}], + }]) self.assertRaises(UsageError, transit.Common, 123) + @inlineCallbacks def test_no_relay_hints(self): - c = transit.Common(None) - self.assertEqual(c.get_relay_hints(), []) + c = transit.Common(None, no_listen=True) + hints = yield c.get_connection_hints() + self.assertEqual(hints, []) - def test_bad_hints(self): + def test_ignore_bad_hints(self): c = transit.Common(u"") - self.assertRaises(TypeError, c.add_their_direct_hints, [123]) - c.add_their_direct_hints([u"URL"]) - self.assertRaises(TypeError, c.add_their_relay_hints, [123]) - c.add_their_relay_hints([u"URL"]) + c.add_connection_hints([{"type": "unknown"}]) + c.add_connection_hints([{"type": "relay-v1", + "hints": [{"type": "unknown"}]}]) + self.assertEqual(c._their_direct_hints, []) + self.assertEqual(c._their_relay_hints, []) def test_transit_key_wait(self): KEY = b"123" @@ -214,8 +221,7 @@ class Listener(unittest.TestCase): hints, ep = c._build_listener() self.assertIsInstance(hints, (list, set)) if hints: - self.assertIsInstance(hints[0], type(u"")) - self.assert_(hints[0].startswith(u"tcp:")) + self.assertIsInstance(hints[0], transit.DirectTCPV1Hint) self.assertIsInstance(ep, endpoints.TCP4ServerEndpoint) def test_get_direct_hints(self): @@ -223,7 +229,7 @@ class Listener(unittest.TestCase): c = transit.TransitSender(u"") results = [] - d = c.get_direct_hints() + d = c.get_connection_hints() d.addBoth(results.append) self.assertEqual(len(results), 1) hints = results[0] @@ -232,7 +238,7 @@ class Listener(unittest.TestCase): # start a second listener self.assert_(c._listener) results = [] - d = c.get_direct_hints() + d = c.get_connection_hints() d.addBoth(results.append) self.assertEqual(results, [hints]) @@ -1166,9 +1172,19 @@ class FileConsumer(unittest.TestCase): self.assertEqual(f.getvalue(), b"."*99+b"!") -DIRECT_HINT = u"tcp:direct:1234" -RELAY_HINT = u"tcp:relay:1234" -UNUSABLE_HINT = u"unusable:foo:bar" +DIRECT_HINT = {u"type": u"direct-tcp-v1", + u"hostname": u"direct", u"port": 1234} +RELAY_HINT = {u"type": u"relay-v1", + u"hints": [{u"type": u"direct-tcp-v1", + u"hostname": u"relay", u"port": 1234}]} +UNUSABLE_HINT = {u"type": u"unknown"} +RELAY_HINT2 = {u"type": u"relay-v1", + u"hints": [{u"type": u"direct-tcp-v1", + u"hostname": u"relay", u"port": 1234}, + UNUSABLE_HINT]} +DIRECT_HINT_INTERNAL = transit.DirectTCPV1Hint(u"direct", 1234) +RELAY_HINT_FIRST = transit.DirectTCPV1Hint(u"relay", 1234) +RELAY_HINT_INTERNAL = transit.RelayV1Hint([RELAY_HINT_FIRST]) class Transit(unittest.TestCase): @inlineCallbacks @@ -1176,10 +1192,9 @@ class Transit(unittest.TestCase): clock = task.Clock() s = transit.TransitSender(u"", reactor=clock) s.set_transit_key(b"key") - hints = yield s.get_direct_hints() # start the listener + hints = yield s.get_connection_hints() # start the listener del hints - s.add_their_direct_hints([DIRECT_HINT, UNUSABLE_HINT]) - s.add_their_relay_hints([]) + s.add_connection_hints([DIRECT_HINT, UNUSABLE_HINT]) connectors = [] def _start_connector(ep, description, is_relay=False): @@ -1198,24 +1213,21 @@ class Transit(unittest.TestCase): self.assertEqual(results, ["winner"]) def _endpoint_from_hint(self, hint): - if hint == DIRECT_HINT: + if hint == DIRECT_HINT_INTERNAL: return "direct" - elif hint == RELAY_HINT: + elif hint == RELAY_HINT_FIRST: return "relay" - elif hint == UNUSABLE_HINT: - return None else: - return "ep" + return None @inlineCallbacks def test_wait_for_relay(self): clock = task.Clock() - s = transit.TransitSender(u"", reactor=clock) + s = transit.TransitSender(u"", reactor=clock, no_listen=True) s.set_transit_key(b"key") - hints = yield s.get_direct_hints() # start the listener + hints = yield s.get_connection_hints() # start the listener del hints - s.add_their_direct_hints([DIRECT_HINT, UNUSABLE_HINT]) - s.add_their_relay_hints([RELAY_HINT]) + s.add_connection_hints([DIRECT_HINT, UNUSABLE_HINT, RELAY_HINT]) direct_connectors = [] relay_connectors = [] @@ -1250,12 +1262,11 @@ class Transit(unittest.TestCase): @inlineCallbacks def test_no_direct_hints(self): clock = task.Clock() - s = transit.TransitSender(u"", reactor=clock) + s = transit.TransitSender(u"", reactor=clock, no_listen=True) s.set_transit_key(b"key") - hints = yield s.get_direct_hints() # start the listener + hints = yield s.get_connection_hints() # start the listener del hints - s.add_their_direct_hints([UNUSABLE_HINT]) - s.add_their_relay_hints([RELAY_HINT, UNUSABLE_HINT]) + s.add_connection_hints([UNUSABLE_HINT, RELAY_HINT2]) direct_connectors = [] relay_connectors = [] @@ -1301,14 +1312,11 @@ class Full(unittest.TestCase): s.set_transit_key(KEY) r.set_transit_key(KEY) - shints = yield s.get_direct_hints() - rhints = yield r.get_direct_hints() + shints = yield s.get_connection_hints() + rhints = yield r.get_connection_hints() - s.add_their_direct_hints(rhints) - r.add_their_direct_hints(shints) - - s.add_their_relay_hints([]) - r.add_their_relay_hints([]) + s.add_connection_hints(rhints) + r.add_connection_hints(shints) (x,y) = yield self.doBoth(s.connect(), r.connect()) self.assertIsInstance(x, transit.Connection) diff --git a/src/wormhole/transit.py b/src/wormhole/transit.py index c94e919..43ebc19 100644 --- a/src/wormhole/transit.py +++ b/src/wormhole/transit.py @@ -1,7 +1,10 @@ from __future__ import print_function, absolute_import -import re, sys, time, socket, collections +import re, sys, time, socket +from collections import namedtuple, deque from binascii import hexlify, unhexlify +import six from zope.interface import implementer +from twisted.python import log from twisted.python.runtime import platformType from twisted.internet import (reactor, interfaces, defer, protocol, endpoints, task, address, error) @@ -76,11 +79,26 @@ def build_relay_handshake(key): token = HKDF(key, 32, CTXinfo=b"transit_relay_token") return b"please relay "+hexlify(token)+b"\n" -# The hint format is: TYPE,VALUE= /^([a-zA-Z0-9]+):(.*)$/ . VALUE depends -# upon TYPE, and it can have more colons in it. For TYPE=tcp (the only one -# currently defined), ADDR,PORT = /^(.*):(\d+)$/ , so ADDR can have colons. -# ADDR can be a hostname, ipv4 dotted-quad, or ipv6 colon-hex. If the hint -# publisher wants anonymity, their only hint's ADDR will end in .onion . +# DirectTCPV1Hint and TorTCPV1Hint mean the following protocol: +# * make a TCP connection (possibly via Tor) +# * send the sender/receiver handshake bytes first +# * expect to see the receiver/sender handshake bytes from the other side +# * the sender writes "go\n", the receiver waits for "go\n" +# * the rest of the connection contains transit data +DirectTCPV1Hint = namedtuple("DirectTCPV1Hint", ["hostname", "port"]) +TorTCPV1Hint = namedtuple("TorTCPV1Hint", ["hostname", "port"]) +# RelayV1Hint contains a list of DirectTCPV1Hint and TorTCPV1Hint hints. For +# each one, make the TCP connection, send the relay handshake, then complete +# the rest of the V1 protocol. Only one hint per relay is useful. +RelayV1Hint = namedtuple("RelayV1Hint", ["hints"]) + +def describe_hint(hint): + if isinstance(hint, DirectTCPV1Hint): + return u"tcp:%s:%d" % (hint.hostname, hint.port) + elif isinstance(hint, TorTCPV1Hint): + return u"tor:%s:%d" % (hint.hostname, hint.port) + else: + return str(hint) def parse_hint_tcp(hint): assert isinstance(hint, type(u"")) @@ -104,7 +122,7 @@ def parse_hint_tcp(hint): except ValueError: print("non-numeric port in TCP hint '%s'" % (hint,)) return None - return hint_host, hint_port + return DirectTCPV1Hint(hint_host, hint_port) TIMEOUT=15 @@ -123,8 +141,8 @@ class Connection(protocol.Protocol, policies.TimeoutMixin): self._consumer_bytes_written = 0 self._consumer_bytes_expected = None self._consumer_deferred = None - self._inbound_records = collections.deque() - self._waiting_reads = collections.deque() + self._inbound_records = deque() + self._waiting_reads = deque() def connectionMade(self): debug("handle %r" % (self.transport,)) @@ -555,9 +573,12 @@ class Common: if transit_relay: if not isinstance(transit_relay, type(u"")): raise UsageError - self._transit_relays = [transit_relay] + relay = RelayV1Hint(hints=[parse_hint_tcp(transit_relay)]) + self._transit_relays = [relay] else: self._transit_relays = [] + self._their_direct_hints = [] + self._their_relay_hints = [] self._tor_manager = tor_manager self._transit_key = None self._no_listen = no_listen @@ -572,12 +593,30 @@ class Common: if self._no_listen or self._tor_manager: return ([], None) portnum = allocate_tcp_port() - direct_hints = [u"tcp:%s:%d" % (addr, portnum) + direct_hints = [DirectTCPV1Hint(six.u(addr), portnum) for addr in ipaddrs.find_addresses()] ep = endpoints.serverFromString(reactor, "tcp:%d" % portnum) return direct_hints, ep - def get_direct_hints(self): + @inlineCallbacks + def get_connection_hints(self): + hints = [] + direct_hints = yield self._get_direct_hints() + for dh in direct_hints: + hints.append({u"type": u"direct-tcp-v1", + u"hostname": dh.hostname, + u"port": dh.port, # integer + }) + for relay in self._transit_relays: + rhint = {u"type": u"relay-v1", u"hints": []} + for rh in relay.hints: + rhint[u"hints"].append({u"type": u"direct-tcp-v1", + u"hostname": rh.hostname, + u"port": rh.port}) + hints.append(rhint) + returnValue(hints) + + def _get_direct_hints(self): if self._listener: return defer.succeed(self._my_direct_hints) # there is a slight race here: if someone calls get_direct_hints() a @@ -619,21 +658,41 @@ class Common: self._listener_d.addErrback(lambda f: None) self._listener_d.cancel() - def get_relay_hints(self): - return self._transit_relays + def _parse_tcp_v1_hint(self, hint): + hint_type = hint.get(u"type", u"") + if hint_type not in [u"direct-tcp-v1", u"tor-tcp-v1"]: + log.msg("unknown hint type: %r" % (hint,)) + return None + if not(u"hostname" in hint + and isinstance(hint[u"hostname"], type(u""))): + log.msg("invalid hostname in hint: %r" % (hint,)) + return None + if not(u"port" in hint and isinstance(hint[u"port"], int)): + log.msg("invalid port in hint: %r" % (hint,)) + return None + if hint_type == u"direct-tcp-v1": + return DirectTCPV1Hint(hint[u"hostname"], hint[u"port"]) + else: + return TorTCPV1Hint(hint[u"hostname"], hint[u"port"]) - def add_their_direct_hints(self, hints): + def add_connection_hints(self, hints): for h in hints: - if not isinstance(h, type(u"")): - raise TypeError("hint '%r' should be unicode, not %s" - % (h, type(h))) - self._their_direct_hints = set(hints) - def add_their_relay_hints(self, hints): - for h in hints: - if not isinstance(h, type(u"")): - raise TypeError("hint '%r' should be unicode, not %s" - % (h, type(h))) - self._their_relay_hints = set(hints) + hint_type = h.get(u"type", u"") + if hint_type in [u"direct-tcp-v1", u"tor-tcp-v1"]: + dh = self._parse_tcp_v1_hint(h) + if dh: + self._their_direct_hints.append(dh) + elif hint_type == u"relay-v1": + # TODO: each relay-v1 clause describes a different relay, + # with a set of equally-valid ways to connect to it. Treat + # them as separate relays, instead of merging them all + # together like this. + for rhs in h.get(u"hints", []): + rh = self._parse_tcp_v1_hint(rhs) + if rh: + self._their_relay_hints.append(rh) + else: + log.msg("unknown hint type: %r" % (h,)) def _send_this(self): assert self._transit_key @@ -717,7 +776,7 @@ class Common: ep = self._endpoint_from_hint(hint) if not ep: continue - description = "->%s" % (hint,) + description = "->%s" % describe_hint(hint) d = self._start_connector(ep, description) contenders.append(d) relay_delay = self.RELAY_DELAY @@ -732,7 +791,7 @@ class Common: ep = self._endpoint_from_hint(hint) if not ep: continue - description = "->relay:%s" % (hint,) + description = "->relay:%s" % describe_hint(hint) d = task.deferLater(self._reactor, relay_delay, self._start_connector, ep, description, is_relay=True) @@ -764,22 +823,17 @@ class Common: return d def _endpoint_from_hint(self, hint): - # TODO: use parse_hint_tcp - if ":" not in hint: - return None - pieces = hint.split(":") - hint_type = hint.split(":")[0] - if hint_type == "tor" and self._tor_manager: - return self._tor_manager.get_endpoint_for(pieces[1], int(pieces[2])) - if hint_type != "tcp": - return None - pieces = hint.split(":") if self._tor_manager: - # our TorManager will return None for non-public IPv4 addresses - # and any IPv6 address - return self._tor_manager.get_endpoint_for(pieces[1], int(pieces[2])) - return endpoints.HostnameEndpoint(self._reactor, pieces[1], - int(pieces[2])) + if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)): + # our TorManager will return None for non-public IPv4 + # addresses and any IPv6 address + return self._tor_manager.get_endpoint_for(hint.hostname, + hint.port) + return None + if isinstance(hint, DirectTCPV1Hint): + return endpoints.HostnameEndpoint(self._reactor, + hint.hostname, hint.port) + return None def connection_ready(self, p): # inbound/outbound Connection protocols call this when they finish