open.source/magic-wormhole
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

INCOMPATIBILITY: deliver hints as JSON, not strings

Browse Source 
The file-send protocol now sends a "hints-v1" key in the "transit"
message, which contains a list of JSON data structures that describe the
connection hints (a mixture of direct, tor, and relay hints, for now).
Previously the direct/tor and relay hints were sent in different keys,
and all were sent as strings like "tcp:hostname:1234" which had to be
parsed by the recipient.

The new structures include a version string, to make it easier to add
new types in the future. Transit logs+ignores hints it cannot
understand.
This commit is contained in:
Brian Warner 2016-05-25 00:11:17 -07:00
parent afdbbe84c3
commit 7aa55e6b65
4 changed files with 158 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/wormhole/cli/cmd_receive.py
View File

@ -130,15 +130,15 @@ class TwistedReceiver:
self._msg(u"Verifier %s." % verifier_hex) self._msg(u"Verifier %s." % verifier_hex)
@inlineCallbacks @inlineCallbacks
def _parse_transit(self, sender_hints, w): def _parse_transit(self, sender_transit, w):
if self._transit_receiver: if self._transit_receiver:
# TODO: accept multiple messages, add the additional hints to the # TODO: accept multiple messages, add the additional hints to the
# existing TransitReceiver # existing TransitReceiver
return return
yield self._build_transit(w, sender_hints) yield self._build_transit(w, sender_transit)
@inlineCallbacks @inlineCallbacks
def _build_transit(self, w, sender_hints): def _build_transit(self, w, sender_transit):
tr = TransitReceiver(self.args.transit_helper, tr = TransitReceiver(self.args.transit_helper,
no_listen=self.args.no_listen, no_listen=self.args.no_listen,
tor_manager=self._tor_manager, tor_manager=self._tor_manager,
@ -148,16 +148,10 @@ class TwistedReceiver:
transit_key = w.derive_key(APPID+u"/transit-key", tr.TRANSIT_KEY_LENGTH) transit_key = w.derive_key(APPID+u"/transit-key", tr.TRANSIT_KEY_LENGTH)
tr.set_transit_key(transit_key) tr.set_transit_key(transit_key)
tr.add_their_direct_hints(sender_hints["direct_connection_hints"]) tr.add_connection_hints(sender_transit.get("hints-v1", []))
tr.add_their_relay_hints(sender_hints["relay_connection_hints"]) receiver_hints = yield tr.get_connection_hints()
receiver_transit = {"hints-v1": receiver_hints}
direct_hints = yield tr.get_direct_hints() self._send_data({u"transit": receiver_transit}, w)
relay_hints = yield tr.get_relay_hints()
receiver_hints = {
"direct_connection_hints": direct_hints,
"relay_connection_hints": relay_hints,
}
self._send_data({u"transit": receiver_hints}, w)
# TODO: send more hints as the TransitReceiver produces them # TODO: send more hints as the TransitReceiver produces them
@inlineCallbacks @inlineCallbacks

12
src/wormhole/cli/cmd_send.py
View File

@ -106,11 +106,8 @@ class Sender:
self._transit_sender = ts self._transit_sender = ts
# for now, send this before the main offer # for now, send this before the main offer
direct_hints = yield ts.get_direct_hints() hints = yield ts.get_connection_hints()
sender_hints = {"relay_connection_hints": ts.get_relay_hints(), self._send_data({u"transit": {"hints-v1": hints}}, w)
"direct_connection_hints": direct_hints,
}
self._send_data({u"transit": sender_hints}, w)
# TODO: move this down below w.get() # TODO: move this down below w.get()
transit_key = w.derive_key(APPID+"/transit-key", transit_key = w.derive_key(APPID+"/transit-key",
@ -146,10 +143,9 @@ class Sender:
if not recognized: if not recognized:
log.msg("unrecognized message %r" % (them_d,)) log.msg("unrecognized message %r" % (them_d,))
def _handle_transit(self, receiver_hints): def _handle_transit(self, receiver_transit):
ts = self._transit_sender ts = self._transit_sender
ts.add_their_direct_hints(receiver_hints["direct_connection_hints"]) ts.add_connection_hints(receiver_transit.get("hints-v1", []))
ts.add_their_relay_hints(receiver_hints["relay_connection_hints"])
def _build_offer(self): def _build_offer(self):
offer = {} offer = {}

94
src/wormhole/test/test_transit_twisted.py
View File

@ -130,31 +130,38 @@ class Misc(unittest.TestCase):
class Hints(unittest.TestCase): class Hints(unittest.TestCase):
def test_endpoint_from_hint(self): def test_endpoint_from_hint(self):
c = transit.Common(u"") c = transit.Common(u"")
ep = c._endpoint_from_hint("tcp:localhost:1234") ep = c._endpoint_from_hint(transit.DirectTCPV1Hint("localhost", 1234))
self.assertIsInstance(ep, endpoints.HostnameEndpoint) self.assertIsInstance(ep, endpoints.HostnameEndpoint)
ep = c._endpoint_from_hint("unknown:stuff:yowza:pivlor") ep = c._endpoint_from_hint("unknown:stuff:yowza:pivlor")
self.assertEqual(ep, None) self.assertEqual(ep, None)
ep = c._endpoint_from_hint("tooshort")
self.assertEqual(ep, None)
class Basic(unittest.TestCase): class Basic(unittest.TestCase):
@inlineCallbacks
def test_relay_hints(self): def test_relay_hints(self):
URL = u"RELAYURL" URL = u"tcp:host:1234"
c = transit.Common(URL) c = transit.Common(URL, no_listen=True)
self.assertEqual(c.get_relay_hints(), [URL]) hints = yield c.get_connection_hints()
self.assertEqual(hints, [{"type": "relay-v1",
"hints": [{"type": "direct-tcp-v1",
"hostname": u"host",
"port": 1234}],
}])
self.assertRaises(UsageError, transit.Common, 123) self.assertRaises(UsageError, transit.Common, 123)
@inlineCallbacks
def test_no_relay_hints(self): def test_no_relay_hints(self):
c = transit.Common(None) c = transit.Common(None, no_listen=True)
self.assertEqual(c.get_relay_hints(), []) hints = yield c.get_connection_hints()
self.assertEqual(hints, [])
def test_bad_hints(self): def test_ignore_bad_hints(self):
c = transit.Common(u"") c = transit.Common(u"")
self.assertRaises(TypeError, c.add_their_direct_hints, [123]) c.add_connection_hints([{"type": "unknown"}])
c.add_their_direct_hints([u"URL"]) c.add_connection_hints([{"type": "relay-v1",
self.assertRaises(TypeError, c.add_their_relay_hints, [123]) "hints": [{"type": "unknown"}]}])
c.add_their_relay_hints([u"URL"]) self.assertEqual(c._their_direct_hints, [])
self.assertEqual(c._their_relay_hints, [])
def test_transit_key_wait(self): def test_transit_key_wait(self):
KEY = b"123" KEY = b"123"
@ -214,8 +221,7 @@ class Listener(unittest.TestCase):
hints, ep = c._build_listener() hints, ep = c._build_listener()
self.assertIsInstance(hints, (list, set)) self.assertIsInstance(hints, (list, set))
if hints: if hints:
self.assertIsInstance(hints[0], type(u"")) self.assertIsInstance(hints[0], transit.DirectTCPV1Hint)
self.assert_(hints[0].startswith(u"tcp:"))
self.assertIsInstance(ep, endpoints.TCP4ServerEndpoint) self.assertIsInstance(ep, endpoints.TCP4ServerEndpoint)
def test_get_direct_hints(self): def test_get_direct_hints(self):
@ -223,7 +229,7 @@ class Listener(unittest.TestCase):
c = transit.TransitSender(u"") c = transit.TransitSender(u"")
results = [] results = []
d = c.get_direct_hints() d = c.get_connection_hints()
d.addBoth(results.append) d.addBoth(results.append)
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
hints = results[0] hints = results[0]
@ -232,7 +238,7 @@ class Listener(unittest.TestCase):
# start a second listener # start a second listener
self.assert_(c._listener) self.assert_(c._listener)
results = [] results = []
d = c.get_direct_hints() d = c.get_connection_hints()
d.addBoth(results.append) d.addBoth(results.append)
self.assertEqual(results, [hints]) self.assertEqual(results, [hints])
@ -1166,9 +1172,19 @@ class FileConsumer(unittest.TestCase):
self.assertEqual(f.getvalue(), b"."*99+b"!") self.assertEqual(f.getvalue(), b"."*99+b"!")
DIRECT_HINT = u"tcp:direct:1234" DIRECT_HINT = {u"type": u"direct-tcp-v1",
RELAY_HINT = u"tcp:relay:1234" u"hostname": u"direct", u"port": 1234}
UNUSABLE_HINT = u"unusable:foo:bar" RELAY_HINT = {u"type": u"relay-v1",
u"hints": [{u"type": u"direct-tcp-v1",
u"hostname": u"relay", u"port": 1234}]}
UNUSABLE_HINT = {u"type": u"unknown"}
RELAY_HINT2 = {u"type": u"relay-v1",
u"hints": [{u"type": u"direct-tcp-v1",
u"hostname": u"relay", u"port": 1234},
UNUSABLE_HINT]}
DIRECT_HINT_INTERNAL = transit.DirectTCPV1Hint(u"direct", 1234)
RELAY_HINT_FIRST = transit.DirectTCPV1Hint(u"relay", 1234)
RELAY_HINT_INTERNAL = transit.RelayV1Hint([RELAY_HINT_FIRST])
class Transit(unittest.TestCase): class Transit(unittest.TestCase):
@inlineCallbacks @inlineCallbacks
@ -1176,10 +1192,9 @@ class Transit(unittest.TestCase):
clock = task.Clock() clock = task.Clock()
s = transit.TransitSender(u"", reactor=clock) s = transit.TransitSender(u"", reactor=clock)
s.set_transit_key(b"key") s.set_transit_key(b"key")
hints = yield s.get_direct_hints() # start the listener hints = yield s.get_connection_hints() # start the listener
del hints del hints
s.add_their_direct_hints([DIRECT_HINT, UNUSABLE_HINT]) s.add_connection_hints([DIRECT_HINT, UNUSABLE_HINT])
s.add_their_relay_hints([])
connectors = [] connectors = []
def _start_connector(ep, description, is_relay=False): def _start_connector(ep, description, is_relay=False):
@ -1198,24 +1213,21 @@ class Transit(unittest.TestCase):
self.assertEqual(results, ["winner"]) self.assertEqual(results, ["winner"])
def _endpoint_from_hint(self, hint): def _endpoint_from_hint(self, hint):
if hint == DIRECT_HINT: if hint == DIRECT_HINT_INTERNAL:
return "direct" return "direct"
elif hint == RELAY_HINT: elif hint == RELAY_HINT_FIRST:
return "relay" return "relay"
elif hint == UNUSABLE_HINT:
return None
else: else:
return "ep" return None
@inlineCallbacks @inlineCallbacks
def test_wait_for_relay(self): def test_wait_for_relay(self):
clock = task.Clock() clock = task.Clock()
s = transit.TransitSender(u"", reactor=clock) s = transit.TransitSender(u"", reactor=clock, no_listen=True)
s.set_transit_key(b"key") s.set_transit_key(b"key")
hints = yield s.get_direct_hints() # start the listener hints = yield s.get_connection_hints() # start the listener
del hints del hints
s.add_their_direct_hints([DIRECT_HINT, UNUSABLE_HINT]) s.add_connection_hints([DIRECT_HINT, UNUSABLE_HINT, RELAY_HINT])
s.add_their_relay_hints([RELAY_HINT])
direct_connectors = [] direct_connectors = []
relay_connectors = [] relay_connectors = []
@ -1250,12 +1262,11 @@ class Transit(unittest.TestCase):
@inlineCallbacks @inlineCallbacks
def test_no_direct_hints(self): def test_no_direct_hints(self):
clock = task.Clock() clock = task.Clock()
s = transit.TransitSender(u"", reactor=clock) s = transit.TransitSender(u"", reactor=clock, no_listen=True)
s.set_transit_key(b"key") s.set_transit_key(b"key")
hints = yield s.get_direct_hints() # start the listener hints = yield s.get_connection_hints() # start the listener
del hints del hints
s.add_their_direct_hints([UNUSABLE_HINT]) s.add_connection_hints([UNUSABLE_HINT, RELAY_HINT2])
s.add_their_relay_hints([RELAY_HINT, UNUSABLE_HINT])
direct_connectors = [] direct_connectors = []
relay_connectors = [] relay_connectors = []
@ -1301,14 +1312,11 @@ class Full(unittest.TestCase):
s.set_transit_key(KEY) s.set_transit_key(KEY)
r.set_transit_key(KEY) r.set_transit_key(KEY)
shints = yield s.get_direct_hints() shints = yield s.get_connection_hints()
rhints = yield r.get_direct_hints() rhints = yield r.get_connection_hints()
s.add_their_direct_hints(rhints) s.add_connection_hints(rhints)
r.add_their_direct_hints(shints) r.add_connection_hints(shints)
s.add_their_relay_hints([])
r.add_their_relay_hints([])
(x,y) = yield self.doBoth(s.connect(), r.connect()) (x,y) = yield self.doBoth(s.connect(), r.connect())
self.assertIsInstance(x, transit.Connection) self.assertIsInstance(x, transit.Connection)

138
src/wormhole/transit.py
View File

@ -1,7 +1,10 @@
from __future__ import print_function, absolute_import from __future__ import print_function, absolute_import
import re, sys, time, socket, collections import re, sys, time, socket
from collections import namedtuple, deque
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
import six
from zope.interface import implementer from zope.interface import implementer
from twisted.python import log
from twisted.python.runtime import platformType from twisted.python.runtime import platformType
from twisted.internet import (reactor, interfaces, defer, protocol, from twisted.internet import (reactor, interfaces, defer, protocol,
endpoints, task, address, error) endpoints, task, address, error)
@ -76,11 +79,26 @@ def build_relay_handshake(key):
token = HKDF(key, 32, CTXinfo=b"transit_relay_token") token = HKDF(key, 32, CTXinfo=b"transit_relay_token")
return b"please relay "+hexlify(token)+b"\n" return b"please relay "+hexlify(token)+b"\n"
# The hint format is: TYPE,VALUE= /^([a-zA-Z0-9]+):(.*)$/ . VALUE depends # DirectTCPV1Hint and TorTCPV1Hint mean the following protocol:
# upon TYPE, and it can have more colons in it. For TYPE=tcp (the only one # * make a TCP connection (possibly via Tor)
# currently defined), ADDR,PORT = /^(.*):(\d+)$/ , so ADDR can have colons. # * send the sender/receiver handshake bytes first
# ADDR can be a hostname, ipv4 dotted-quad, or ipv6 colon-hex. If the hint # * expect to see the receiver/sender handshake bytes from the other side
# publisher wants anonymity, their only hint's ADDR will end in .onion . # * the sender writes "go\n", the receiver waits for "go\n"
# * the rest of the connection contains transit data
DirectTCPV1Hint = namedtuple("DirectTCPV1Hint", ["hostname", "port"])
TorTCPV1Hint = namedtuple("TorTCPV1Hint", ["hostname", "port"])
# RelayV1Hint contains a list of DirectTCPV1Hint and TorTCPV1Hint hints. For
# each one, make the TCP connection, send the relay handshake, then complete
# the rest of the V1 protocol. Only one hint per relay is useful.
RelayV1Hint = namedtuple("RelayV1Hint", ["hints"])
def describe_hint(hint):
if isinstance(hint, DirectTCPV1Hint):
return u"tcp:%s:%d" % (hint.hostname, hint.port)
elif isinstance(hint, TorTCPV1Hint):
return u"tor:%s:%d" % (hint.hostname, hint.port)
else:
return str(hint)
def parse_hint_tcp(hint): def parse_hint_tcp(hint):
assert isinstance(hint, type(u"")) assert isinstance(hint, type(u""))
@ -104,7 +122,7 @@ def parse_hint_tcp(hint):
except ValueError: except ValueError:
print("non-numeric port in TCP hint '%s'" % (hint,)) print("non-numeric port in TCP hint '%s'" % (hint,))
return None return None
return hint_host, hint_port return DirectTCPV1Hint(hint_host, hint_port)
TIMEOUT=15 TIMEOUT=15
@ -123,8 +141,8 @@ class Connection(protocol.Protocol, policies.TimeoutMixin):
self._consumer_bytes_written = 0 self._consumer_bytes_written = 0
self._consumer_bytes_expected = None self._consumer_bytes_expected = None
self._consumer_deferred = None self._consumer_deferred = None
self._inbound_records = collections.deque() self._inbound_records = deque()
self._waiting_reads = collections.deque() self._waiting_reads = deque()
def connectionMade(self): def connectionMade(self):
debug("handle %r" % (self.transport,)) debug("handle %r" % (self.transport,))
@ -555,9 +573,12 @@ class Common:
if transit_relay: if transit_relay:
if not isinstance(transit_relay, type(u"")): if not isinstance(transit_relay, type(u"")):
raise UsageError raise UsageError
self._transit_relays = [transit_relay] relay = RelayV1Hint(hints=[parse_hint_tcp(transit_relay)])
self._transit_relays = [relay]
else: else:
self._transit_relays = [] self._transit_relays = []
self._their_direct_hints = []
self._their_relay_hints = []
self._tor_manager = tor_manager self._tor_manager = tor_manager
self._transit_key = None self._transit_key = None
self._no_listen = no_listen self._no_listen = no_listen
@ -572,12 +593,30 @@ class Common:
if self._no_listen or self._tor_manager: if self._no_listen or self._tor_manager:
return ([], None) return ([], None)
portnum = allocate_tcp_port() portnum = allocate_tcp_port()
direct_hints = [u"tcp:%s:%d" % (addr, portnum) direct_hints = [DirectTCPV1Hint(six.u(addr), portnum)
for addr in ipaddrs.find_addresses()] for addr in ipaddrs.find_addresses()]
ep = endpoints.serverFromString(reactor, "tcp:%d" % portnum) ep = endpoints.serverFromString(reactor, "tcp:%d" % portnum)
return direct_hints, ep return direct_hints, ep
def get_direct_hints(self): @inlineCallbacks
def get_connection_hints(self):
hints = []
direct_hints = yield self._get_direct_hints()
for dh in direct_hints:
hints.append({u"type": u"direct-tcp-v1",
u"hostname": dh.hostname,
u"port": dh.port, # integer
})
for relay in self._transit_relays:
rhint = {u"type": u"relay-v1", u"hints": []}
for rh in relay.hints:
rhint[u"hints"].append({u"type": u"direct-tcp-v1",
u"hostname": rh.hostname,
u"port": rh.port})
hints.append(rhint)
returnValue(hints)
def _get_direct_hints(self):
if self._listener: if self._listener:
return defer.succeed(self._my_direct_hints) return defer.succeed(self._my_direct_hints)
# there is a slight race here: if someone calls get_direct_hints() a # there is a slight race here: if someone calls get_direct_hints() a
@ -619,21 +658,41 @@ class Common:
self._listener_d.addErrback(lambda f: None) self._listener_d.addErrback(lambda f: None)
self._listener_d.cancel() self._listener_d.cancel()
def get_relay_hints(self): def _parse_tcp_v1_hint(self, hint):
return self._transit_relays hint_type = hint.get(u"type", u"")
if hint_type not in [u"direct-tcp-v1", u"tor-tcp-v1"]:
log.msg("unknown hint type: %r" % (hint,))
return None
if not(u"hostname" in hint
and isinstance(hint[u"hostname"], type(u""))):
log.msg("invalid hostname in hint: %r" % (hint,))
return None
if not(u"port" in hint and isinstance(hint[u"port"], int)):
log.msg("invalid port in hint: %r" % (hint,))
return None
if hint_type == u"direct-tcp-v1":
return DirectTCPV1Hint(hint[u"hostname"], hint[u"port"])
else:
return TorTCPV1Hint(hint[u"hostname"], hint[u"port"])
def add_their_direct_hints(self, hints): def add_connection_hints(self, hints):
for h in hints: for h in hints:
if not isinstance(h, type(u"")): hint_type = h.get(u"type", u"")
raise TypeError("hint '%r' should be unicode, not %s" if hint_type in [u"direct-tcp-v1", u"tor-tcp-v1"]:
% (h, type(h))) dh = self._parse_tcp_v1_hint(h)
self._their_direct_hints = set(hints) if dh:
def add_their_relay_hints(self, hints): self._their_direct_hints.append(dh)
for h in hints: elif hint_type == u"relay-v1":
if not isinstance(h, type(u"")): # TODO: each relay-v1 clause describes a different relay,
raise TypeError("hint '%r' should be unicode, not %s" # with a set of equally-valid ways to connect to it. Treat
% (h, type(h))) # them as separate relays, instead of merging them all
self._their_relay_hints = set(hints) # together like this.
for rhs in h.get(u"hints", []):
rh = self._parse_tcp_v1_hint(rhs)
if rh:
self._their_relay_hints.append(rh)
else:
log.msg("unknown hint type: %r" % (h,))
def _send_this(self): def _send_this(self):
assert self._transit_key assert self._transit_key
@ -717,7 +776,7 @@ class Common:
ep = self._endpoint_from_hint(hint) ep = self._endpoint_from_hint(hint)
if not ep: if not ep:
continue continue
description = "->%s" % (hint,) description = "->%s" % describe_hint(hint)
d = self._start_connector(ep, description) d = self._start_connector(ep, description)
contenders.append(d) contenders.append(d)
relay_delay = self.RELAY_DELAY relay_delay = self.RELAY_DELAY
@ -732,7 +791,7 @@ class Common:
ep = self._endpoint_from_hint(hint) ep = self._endpoint_from_hint(hint)
if not ep: if not ep:
continue continue
description = "->relay:%s" % (hint,) description = "->relay:%s" % describe_hint(hint)
d = task.deferLater(self._reactor, relay_delay, d = task.deferLater(self._reactor, relay_delay,
self._start_connector, ep, description, self._start_connector, ep, description,
is_relay=True) is_relay=True)
@ -764,22 +823,17 @@ class Common:
return d return d
def _endpoint_from_hint(self, hint): def _endpoint_from_hint(self, hint):
# TODO: use parse_hint_tcp
if ":" not in hint:
return None
pieces = hint.split(":")
hint_type = hint.split(":")[0]
if hint_type == "tor" and self._tor_manager:
return self._tor_manager.get_endpoint_for(pieces[1], int(pieces[2]))
if hint_type != "tcp":
return None
pieces = hint.split(":")
if self._tor_manager: if self._tor_manager:
# our TorManager will return None for non-public IPv4 addresses if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)):
# and any IPv6 address # our TorManager will return None for non-public IPv4
return self._tor_manager.get_endpoint_for(pieces[1], int(pieces[2])) # addresses and any IPv6 address
return endpoints.HostnameEndpoint(self._reactor, pieces[1], return self._tor_manager.get_endpoint_for(hint.hostname,
int(pieces[2])) hint.port)
return None
if isinstance(hint, DirectTCPV1Hint):
return endpoints.HostnameEndpoint(self._reactor,
hint.hostname, hint.port)
return None
def connection_ready(self, p): def connection_ready(self, p):
# inbound/outbound Connection protocols call this when they finish # inbound/outbound Connection protocols call this when they finish