open.source/magic-wormhole
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

INCOMPATIBILITY: deliver hints as JSON, not strings

Browse Source 
The file-send protocol now sends a "hints-v1" key in the "transit"
message, which contains a list of JSON data structures that describe the
connection hints (a mixture of direct, tor, and relay hints, for now).
Previously the direct/tor and relay hints were sent in different keys,
and all were sent as strings like "tcp:hostname:1234" which had to be
parsed by the recipient.

The new structures include a version string, to make it easier to add
new types in the future. Transit logs+ignores hints it cannot
understand.
This commit is contained in:
Brian Warner 2016-05-25 00:11:17 -07:00
parent afdbbe84c3
commit 7aa55e6b65
4 changed files with 158 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/wormhole/cli/cmd_receive.py
View File

@ -130,15 +130,15 @@ class TwistedReceiver:
self._msg(u"Verifier %s." % verifier_hex)
@inlineCallbacks
def _parse_transit(self, sender_hints, w):
def _parse_transit(self, sender_transit, w):
if self._transit_receiver:
# TODO: accept multiple messages, add the additional hints to the
# existing TransitReceiver
return
yield self._build_transit(w, sender_hints)
yield self._build_transit(w, sender_transit)
@inlineCallbacks
def _build_transit(self, w, sender_hints):
def _build_transit(self, w, sender_transit):
tr = TransitReceiver(self.args.transit_helper,
no_listen=self.args.no_listen,
tor_manager=self._tor_manager,
@ -148,16 +148,10 @@ class TwistedReceiver:
transit_key = w.derive_key(APPID+u"/transit-key", tr.TRANSIT_KEY_LENGTH)
tr.set_transit_key(transit_key)
tr.add_their_direct_hints(sender_hints["direct_connection_hints"])
tr.add_their_relay_hints(sender_hints["relay_connection_hints"])
direct_hints = yield tr.get_direct_hints()
relay_hints = yield tr.get_relay_hints()
receiver_hints = {
"direct_connection_hints": direct_hints,
"relay_connection_hints": relay_hints,
}
self._send_data({u"transit": receiver_hints}, w)
tr.add_connection_hints(sender_transit.get("hints-v1", []))
receiver_hints = yield tr.get_connection_hints()
receiver_transit = {"hints-v1": receiver_hints}
self._send_data({u"transit": receiver_transit}, w)
# TODO: send more hints as the TransitReceiver produces them
@inlineCallbacks

12
src/wormhole/cli/cmd_send.py
View File

@ -106,11 +106,8 @@ class Sender:
self._transit_sender = ts
# for now, send this before the main offer
direct_hints = yield ts.get_direct_hints()
sender_hints = {"relay_connection_hints": ts.get_relay_hints(),
"direct_connection_hints": direct_hints,
}
self._send_data({u"transit": sender_hints}, w)
hints = yield ts.get_connection_hints()
self._send_data({u"transit": {"hints-v1": hints}}, w)
# TODO: move this down below w.get()
transit_key = w.derive_key(APPID+"/transit-key",
@ -146,10 +143,9 @@ class Sender:
if not recognized:
log.msg("unrecognized message %r" % (them_d,))
def _handle_transit(self, receiver_hints):
def _handle_transit(self, receiver_transit):
ts = self._transit_sender
ts.add_their_direct_hints(receiver_hints["direct_connection_hints"])
ts.add_their_relay_hints(receiver_hints["relay_connection_hints"])
ts.add_connection_hints(receiver_transit.get("hints-v1", []))
def _build_offer(self):
offer = {}

94
src/wormhole/test/test_transit_twisted.py
View File

@ -130,31 +130,38 @@ class Misc(unittest.TestCase):
class Hints(unittest.TestCase):
def test_endpoint_from_hint(self):
c = transit.Common(u"")
ep = c._endpoint_from_hint("tcp:localhost:1234")
ep = c._endpoint_from_hint(transit.DirectTCPV1Hint("localhost", 1234))
self.assertIsInstance(ep, endpoints.HostnameEndpoint)
ep = c._endpoint_from_hint("unknown:stuff:yowza:pivlor")
self.assertEqual(ep, None)
ep = c._endpoint_from_hint("tooshort")
self.assertEqual(ep, None)
class Basic(unittest.TestCase):
@inlineCallbacks
def test_relay_hints(self):
URL = u"RELAYURL"
c = transit.Common(URL)
self.assertEqual(c.get_relay_hints(), [URL])
URL = u"tcp:host:1234"
c = transit.Common(URL, no_listen=True)
hints = yield c.get_connection_hints()
self.assertEqual(hints, [{"type": "relay-v1",
"hints": [{"type": "direct-tcp-v1",
"hostname": u"host",
"port": 1234}],
}])
self.assertRaises(UsageError, transit.Common, 123)
@inlineCallbacks
def test_no_relay_hints(self):
c = transit.Common(None)
self.assertEqual(c.get_relay_hints(), [])
c = transit.Common(None, no_listen=True)
hints = yield c.get_connection_hints()
self.assertEqual(hints, [])
def test_bad_hints(self):
def test_ignore_bad_hints(self):
c = transit.Common(u"")
self.assertRaises(TypeError, c.add_their_direct_hints, [123])
c.add_their_direct_hints([u"URL"])
self.assertRaises(TypeError, c.add_their_relay_hints, [123])
c.add_their_relay_hints([u"URL"])
c.add_connection_hints([{"type": "unknown"}])
c.add_connection_hints([{"type": "relay-v1",
"hints": [{"type": "unknown"}]}])
self.assertEqual(c._their_direct_hints, [])
self.assertEqual(c._their_relay_hints, [])
def test_transit_key_wait(self):
KEY = b"123"
@ -214,8 +221,7 @@ class Listener(unittest.TestCase):
hints, ep = c._build_listener()
self.assertIsInstance(hints, (list, set))
if hints:
self.assertIsInstance(hints[0], type(u""))
self.assert_(hints[0].startswith(u"tcp:"))
self.assertIsInstance(hints[0], transit.DirectTCPV1Hint)
self.assertIsInstance(ep, endpoints.TCP4ServerEndpoint)
def test_get_direct_hints(self):
@ -223,7 +229,7 @@ class Listener(unittest.TestCase):
c = transit.TransitSender(u"")
results = []
d = c.get_direct_hints()
d = c.get_connection_hints()
d.addBoth(results.append)
self.assertEqual(len(results), 1)
hints = results[0]
@ -232,7 +238,7 @@ class Listener(unittest.TestCase):
# start a second listener
self.assert_(c._listener)
results = []
d = c.get_direct_hints()
d = c.get_connection_hints()
d.addBoth(results.append)
self.assertEqual(results, [hints])
@ -1166,9 +1172,19 @@ class FileConsumer(unittest.TestCase):
self.assertEqual(f.getvalue(), b"."*99+b"!")
DIRECT_HINT = u"tcp:direct:1234"
RELAY_HINT = u"tcp:relay:1234"
UNUSABLE_HINT = u"unusable:foo:bar"
DIRECT_HINT = {u"type": u"direct-tcp-v1",
u"hostname": u"direct", u"port": 1234}
RELAY_HINT = {u"type": u"relay-v1",
u"hints": [{u"type": u"direct-tcp-v1",
u"hostname": u"relay", u"port": 1234}]}
UNUSABLE_HINT = {u"type": u"unknown"}
RELAY_HINT2 = {u"type": u"relay-v1",
u"hints": [{u"type": u"direct-tcp-v1",
u"hostname": u"relay", u"port": 1234},
UNUSABLE_HINT]}
DIRECT_HINT_INTERNAL = transit.DirectTCPV1Hint(u"direct", 1234)
RELAY_HINT_FIRST = transit.DirectTCPV1Hint(u"relay", 1234)
RELAY_HINT_INTERNAL = transit.RelayV1Hint([RELAY_HINT_FIRST])
class Transit(unittest.TestCase):
@inlineCallbacks
@ -1176,10 +1192,9 @@ class Transit(unittest.TestCase):
clock = task.Clock()
s = transit.TransitSender(u"", reactor=clock)
s.set_transit_key(b"key")
hints = yield s.get_direct_hints() # start the listener
hints = yield s.get_connection_hints() # start the listener
del hints
s.add_their_direct_hints([DIRECT_HINT, UNUSABLE_HINT])
s.add_their_relay_hints([])
s.add_connection_hints([DIRECT_HINT, UNUSABLE_HINT])
connectors = []
def _start_connector(ep, description, is_relay=False):
@ -1198,24 +1213,21 @@ class Transit(unittest.TestCase):
self.assertEqual(results, ["winner"])
def _endpoint_from_hint(self, hint):
if hint == DIRECT_HINT:
if hint == DIRECT_HINT_INTERNAL:
return "direct"
elif hint == RELAY_HINT:
elif hint == RELAY_HINT_FIRST:
return "relay"
elif hint == UNUSABLE_HINT:
return None
else:
return "ep"
return None
@inlineCallbacks
def test_wait_for_relay(self):
clock = task.Clock()
s = transit.TransitSender(u"", reactor=clock)
s = transit.TransitSender(u"", reactor=clock, no_listen=True)
s.set_transit_key(b"key")
hints = yield s.get_direct_hints() # start the listener
hints = yield s.get_connection_hints() # start the listener
del hints
s.add_their_direct_hints([DIRECT_HINT, UNUSABLE_HINT])
s.add_their_relay_hints([RELAY_HINT])
s.add_connection_hints([DIRECT_HINT, UNUSABLE_HINT, RELAY_HINT])
direct_connectors = []
relay_connectors = []
@ -1250,12 +1262,11 @@ class Transit(unittest.TestCase):
@inlineCallbacks
def test_no_direct_hints(self):
clock = task.Clock()
s = transit.TransitSender(u"", reactor=clock)
s = transit.TransitSender(u"", reactor=clock, no_listen=True)
s.set_transit_key(b"key")
hints = yield s.get_direct_hints() # start the listener
hints = yield s.get_connection_hints() # start the listener
del hints
s.add_their_direct_hints([UNUSABLE_HINT])
s.add_their_relay_hints([RELAY_HINT, UNUSABLE_HINT])
s.add_connection_hints([UNUSABLE_HINT, RELAY_HINT2])
direct_connectors = []
relay_connectors = []
@ -1301,14 +1312,11 @@ class Full(unittest.TestCase):
s.set_transit_key(KEY)
r.set_transit_key(KEY)
shints = yield s.get_direct_hints()
rhints = yield r.get_direct_hints()
shints = yield s.get_connection_hints()
rhints = yield r.get_connection_hints()
s.add_their_direct_hints(rhints)
r.add_their_direct_hints(shints)
s.add_their_relay_hints([])
r.add_their_relay_hints([])
s.add_connection_hints(rhints)
r.add_connection_hints(shints)
(x,y) = yield self.doBoth(s.connect(), r.connect())
self.assertIsInstance(x, transit.Connection)

138
src/wormhole/transit.py
View File

@ -1,7 +1,10 @@
from __future__ import print_function, absolute_import
import re, sys, time, socket, collections
import re, sys, time, socket
from collections import namedtuple, deque
from binascii import hexlify, unhexlify
import six
from zope.interface import implementer
from twisted.python import log
from twisted.python.runtime import platformType
from twisted.internet import (reactor, interfaces, defer, protocol,
endpoints, task, address, error)
@ -76,11 +79,26 @@ def build_relay_handshake(key):
token = HKDF(key, 32, CTXinfo=b"transit_relay_token")
return b"please relay "+hexlify(token)+b"\n"
# The hint format is: TYPE,VALUE= /^([a-zA-Z0-9]+):(.*)$/ . VALUE depends
# upon TYPE, and it can have more colons in it. For TYPE=tcp (the only one
# currently defined), ADDR,PORT = /^(.*):(\d+)$/ , so ADDR can have colons.
# ADDR can be a hostname, ipv4 dotted-quad, or ipv6 colon-hex. If the hint
# publisher wants anonymity, their only hint's ADDR will end in .onion .
# DirectTCPV1Hint and TorTCPV1Hint mean the following protocol:
# * make a TCP connection (possibly via Tor)
# * send the sender/receiver handshake bytes first
# * expect to see the receiver/sender handshake bytes from the other side
# * the sender writes "go\n", the receiver waits for "go\n"
# * the rest of the connection contains transit data
DirectTCPV1Hint = namedtuple("DirectTCPV1Hint", ["hostname", "port"])
TorTCPV1Hint = namedtuple("TorTCPV1Hint", ["hostname", "port"])
# RelayV1Hint contains a list of DirectTCPV1Hint and TorTCPV1Hint hints. For
# each one, make the TCP connection, send the relay handshake, then complete
# the rest of the V1 protocol. Only one hint per relay is useful.
RelayV1Hint = namedtuple("RelayV1Hint", ["hints"])
def describe_hint(hint):
if isinstance(hint, DirectTCPV1Hint):
return u"tcp:%s:%d" % (hint.hostname, hint.port)
elif isinstance(hint, TorTCPV1Hint):
return u"tor:%s:%d" % (hint.hostname, hint.port)
else:
return str(hint)
def parse_hint_tcp(hint):
assert isinstance(hint, type(u""))
@ -104,7 +122,7 @@ def parse_hint_tcp(hint):
except ValueError:
print("non-numeric port in TCP hint '%s'" % (hint,))
return None
return hint_host, hint_port
return DirectTCPV1Hint(hint_host, hint_port)
TIMEOUT=15
@ -123,8 +141,8 @@ class Connection(protocol.Protocol, policies.TimeoutMixin):
self._consumer_bytes_written = 0
self._consumer_bytes_expected = None
self._consumer_deferred = None
self._inbound_records = collections.deque()
self._waiting_reads = collections.deque()
self._inbound_records = deque()
self._waiting_reads = deque()
def connectionMade(self):
debug("handle %r" % (self.transport,))
@ -555,9 +573,12 @@ class Common:
if transit_relay:
if not isinstance(transit_relay, type(u"")):
raise UsageError
self._transit_relays = [transit_relay]
relay = RelayV1Hint(hints=[parse_hint_tcp(transit_relay)])
self._transit_relays = [relay]
else:
self._transit_relays = []
self._their_direct_hints = []
self._their_relay_hints = []
self._tor_manager = tor_manager
self._transit_key = None
self._no_listen = no_listen
@ -572,12 +593,30 @@ class Common:
if self._no_listen or self._tor_manager:
return ([], None)
portnum = allocate_tcp_port()
direct_hints = [u"tcp:%s:%d" % (addr, portnum)
direct_hints = [DirectTCPV1Hint(six.u(addr), portnum)
for addr in ipaddrs.find_addresses()]
ep = endpoints.serverFromString(reactor, "tcp:%d" % portnum)
return direct_hints, ep
def get_direct_hints(self):
@inlineCallbacks
def get_connection_hints(self):
hints = []
direct_hints = yield self._get_direct_hints()
for dh in direct_hints:
hints.append({u"type": u"direct-tcp-v1",
u"hostname": dh.hostname,
u"port": dh.port, # integer
})
for relay in self._transit_relays:
rhint = {u"type": u"relay-v1", u"hints": []}
for rh in relay.hints:
rhint[u"hints"].append({u"type": u"direct-tcp-v1",
u"hostname": rh.hostname,
u"port": rh.port})
hints.append(rhint)
returnValue(hints)
def _get_direct_hints(self):
if self._listener:
return defer.succeed(self._my_direct_hints)
# there is a slight race here: if someone calls get_direct_hints() a
@ -619,21 +658,41 @@ class Common:
self._listener_d.addErrback(lambda f: None)
self._listener_d.cancel()
def get_relay_hints(self):
return self._transit_relays
def _parse_tcp_v1_hint(self, hint):
hint_type = hint.get(u"type", u"")
if hint_type not in [u"direct-tcp-v1", u"tor-tcp-v1"]:
log.msg("unknown hint type: %r" % (hint,))
return None
if not(u"hostname" in hint
and isinstance(hint[u"hostname"], type(u""))):
log.msg("invalid hostname in hint: %r" % (hint,))
return None
if not(u"port" in hint and isinstance(hint[u"port"], int)):
log.msg("invalid port in hint: %r" % (hint,))
return None
if hint_type == u"direct-tcp-v1":
return DirectTCPV1Hint(hint[u"hostname"], hint[u"port"])
else:
return TorTCPV1Hint(hint[u"hostname"], hint[u"port"])
def add_their_direct_hints(self, hints):
def add_connection_hints(self, hints):
for h in hints:
if not isinstance(h, type(u"")):
raise TypeError("hint '%r' should be unicode, not %s"
% (h, type(h)))
self._their_direct_hints = set(hints)
def add_their_relay_hints(self, hints):
for h in hints:
if not isinstance(h, type(u"")):
raise TypeError("hint '%r' should be unicode, not %s"
% (h, type(h)))
self._their_relay_hints = set(hints)
hint_type = h.get(u"type", u"")
if hint_type in [u"direct-tcp-v1", u"tor-tcp-v1"]:
dh = self._parse_tcp_v1_hint(h)
if dh:
self._their_direct_hints.append(dh)
elif hint_type == u"relay-v1":
# TODO: each relay-v1 clause describes a different relay,
# with a set of equally-valid ways to connect to it. Treat
# them as separate relays, instead of merging them all
# together like this.
for rhs in h.get(u"hints", []):
rh = self._parse_tcp_v1_hint(rhs)
if rh:
self._their_relay_hints.append(rh)
else:
log.msg("unknown hint type: %r" % (h,))
def _send_this(self):
assert self._transit_key
@ -717,7 +776,7 @@ class Common:
ep = self._endpoint_from_hint(hint)
if not ep:
continue
description = "->%s" % (hint,)
description = "->%s" % describe_hint(hint)
d = self._start_connector(ep, description)
contenders.append(d)
relay_delay = self.RELAY_DELAY
@ -732,7 +791,7 @@ class Common:
ep = self._endpoint_from_hint(hint)
if not ep:
continue
description = "->relay:%s" % (hint,)
description = "->relay:%s" % describe_hint(hint)
d = task.deferLater(self._reactor, relay_delay,
self._start_connector, ep, description,
is_relay=True)
@ -764,22 +823,17 @@ class Common:
return d
def _endpoint_from_hint(self, hint):
# TODO: use parse_hint_tcp
if ":" not in hint:
return None
pieces = hint.split(":")
hint_type = hint.split(":")[0]
if hint_type == "tor" and self._tor_manager:
return self._tor_manager.get_endpoint_for(pieces[1], int(pieces[2]))
if hint_type != "tcp":
return None
pieces = hint.split(":")
if self._tor_manager:
# our TorManager will return None for non-public IPv4 addresses
# and any IPv6 address
return self._tor_manager.get_endpoint_for(pieces[1], int(pieces[2]))
return endpoints.HostnameEndpoint(self._reactor, pieces[1],
int(pieces[2]))
if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)):
# our TorManager will return None for non-public IPv4
# addresses and any IPv6 address
return self._tor_manager.get_endpoint_for(hint.hostname,
hint.port)
return None
if isinstance(hint, DirectTCPV1Hint):
return endpoints.HostnameEndpoint(self._reactor,
hint.hostname, hint.port)
return None
def connection_ready(self, p):
# inbound/outbound Connection protocols call this when they finish