From 1ab66d2fd0b33d18cc8a38298b0124917f33fa73 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 3 Dec 2015 21:15:19 -0800 Subject: [PATCH 1/2] privacy: only store coarse timestamps in the usage table --- src/wormhole/scripts/runner.py | 6 ++++++ src/wormhole/servers/cmd_server.py | 2 +- src/wormhole/servers/relay_server.py | 15 +++++++++++---- src/wormhole/servers/server.py | 15 ++++++++++++--- src/wormhole/servers/transit_server.py | 5 ++++- src/wormhole/test/test_server.py | 2 +- 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/wormhole/scripts/runner.py b/src/wormhole/scripts/runner.py index e4497b9..6d2c856 100644 --- a/src/wormhole/scripts/runner.py +++ b/src/wormhole/scripts/runner.py @@ -41,6 +41,9 @@ sp_start.add_argument("--transit", default="tcp:3001", metavar="tcp:PORT", help="endpoint specification for the transit-relay port") sp_start.add_argument("--advertise-version", metavar="VERSION", help="version to recommend to clients") +sp_start.add_argument("--blur-usage", default=None, type=int, + metavar="SECONDS", + help="round logged access times to improve privacy") sp_start.add_argument("-n", "--no-daemon", action="store_true") #sp_start.add_argument("twistd_args", nargs="*", default=None, # metavar="[TWISTD-ARGS..]", @@ -61,6 +64,9 @@ sp_restart.add_argument("--transit", default="tcp:3001", metavar="tcp:PORT", help="endpoint specification for the transit-relay port") sp_restart.add_argument("--advertise-version", metavar="VERSION", help="version to recommend to clients") +sp_restart.add_argument("--blur-usage", default=None, type=int, + metavar="SECONDS", + help="round logged access times to improve privacy") sp_restart.add_argument("-n", "--no-daemon", action="store_true") sp_restart.set_defaults(func=cmd_server.restart_server) diff --git a/src/wormhole/servers/cmd_server.py b/src/wormhole/servers/cmd_server.py index 0110cb1..a42267b 100644 --- a/src/wormhole/servers/cmd_server.py +++ b/src/wormhole/servers/cmd_server.py @@ -11,7 +11,7 @@ class MyPlugin: from .server import RelayServer return RelayServer(self.args.rendezvous, self.args.transit, self.args.advertise_version, - "relay.sqlite") + "relay.sqlite", self.args.blur_usage) def start_server(args): from twisted.python import usage diff --git a/src/wormhole/servers/relay_server.py b/src/wormhole/servers/relay_server.py index c520466..55fbfd6 100644 --- a/src/wormhole/servers/relay_server.py +++ b/src/wormhole/servers/relay_server.py @@ -207,10 +207,11 @@ class Deallocator(RelayResource): class Channel: - def __init__(self, app, db, welcome, appid, channelid): + def __init__(self, app, db, welcome, blur_usage, appid, channelid): self._app = app self._db = db self._welcome = welcome + self._blur_usage = blur_usage self._appid = appid self._channelid = channelid self._listeners = set() # callbacks that take JSONable object @@ -297,6 +298,8 @@ class Channel: def _store_summary(self, summary): (started, result, total_time, waiting_time) = summary + if self._blur_usage: + started = self._blur_usage * (started // self._blur_usage) self._db.execute("INSERT INTO `usage`" " (`type`, `started`, `result`," " `total_time`, `waiting_time`)" @@ -382,9 +385,10 @@ class Channel: class AppNamespace: - def __init__(self, db, welcome, appid): + def __init__(self, db, welcome, blur_usage, appid): self._db = db self._welcome = welcome + self._blur_usage = blur_usage self._appid = appid self._channels = {} @@ -420,6 +424,7 @@ class AppNamespace: if not channelid in self._channels: log.msg("spawning #%d for appid %s" % (channelid, self._appid)) self._channels[channelid] = Channel(self, self._db, self._welcome, + self._blur_usage, self._appid, channelid) return self._channels[channelid] @@ -448,11 +453,12 @@ class AppNamespace: return bool(self._channels) class Relay(resource.Resource, service.MultiService): - def __init__(self, db, welcome): + def __init__(self, db, welcome, blur_usage): resource.Resource.__init__(self) service.MultiService.__init__(self) self._db = db self._welcome = welcome + self._blur_usage = blur_usage self._apps = {} t = internet.TimerService(EXPIRATION_CHECK_PERIOD, self.prune) t.setServiceParent(self) @@ -476,7 +482,8 @@ class Relay(resource.Resource, service.MultiService): assert isinstance(appid, type(u"")) if not appid in self._apps: log.msg("spawning appid %s" % (appid,)) - self._apps[appid] = AppNamespace(self._db, self._welcome, appid) + self._apps[appid] = AppNamespace(self._db, self._welcome, + self._blur_usage, appid) return self._apps[appid] def prune(self): diff --git a/src/wormhole/servers/server.py b/src/wormhole/servers/server.py index 8e4118a..cf5ffbd 100644 --- a/src/wormhole/servers/server.py +++ b/src/wormhole/servers/server.py @@ -1,4 +1,5 @@ from __future__ import print_function +from twisted.python import log from twisted.internet import reactor, endpoints from twisted.application import service from twisted.web import server, static, resource @@ -16,8 +17,9 @@ class Root(resource.Resource): class RelayServer(service.MultiService): def __init__(self, relayport, transitport, advertise_version, - db_url=":memory:"): + db_url=":memory:", blur_usage=None): service.MultiService.__init__(self) + self._blur_usage = blur_usage self.db = get_db(db_url) welcome = { "current_version": __version__, @@ -35,11 +37,18 @@ class RelayServer(service.MultiService): r = endpoints.serverFromString(reactor, relayport) self.relayport_service = ServerEndpointService(r, site) self.relayport_service.setServiceParent(self) - self.relay = Relay(self.db, welcome) # accessible from tests + self.relay = Relay(self.db, welcome, blur_usage) # accessible from tests self.root.putChild(b"wormhole-relay", self.relay) if transitport: - self.transit = Transit(self.db) + self.transit = Transit(self.db, blur_usage) self.transit.setServiceParent(self) # for the timer t = endpoints.serverFromString(reactor, transitport) self.transport_service = ServerEndpointService(t, self.transit) self.transport_service.setServiceParent(self) + def startService(self): + service.MultiService.startService(self) + log.msg("Wormhole relay server (Rendezvous and Transit) running") + if self._blur_usage: + log.msg("blurring access times to %d seconds" % self._blur_usage) + else: + log.msg("not blurring access times") diff --git a/src/wormhole/servers/transit_server.py b/src/wormhole/servers/transit_server.py index be89da7..da54e69 100644 --- a/src/wormhole/servers/transit_server.py +++ b/src/wormhole/servers/transit_server.py @@ -148,9 +148,10 @@ class Transit(protocol.ServerFactory, service.MultiService): MAXTIME = 60*SECONDS protocol = TransitConnection - def __init__(self, db): + def __init__(self, db, blur_usage): service.MultiService.__init__(self) self._db = db + self._blur_usage = blur_usage self._pending_requests = {} # token -> TransitConnection self._active_connections = set() # TransitConnection @@ -170,6 +171,8 @@ class Transit(protocol.ServerFactory, service.MultiService): def recordUsage(self, started, result, total_bytes, total_time, waiting_time): log.msg("Transit.recordUsage (%dB)" % total_bytes) + if self._blur_usage: + started = self._blur_usage * (started // self._blur_usage) self._db.execute("INSERT INTO `usage`" " (`type`, `started`, `result`, `total_bytes`," " `total_time`, `waiting_time`)" diff --git a/src/wormhole/test/test_server.py b/src/wormhole/test/test_server.py index 989de29..bfa54ff 100644 --- a/src/wormhole/test/test_server.py +++ b/src/wormhole/test/test_server.py @@ -363,7 +363,7 @@ class OneEventAtATime: class Summary(unittest.TestCase): def test_summarize(self): - c = relay_server.Channel(None, None, None, None, None) + c = relay_server.Channel(None, None, None, None, None, None) A = relay_server.ALLOCATE D = relay_server.DEALLOCATE From 9ed39be346b92a3865f0ab1a4e26f8d628f6f54a Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 4 Dec 2015 17:35:56 -0800 Subject: [PATCH 2/2] don't log HTTP requests when blur-usage is on --- src/wormhole/servers/relay_server.py | 50 +++++++++++++++++++--------- src/wormhole/servers/server.py | 12 ++++++- src/wormhole/test/test_server.py | 2 +- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/wormhole/servers/relay_server.py b/src/wormhole/servers/relay_server.py index 55fbfd6..b0a2681 100644 --- a/src/wormhole/servers/relay_server.py +++ b/src/wormhole/servers/relay_server.py @@ -63,10 +63,11 @@ class EventsProtocol: # all JSON responses include a "welcome:{..}" key class RelayResource(resource.Resource): - def __init__(self, relay, welcome): + def __init__(self, relay, welcome, log_requests): resource.Resource.__init__(self) self._relay = relay self._welcome = welcome + self._log_requests = log_requests class ChannelLister(RelayResource): def render_GET(self, request): @@ -93,8 +94,9 @@ class Allocator(RelayResource): app = self._relay.get_app(appid) channelid = app.find_available_channelid() app.allocate_channel(channelid, side) - log.msg("allocated #%d, now have %d DB channels" % - (channelid, len(app.get_allocated()))) + if self._log_requests: + log.msg("allocated #%d, now have %d DB channels" % + (channelid, len(app.get_allocated()))) request.setHeader(b"content-type", b"application/json; charset=utf-8") data = {"welcome": self._welcome, "channelid": channelid} return (json.dumps(data)+"\n").encode("utf-8") @@ -207,11 +209,13 @@ class Deallocator(RelayResource): class Channel: - def __init__(self, app, db, welcome, blur_usage, appid, channelid): + def __init__(self, app, db, welcome, blur_usage, log_requests, + appid, channelid): self._app = app self._db = db self._welcome = welcome self._blur_usage = blur_usage + self._log_requests = log_requests self._appid = appid self._channelid = channelid self._listeners = set() # callbacks that take JSONable object @@ -385,10 +389,11 @@ class Channel: class AppNamespace: - def __init__(self, db, welcome, blur_usage, appid): + def __init__(self, db, welcome, blur_usage, log_requests, appid): self._db = db self._welcome = welcome self._blur_usage = blur_usage + self._log_requests = log_requests self._appid = appid self._channels = {} @@ -422,9 +427,11 @@ class AppNamespace: def get_channel(self, channelid): assert isinstance(channelid, int) if not channelid in self._channels: - log.msg("spawning #%d for appid %s" % (channelid, self._appid)) + if self._log_requests: + log.msg("spawning #%d for appid %s" % (channelid, self._appid)) self._channels[channelid] = Channel(self, self._db, self._welcome, self._blur_usage, + self._log_requests, self._appid, channelid) return self._channels[channelid] @@ -434,10 +441,16 @@ class AppNamespace: if channelid in self._channels: self._channels.pop(channelid) - log.msg("freed+killed #%d, now have %d DB channels, %d live" % - (channelid, len(self.get_allocated()), len(self._channels))) + if self._log_requests: + log.msg("freed+killed #%d, now have %d DB channels, %d live" % + (channelid, len(self.get_allocated()), len(self._channels))) def prune_old_channels(self): + # For now, pruning is logged even if log_requests is False, to debug + # the pruning process, and since pruning is triggered by a timer + # instead of by user action. It does reveal which channels were + # present when the pruning process began, though, so in the log run + # it should do less logging. log.msg(" channel prune begins") # a channel is deleted when there are no listeners and there have # been no messages added in CHANNEL_EXPIRATION_TIME seconds @@ -459,15 +472,17 @@ class Relay(resource.Resource, service.MultiService): self._db = db self._welcome = welcome self._blur_usage = blur_usage + log_requests = blur_usage is None + self._log_requests = log_requests self._apps = {} t = internet.TimerService(EXPIRATION_CHECK_PERIOD, self.prune) t.setServiceParent(self) - self.putChild(b"list", ChannelLister(self, welcome)) - self.putChild(b"allocate", Allocator(self, welcome)) - self.putChild(b"add", Adder(self, welcome)) - self.putChild(b"get", GetterOrWatcher(self, welcome)) - self.putChild(b"watch", Watcher(self, welcome)) - self.putChild(b"deallocate", Deallocator(self, welcome)) + self.putChild(b"list", ChannelLister(self, welcome, log_requests)) + self.putChild(b"allocate", Allocator(self, welcome, log_requests)) + self.putChild(b"add", Adder(self, welcome, log_requests)) + self.putChild(b"get", GetterOrWatcher(self, welcome, log_requests)) + self.putChild(b"watch", Watcher(self, welcome, log_requests)) + self.putChild(b"deallocate", Deallocator(self, welcome, log_requests)) def getChild(self, path, req): # 0.4.0 used "POST /CID/SIDE/post/MSGNUM" @@ -481,12 +496,15 @@ class Relay(resource.Resource, service.MultiService): def get_app(self, appid): assert isinstance(appid, type(u"")) if not appid in self._apps: - log.msg("spawning appid %s" % (appid,)) + if self._log_requests: + log.msg("spawning appid %s" % (appid,)) self._apps[appid] = AppNamespace(self._db, self._welcome, - self._blur_usage, appid) + self._blur_usage, + self._log_requests, appid) return self._apps[appid] def prune(self): + # As with AppNamespace.prune_old_channels, we log for now. log.msg("beginning app prune") c = self._db.execute("SELECT DISTINCT `appid` FROM `messages`") apps = set([row["appid"] for row in c.fetchall()]) # these have messages diff --git a/src/wormhole/servers/server.py b/src/wormhole/servers/server.py index cf5ffbd..111583d 100644 --- a/src/wormhole/servers/server.py +++ b/src/wormhole/servers/server.py @@ -15,6 +15,12 @@ class Root(resource.Resource): resource.Resource.__init__(self) self.putChild(b"", static.Data(b"Wormhole Relay\n", "text/plain")) +class PrivacyEnhancedSite(server.Site): + logRequests = True + def log(self, request): + if self.logRequests: + return server.Site.log(self, request) + class RelayServer(service.MultiService): def __init__(self, relayport, transitport, advertise_version, db_url=":memory:", blur_usage=None): @@ -33,7 +39,9 @@ class RelayServer(service.MultiService): if advertise_version: welcome["current_version"] = advertise_version self.root = Root() - site = server.Site(self.root) + site = PrivacyEnhancedSite(self.root) + if blur_usage: + site.logRequests = False r = endpoints.serverFromString(reactor, relayport) self.relayport_service = ServerEndpointService(r, site) self.relayport_service.setServiceParent(self) @@ -45,10 +53,12 @@ class RelayServer(service.MultiService): t = endpoints.serverFromString(reactor, transitport) self.transport_service = ServerEndpointService(t, self.transit) self.transport_service.setServiceParent(self) + def startService(self): service.MultiService.startService(self) log.msg("Wormhole relay server (Rendezvous and Transit) running") if self._blur_usage: log.msg("blurring access times to %d seconds" % self._blur_usage) + log.msg("not logging HTTP requests") else: log.msg("not blurring access times") diff --git a/src/wormhole/test/test_server.py b/src/wormhole/test/test_server.py index bfa54ff..3b8be7e 100644 --- a/src/wormhole/test/test_server.py +++ b/src/wormhole/test/test_server.py @@ -363,7 +363,7 @@ class OneEventAtATime: class Summary(unittest.TestCase): def test_summarize(self): - c = relay_server.Channel(None, None, None, None, None, None) + c = relay_server.Channel(None, None, None, None, False, None, None) A = relay_server.ALLOCATE D = relay_server.DEALLOCATE