diff --git a/misc/dump-timing.py b/misc/dump-timing.py
index 6a222a8..c36e44d 100644
--- a/misc/dump-timing.py
+++ b/misc/dump-timing.py
@@ -1,78 +1,122 @@
# To use the web() option, you should do:
# * cd misc
-# * npm install vis zepto
+# * npm install d3 zepto
from __future__ import print_function
-import os, sys, time, json
+import os, sys, time, json, random
streams = sys.argv[1:]
+if not streams:
+ print("run like: python dump-timing.py tx.json rx.json")
+ sys.exit(1)
num_streams = len(streams)
labels = dict([(num, " "*num + "[%d]" % (num+1) + " "*(num_streams-1-num))
for num in range(num_streams)])
-timeline = []
-groups_out = []
-for which,fn in enumerate(streams):
+abs_timeline = []
+sides = []
+
+for side,fn in enumerate(streams):
with open(fn, "rb") as f:
- for (start, sent, finish, what, start_d, finish_d) in json.load(f):
- timeline.append( (start, sent, finish, which, what,
- start_d, finish_d) )
- print("%s is %s" % (labels[which], fn))
- groups_out.append({"id": which, "content": fn,
- "className": "group-%d" % which})
+ for (start, sent, finish, what, details) in json.load(f):
+ abs_timeline.append( (start, sent, finish, side, what, details) )
+ print("%s is %s" % (labels[side], fn))
+ sides.append(os.path.basename(fn))
+# relativize all timestamps
+all_times = [e[0] for e in abs_timeline] + [e[2] for e in abs_timeline if e[2]]
+all_times.sort()
+earliest = all_times[0]
+def rel(t):
+ if t is None: return None
+ return t - earliest
+timeline = [ (rel(start), rel(sent), rel(finish), side, what, details)
+ for (start, sent, finish, side, what, details)
+ in abs_timeline ]
+data = {}
-timeline.sort(key=lambda row: row[0])
-first = timeline[0][0]
-print("started at %s" % time.ctime(start))
-viz_out = []
-for num, (start, sent, finish, which, what,
- start_d, finish_d) in enumerate(timeline):
- delta = start - first
- delta_s = "%.6f" % delta
- start_d_str = ", ".join(["%s=%s" % (name, start_d[name])
- for name in sorted(start_d)])
- finish_d_str = ", ".join(["%s=%s" % (name, finish_d[name])
- for name in sorted(finish_d)])
- details_str = start_d_str
- if finish_d_str:
- details_str += "/" + finish_d_str
- finish_str = ""
- if finish is not None:
- finish_str = " +%.6f" % (finish - start)
- print("%9s: %s %s %s%s" % (delta_s, labels[which], what, details_str,
- finish_str))
- viz_start = start*1000
- viz_end = None if finish is None else finish*1000
- viz_type = "range" if finish else "point"
- if what == "wormhole started" or what == "wormhole":
- viz_type = "background"
- viz_content = '%s' % (details_str or "(No details)",
- what) # sigh
- viz_className = "item-group-%d" % which
- if "waiting" in start_d:
- viz_className += " wait-%s" % start_d["waiting"]
- viz_out.append({"id": num, "start": viz_start, "end": viz_end,
- "group": which, #"subgroup": num,
- "content": viz_content,
- "className": viz_className, # or style:
- "type": viz_type,
- })
- if sent is not None:
- viz_out.append({"id": "%d.sent" % num, "start": sent*1000,
- "group": which, #"subgroup": num,
- "content": "sent",
- "className": viz_className,
- "type": "point"})
+# we pre-calculate the "lane" that each item uses, here in python, rather
+# than leaving that up to the javascript.
+data["lanes"] = ["proc", # 0 gets high-level events and spans: process start,
+ # imports, command dispatch, code established, key
+ # established, transit connected, process exit
+ "API", # 1 gets API call spans (apps usually only wait for
+ # one at a time, so they won't overlap): get_code,
+ # input_code, get_verifier, get_data, send_data,
+ # close
+ "wait", # 2 shows waiting-for-human: input code, get
+ # permission
+ "app", # 3: file-xfer events
+ "skt", # 4: websocket message send/receives
+ "misc", # 5: anything else
+ ]
+data["bounds"] = {"min": rel(all_times[0]), "max": rel(all_times[-1]),
+ }
+data["sides"] = sides
+print("started at %s" % time.ctime(earliest))
+print("duration %s seconds" % data["bounds"]["max"])
+items = data["items"] = []
+for num, (start, sent, finish, side, what, details) in enumerate(timeline):
+ background = False
+ if what in ["wormhole",]:
+ # background region for wormhole lifetime
+ lane = 0
+ background = True
+ elif what in ["process start", "import", "command dispatch",
+ "code established", "key established", "transit connected",
+ "exit"]:
+ lane = 0
+ elif what in ["API get_code", "API input_code", "API set_code",
+ "API get_verifier", "API get_data", "API send_data",
+ #"API get data", "API send data",
+ "API close"]:
+ lane = 1
+ elif details.get("waiting") in ["user", "crypto"]: # permission or math
+ lane = 2
+ elif what in ["tx file", "get ack", "rx file", "unpack zip", "send ack"]:
+ lane = 3
+ elif what in ["websocket"]:
+ # connection establishment
+ lane = 4
+ background = True
+ elif (what in ["welcome", "error"] # rendezvous message receives
+ or what in ["allocate", "list", "get", "add", "deallocate"]
+ # rendezvous message sends
+ ):
+ lane = 4
+ else:
+ lane = 5 # unknown
+
+ if background:
+ continue # disable until I figure out how to draw these better
+
+ details_str = ", ".join(["%s=%s" % (name, details[name])
+ for name in sorted(details)])
+
+ items.append({"side": side,
+ "lane": lane,
+ "start_time": start,
+ "server_sent": sent,
+ "finish_time": finish, # maybe None
+ "what": what,
+ "details": details,
+ "details_str": details_str,
+ "wiggle": random.randint(0,4),
+ })
+
+ #if "waiting" in details:
+ # viz_className += " wait-%s" % details["waiting"]
+
+from pprint import pprint
+pprint(data)
here = os.path.dirname(__file__)
web_root = os.path.join(here, "web")
-vis_root = os.path.join(here, "node_modules", "vis", "dist")
-zepto_root = os.path.join(here, "node_modules", "zepto")
-if not os.path.isdir(vis_root) or not os.path.isdir(zepto_root):
- print("Cannot find 'vis' and 'zepto' in misc/node_modules/")
- print("Please run 'npm install vis zepto' from the misc/ directory.")
+lib_root = os.path.join(here, "node_modules")
+if not os.path.isdir(lib_root):
+ print("Cannot find 'd3' and 'd3-tip' in misc/node_modules/")
+ print("Please run 'npm install d3 d3-tip zepto' from the misc/ directory.")
sys.exit(1)
def web():
@@ -82,15 +126,14 @@ def web():
from twisted.internet import reactor, endpoints
ep = endpoints.serverFromString(reactor, "tcp:8066:interface=127.0.0.1")
root = static.File(web_root)
- data_json = {"items": viz_out, "groups": groups_out}
- data = json.dumps(data_json).encode("utf-8")
- root.putChild("data.json", static.Data(data, "application/json"))
- root.putChild("vis", static.File(vis_root))
- root.putChild("zepto", static.File(zepto_root))
+ root.putChild("data.json", static.Data(json.dumps(data).encode("utf-8"),
+ "application/json"))
+ root.putChild("lib", static.File(lib_root))
class Shutdown(resource.Resource):
def render_GET(self, request):
- print("timeline ready, server shutting down")
- reactor.stop()
+ if 0:
+ print("timeline ready, server shutting down")
+ reactor.stop()
return "shutting down"
root.putChild("done", Shutdown())
site = server.Site(root)
diff --git a/misc/web/timeline.css b/misc/web/timeline.css
index c49565d..ea90e15 100644
--- a/misc/web/timeline.css
+++ b/misc/web/timeline.css
@@ -1,20 +1,51 @@
-div.item-group-0 {
- background-color: #fcc;
+rect.bar {
+ stroke: black;
}
-div.item-group-1 {
- background-color: #cfc;
+.lane-0 {
+ fill: #fcc;
}
-div.item-group-2 {
- background-color: #ccf;
+.lane-1 {
+ fill: #cfc;
}
-div.wait-user {
- background-color: #ccc;
+.lane-2 {
+ fill: #ccf;
+}
+
+.lane-3 {
+ fill: #ccf;
+}
+
+.lane-4 {
+ fill: #ccf;
+}
+
+.lane-5 {
+ fill: #ccf;
+}
+
+.lane-6 {
+ fill: #ccf;
+}
+
+rect.wait-user {
+ fill: #ccc;
+}
+
+rect.wait-crypto {
+ fill: #bbe;
}
.vis-item .vis-item-overflow {
overflow: visible;
}
+
+.d3-tip {
+ margin: 4px;
+ padding: 2px;
+ background: #111;
+ color: #fff;
+}
diff --git a/misc/web/timeline.html b/misc/web/timeline.html
index 190fdd9..41e92fe 100644
--- a/misc/web/timeline.html
+++ b/misc/web/timeline.html
@@ -2,9 +2,9 @@
Timeline Visualizer
-
-
-
+
+
+
diff --git a/misc/web/timeline.js b/misc/web/timeline.js
index 1554f29..0c9978f 100644
--- a/misc/web/timeline.js
+++ b/misc/web/timeline.js
@@ -1,59 +1,356 @@
-var vis, $; // hush
+var d3, $; // hush
-var container = document.getElementById("viz");
-var options = {editable: false,
- showCurrentTime: false,
- snap: null,
- order: function(a,b) { return a.id - b.id; }
- };
-var timeline = new vis.Timeline(container, options);
+var container = d3.select("#viz");
+var data;
var items;
+var globals = {};
-$.getJSON("data.json", function(data) {
- items = new vis.DataSet(data.items);
- timeline.setData({groups: new vis.DataSet(data.groups),
- items: items});
- var start = data.items[0].start;
- var end = data.items[data.items.length-1].start;
- var span = end - start;
- timeline.setWindow(start - (span/10), end + (span/10));
- //timeline.fit(); // doesn't leave space on the ends
- timeline.setOptions({min: start - (span/10),
- max: end + (span/10),
- zoomMin: 50,
- zoomMax: 1.2*span});
- var bar = timeline.addCustomTime(start, "cursor");
- timeline.on("timechange", update_cursor);
- update_cursor({time: new Date(start)});
- timeline.on("doubleClick", zoom);
- timeline.on("select", select_item);
+var zoom = d3.behavior.zoom().scaleExtent([1, Infinity]);
+function zoomin() {
+ //var w = Number(container.style("width").slice(0,-2));
+ //console.log("zoomin", w);
+ //zoom.center([w/2, 20]); // doesn't work yet
+ zoom.scale(zoom.scale() * 2);
+ globals.redraw();
+}
+function zoomout() {
+ zoom.scale(zoom.scale() * 0.5);
+ globals.redraw();
+}
+
+$.getJSON("data.json", function(d) {
+ data = d;
+
+ const LANE_HEIGHT = 30;
+ const RECT_HEIGHT = 20;
+
+ // the Y axis is as follows:
+ // * each lane is LANE_HEIGHT tall (e.g. 30px)
+ // * the actual rects are RECT_HEIGHT tall (e.g. 20px)
+ // * there is a one-lane-tall gap at the top of the chart
+ // * there are data.sides.length sides (e.g. one tx, one rx)
+ // * there are data.lanes.length lanes (e.g. 6), each with a name
+ // * there is a one-lane-tall gap between each side
+ // * there is a one-lane-tall gap after the last lane
+ // * the horizontal scale markers begin after that gap
+ // * the tick marks extend another 6 pixels down
+
+ var w = Number(container.style("width").slice(0,-2));
+
+ function y_off(d) {
+ return (LANE_HEIGHT * (d.side*(data.lanes.length+1) + d.lane)
+ + d.wiggle);
+ }
+ var bottom_rule_y = LANE_HEIGHT * data.sides.length * (data.lanes.length+1);
+ var bottom_y = bottom_rule_y + 45;
+
+ var tip = d3.tip()
+ .attr("class", "d3-tip")
+ .html(function(d) { return "" + d.details_str + ""; })
+ .direction("s")
+ ;
+
+ var chart = container.append("svg:svg")
+ .attr("id", "outer_chart")
+ .attr("width", w)
+ .attr("pointer-events", "all")
+ .call(zoom)
+ .call(tip)
+ ;
+ //var chart_g = chart.append("svg:g");
+
+ // this "backboard" rect lets us catch mouse events anywhere in the
+ // chart, even between the bars. Without it, we only see events on solid
+ // objects like bars and text, but not in the gaps between.
+ chart.append("svg:rect")
+ .attr("id", "outer_rect")
+ .attr("width", w).attr("height", bottom_y).attr("fill", "none");
+
+ // but the stuff we put inside it should have some room
+ w = w-50;
+
+ chart.selectAll("text.sides-label").data(data.sides).enter()
+ .append("svg:text")
+ .attr("class", "sides-label")
+ .attr("x", "0px")
+ .attr("y", function(d,idx) {
+ return y_off({side: idx, lane: data.lanes.length/2,
+ wiggle: 0}) ;})
+ .attr("text-anchor", "start") // anchor at top-left
+ .attr("dy", ".71em")
+ .attr("fill", "black")
+ .text(function(d) { return d; })
+ ;
+
+ var lanes_by_sides = [];
+ data.sides.forEach(function(side, side_index) {
+ data.lanes.forEach(function(lane, lane_index) {
+ lanes_by_sides.push({side: side, side_index: side_index,
+ lane: lane, lane_index: lane_index});
+ });
+ });
+
+ chart.selectAll("text.lanes-label").data(lanes_by_sides).enter()
+ .append("svg:text")
+ .attr("class", "lanes-label")
+ .attr("x", "50px")
+ .attr("y", function(d) {
+ return y_off({side: d.side_index, lane: d.lane_index,
+ wiggle: 0}) ;})
+ .attr("text-anchor", "start") // anchor at top-left
+ .attr("dy", ".91em")
+ .attr("fill", "#f88")
+ .text(function(d) { return d.lane; })
+ ;
+
+ chart.append("svg:text")
+ .attr("class", "seconds-label")
+ //.attr("x", w/2).attr("y", y + 35)
+ .attr("text-anchor", "middle")
+ .attr("fill", "black")
+ .text("seconds");
+
+ d3.select("#outer_chart").attr("height", bottom_y);
+ d3.select("#outer_rect").attr("height", bottom_y);
+ d3.select("#zoom").attr("transform", "translate("+(w-10)+","+10+")");
+
+ function reltime(t) {return t-data.bounds.min;}
+ var last = data.bounds.max - data.bounds.min;
+ //last = reltime(d3.max(data.dyhb, function(d){return d.finish_time;}));
+ last = last * 1.05;
+ // long downloads are likely to have too much info, start small
+ if (last > 10.0)
+ last = 10.0;
+ // d3.time.scale() has no support for ms or us.
+ var xOFF = d3.time.scale().domain([data.bounds.min, data.bounds.max])
+ .range([0,w]);
+ var x = d3.scale.linear().domain([-last*0.05, last])
+ .range([0,w]);
+ zoom.x(x);
+ function tx(d) { return "translate(" +x(d) + ",0)"; }
+ function left(d) { return x(reltime(d.start_time)); }
+ function left_server(d) { return x(reltime(d.server_sent)); }
+ function right(d) {
+ return d.finish_time ? x(reltime(d.finish_time)) : "1px";
+ }
+ function width(d) {
+ return d.finish_time ? x(reltime(d.finish_time))-x(reltime(d.start_time)) : "1px";
+ }
+ function halfwidth(d) {
+ if (d.finish_time)
+ return (x(reltime(d.finish_time))-x(reltime(d.start_time)))/2;
+ return "1px";
+ }
+ function middle(d) {
+ if (d.finish_time)
+ return (x(reltime(d.start_time))+x(reltime(d.finish_time)))/2;
+ else
+ return x(reltime(d.start_time)) + 1;
+ }
+ function color(d) { return data.server_info[d.serverid].color; }
+ function servername(d) { return data.server_info[d.serverid].short; }
+ function timeformat(duration) {
+ // TODO: trim to microseconds, maybe humanize
+ return duration;
+ }
+
+ function redraw() {
+ // at this point zoom/pan must be fixed
+ var min = data.bounds.min + x.domain()[0];
+ var max = data.bounds.min + x.domain()[1];
+ function inside(d) {
+ var finish_time = d.finish_time || d.start_time;
+ if (Math.max(d.start_time, min) <= Math.min(finish_time, max))
+ return true;
+ return false;
+ }
+
+ // from the data, build a list of bars, dots, and lines
+ var clipped = {bars: [], dots: [], lines: []};
+ data.items.filter(inside).forEach(function(d) {
+ if (!d.finish_time) {
+ clipped.dots.push(d);
+ } else {
+ clipped.bars.push(d);
+ if (!!d.server_sent) {
+ clipped.lines.push(d);
+ }
+ }
+ });
+ globals.clipped = clipped;
+
+ //chart.select(".dyhb-label")
+ // .attr("x", x(0))//"20px")
+ // .attr("y", y);
+
+ // Panning and zooming will re-run this function multiple times, and
+ // bars will come and go, so we must process all three selections
+ // (including enter() and exit()).
+
+ // TODO: add dots for events that have only start, not finish. Add
+ // the server-sent bar (a vertical line, half height, centered
+ // vertically) for events that have server-sent as well as finish.
+ // This probably requires creating a dot for everything, but making
+ // it invisible if finished is non-null, likewise for the server-sent
+ // bar.
+
+ // each item gets an SVG group (g.bars), translated left and down
+ // to match the start time and side/lane of the event
+ var bars = chart.selectAll("g.bars")
+ .data(clipped.bars, function(d) { return d.start_time; })
+ .attr("transform", function(d) {
+ return "translate("+left(d)+","+y_off(d)+")"; })
+ ;
+ // update the variable parts of each bar, which depends upon the
+ // current pan/zoom values
+ bars.select("rect")
+ .attr("width", width);
+ bars.select("text")
+ .attr("x", halfwidth);
+ bars.exit().remove();
+ var new_bars = bars.enter()
+ .append("svg:g")
+ .attr("class", "bars")
+ .attr("transform", function(d) {
+ return "translate("+left(d)+","+y_off(d)+")"; })
+ ;
+
+ // inside the group, we have a rect with a width for the duration of
+ // the event, and a fixed height. The fill and stroke color depend
+ // upon the event, and the title has the details. We append the rects
+ // first, so the text is drawn on top (higher z-order)
+ //y += 30*(1+d3.max(data.bars, function(d){return d.row;}));
+ new_bars.append("svg:rect")
+ .attr("width", width)
+ .attr("height", RECT_HEIGHT)
+ .attr("class", function(d) {
+ var c = ["bar", "lane-" + d.lane];
+ if (d.details.waiting)
+ c.push("wait-" + d.details.waiting);
+ return c.join(" ");
+ })
+ .on("mouseover", function(d) {if (d.details_str) tip.show(d);})
+ .on("mouseout", tip.hide)
+ //.attr("title", function(d) {return d.details_str;})
+ ;
+
+ // each group also has a text, with 'x' set to place it in the middle
+ // of the rect, and text contents that are drawn in the rect
+ new_bars.append("svg:text")
+ .attr("x", halfwidth)
+ .attr("text-anchor", "middle")
+ .attr("dy", "0.9em")
+ //.attr("fill", "black")
+ .text((d) => d.what)
+ .on("mouseover", function(d) {if (d.details_str) tip.show(d);})
+ .on("mouseout", tip.hide)
+ ;
+
+ // dots: events that have a single timestamp, rather than a range.
+ // These get an SVG group, and a circle and some text.
+ var dots = chart.selectAll("g.dots")
+ .data(clipped.dots, (d) => d.start_time)
+ .attr("transform",
+ (d) => "translate("+left(d)+","+(y_off(d)+LANE_HEIGHT/3)+")")
+ ;
+ dots.exit().remove();
+ var new_dots = dots.enter()
+ .append("svg:g")
+ .attr("class", "dots")
+ .attr("transform",
+ (d) => "translate("+left(d)+","+(y_off(d)+LANE_HEIGHT/3)+")")
+ ;
+ new_dots.append("svg:circle")
+ .attr("r", "5")
+ .attr("class", (d) => "dot lane-"+d.lane)
+ .attr("fill", "#888")
+ .attr("stroke", "black")
+ .on("mouseover", function(d) {if (d.details_str) tip.show(d);})
+ .on("mouseout", tip.hide)
+ ;
+ new_dots.append("svg:text")
+ .attr("x", "5px")
+ .attr("text-anchor", "start")
+ .attr("dy", "0.2em")
+ .text((d) => d.what)
+ .on("mouseover", function(d) {if (d.details_str) tip.show(d);})
+ .on("mouseout", tip.hide)
+ ;
+
+ // lines: these represent the time at which the server sent a message
+ // which finished a bar. These get an SVG group, and a line
+ var lines = chart.selectAll("g.lines")
+ .data(clipped.lines, (d) => d.start_time)
+ .attr("transform",
+ (d) => "translate("+left_server(d)+","+y_off(d)+")")
+ ;
+ lines.exit().remove();
+ var new_lines = lines.enter()
+ .append("svg:g")
+ .attr("class", "lines")
+ .attr("transform",
+ (d) => "translate("+left_server(d)+","+(y_off(d))+")")
+ ;
+ new_lines.append("svg:line")
+ .attr("x1", 0).attr("y1", -5).attr("x2", "0").attr("y2", LANE_HEIGHT)
+ .attr("class", (d) => "line lane-"+d.lane)
+ .attr("stroke", "red")
+ ;
+
+
+
+
+ // horizontal scale markers: vertical lines at rational timestamps
+ var rules = chart.selectAll("g.rule")
+ .data(x.ticks(10))
+ .attr("transform", tx);
+ rules.select("text").text(x.tickFormat(10));
+
+ var newrules = rules.enter().insert("svg:g")
+ .attr("class", "rule")
+ .attr("transform", tx)
+ ;
+
+ newrules.append("svg:line")
+ .attr("class", "rule-tick")
+ .attr("stroke", "black");
+ chart.selectAll("line.rule-tick")
+ .attr("y1", bottom_rule_y)
+ .attr("y2", bottom_rule_y + 6);
+ newrules.append("svg:line")
+ .attr("class", "rule-red")
+ .attr("stroke", "red")
+ .attr("stroke-opacity", .3);
+ chart.selectAll("line.rule-red")
+ .attr("y1", 0)
+ .attr("y2", bottom_rule_y);
+ newrules.append("svg:text")
+ .attr("class", "rule-text")
+ .attr("dy", ".71em")
+ .attr("text-anchor", "middle")
+ .attr("fill", "black")
+ .text(x.tickFormat(10));
+ chart.selectAll("text.rule-text")
+ .attr("y", bottom_rule_y + 9);
+ rules.exit().remove();
+ chart.select(".seconds-label")
+ .attr("x", w/2)
+ .attr("y", bottom_rule_y + 35);
+
+ }
+ globals.x = x;
+ globals.redraw = redraw;
+
+ zoom.on("zoom", redraw);
+
+ d3.select("#zoom_in_button").on("click", zoomin);
+ d3.select("#zoom_out_button").on("click", zoomout);
+ d3.select("#reset_button").on("click",
+ function() {
+ x.domain([-last*0.05, last]).range([0,w]);
+ redraw();
+ });
+
+ redraw();
$.get("done", function(_) {});
});
-
-function zoom(properties) {
- var target = properties.time.valueOf();
- var w = timeline.getWindow();
- var span = w.end - w.start;
- var new_span = span / 2;
- var new_start = target - new_span/2;
- var new_end = target + new_span/2;
- timeline.setWindow(new_start, new_end, {animation: true});
-}
-
-function update_cursor(properties) {
- var t = properties.time;
- document.getElementById("cursor_date").innerText = t;
- var m = vis.moment(t);
- document.getElementById("cursor_time").innerText = m.format("ss.SSSSSS");
-}
-
-function select_item(properties) {
- var item_id = properties.items[0];
- var i = items.get(item_id);
- if (i.end) {
- var elapsed = (i.end - i.start) / 1000;
- $("div#elapsed").text("elapsed: " + elapsed + " s");
- } else {
- $("div#elapsed").text("");
- }
-}