From 7f65b1348853fdfe9bc69b5b86cc783db1565ac4 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 29 Apr 2016 14:28:04 -0700 Subject: [PATCH] rewrite timeline with d3.js --- misc/dump-timing.py | 173 +++++++++++------- misc/web/timeline.css | 47 ++++- misc/web/timeline.html | 6 +- misc/web/timeline.js | 405 +++++++++++++++++++++++++++++++++++------ 4 files changed, 501 insertions(+), 130 deletions(-) diff --git a/misc/dump-timing.py b/misc/dump-timing.py index 6a222a8..c36e44d 100644 --- a/misc/dump-timing.py +++ b/misc/dump-timing.py @@ -1,78 +1,122 @@ # To use the web() option, you should do: # * cd misc -# * npm install vis zepto +# * npm install d3 zepto from __future__ import print_function -import os, sys, time, json +import os, sys, time, json, random streams = sys.argv[1:] +if not streams: + print("run like: python dump-timing.py tx.json rx.json") + sys.exit(1) num_streams = len(streams) labels = dict([(num, " "*num + "[%d]" % (num+1) + " "*(num_streams-1-num)) for num in range(num_streams)]) -timeline = [] -groups_out = [] -for which,fn in enumerate(streams): +abs_timeline = [] +sides = [] + +for side,fn in enumerate(streams): with open(fn, "rb") as f: - for (start, sent, finish, what, start_d, finish_d) in json.load(f): - timeline.append( (start, sent, finish, which, what, - start_d, finish_d) ) - print("%s is %s" % (labels[which], fn)) - groups_out.append({"id": which, "content": fn, - "className": "group-%d" % which}) + for (start, sent, finish, what, details) in json.load(f): + abs_timeline.append( (start, sent, finish, side, what, details) ) + print("%s is %s" % (labels[side], fn)) + sides.append(os.path.basename(fn)) +# relativize all timestamps +all_times = [e[0] for e in abs_timeline] + [e[2] for e in abs_timeline if e[2]] +all_times.sort() +earliest = all_times[0] +def rel(t): + if t is None: return None + return t - earliest +timeline = [ (rel(start), rel(sent), rel(finish), side, what, details) + for (start, sent, finish, side, what, details) + in abs_timeline ] +data = {} -timeline.sort(key=lambda row: row[0]) -first = timeline[0][0] -print("started at %s" % time.ctime(start)) -viz_out = [] -for num, (start, sent, finish, which, what, - start_d, finish_d) in enumerate(timeline): - delta = start - first - delta_s = "%.6f" % delta - start_d_str = ", ".join(["%s=%s" % (name, start_d[name]) - for name in sorted(start_d)]) - finish_d_str = ", ".join(["%s=%s" % (name, finish_d[name]) - for name in sorted(finish_d)]) - details_str = start_d_str - if finish_d_str: - details_str += "/" + finish_d_str - finish_str = "" - if finish is not None: - finish_str = " +%.6f" % (finish - start) - print("%9s: %s %s %s%s" % (delta_s, labels[which], what, details_str, - finish_str)) - viz_start = start*1000 - viz_end = None if finish is None else finish*1000 - viz_type = "range" if finish else "point" - if what == "wormhole started" or what == "wormhole": - viz_type = "background" - viz_content = '%s' % (details_str or "(No details)", - what) # sigh - viz_className = "item-group-%d" % which - if "waiting" in start_d: - viz_className += " wait-%s" % start_d["waiting"] - viz_out.append({"id": num, "start": viz_start, "end": viz_end, - "group": which, #"subgroup": num, - "content": viz_content, - "className": viz_className, # or style: - "type": viz_type, - }) - if sent is not None: - viz_out.append({"id": "%d.sent" % num, "start": sent*1000, - "group": which, #"subgroup": num, - "content": "sent", - "className": viz_className, - "type": "point"}) +# we pre-calculate the "lane" that each item uses, here in python, rather +# than leaving that up to the javascript. +data["lanes"] = ["proc", # 0 gets high-level events and spans: process start, + # imports, command dispatch, code established, key + # established, transit connected, process exit + "API", # 1 gets API call spans (apps usually only wait for + # one at a time, so they won't overlap): get_code, + # input_code, get_verifier, get_data, send_data, + # close + "wait", # 2 shows waiting-for-human: input code, get + # permission + "app", # 3: file-xfer events + "skt", # 4: websocket message send/receives + "misc", # 5: anything else + ] +data["bounds"] = {"min": rel(all_times[0]), "max": rel(all_times[-1]), + } +data["sides"] = sides +print("started at %s" % time.ctime(earliest)) +print("duration %s seconds" % data["bounds"]["max"]) +items = data["items"] = [] +for num, (start, sent, finish, side, what, details) in enumerate(timeline): + background = False + if what in ["wormhole",]: + # background region for wormhole lifetime + lane = 0 + background = True + elif what in ["process start", "import", "command dispatch", + "code established", "key established", "transit connected", + "exit"]: + lane = 0 + elif what in ["API get_code", "API input_code", "API set_code", + "API get_verifier", "API get_data", "API send_data", + #"API get data", "API send data", + "API close"]: + lane = 1 + elif details.get("waiting") in ["user", "crypto"]: # permission or math + lane = 2 + elif what in ["tx file", "get ack", "rx file", "unpack zip", "send ack"]: + lane = 3 + elif what in ["websocket"]: + # connection establishment + lane = 4 + background = True + elif (what in ["welcome", "error"] # rendezvous message receives + or what in ["allocate", "list", "get", "add", "deallocate"] + # rendezvous message sends + ): + lane = 4 + else: + lane = 5 # unknown + + if background: + continue # disable until I figure out how to draw these better + + details_str = ", ".join(["%s=%s" % (name, details[name]) + for name in sorted(details)]) + + items.append({"side": side, + "lane": lane, + "start_time": start, + "server_sent": sent, + "finish_time": finish, # maybe None + "what": what, + "details": details, + "details_str": details_str, + "wiggle": random.randint(0,4), + }) + + #if "waiting" in details: + # viz_className += " wait-%s" % details["waiting"] + +from pprint import pprint +pprint(data) here = os.path.dirname(__file__) web_root = os.path.join(here, "web") -vis_root = os.path.join(here, "node_modules", "vis", "dist") -zepto_root = os.path.join(here, "node_modules", "zepto") -if not os.path.isdir(vis_root) or not os.path.isdir(zepto_root): - print("Cannot find 'vis' and 'zepto' in misc/node_modules/") - print("Please run 'npm install vis zepto' from the misc/ directory.") +lib_root = os.path.join(here, "node_modules") +if not os.path.isdir(lib_root): + print("Cannot find 'd3' and 'd3-tip' in misc/node_modules/") + print("Please run 'npm install d3 d3-tip zepto' from the misc/ directory.") sys.exit(1) def web(): @@ -82,15 +126,14 @@ def web(): from twisted.internet import reactor, endpoints ep = endpoints.serverFromString(reactor, "tcp:8066:interface=127.0.0.1") root = static.File(web_root) - data_json = {"items": viz_out, "groups": groups_out} - data = json.dumps(data_json).encode("utf-8") - root.putChild("data.json", static.Data(data, "application/json")) - root.putChild("vis", static.File(vis_root)) - root.putChild("zepto", static.File(zepto_root)) + root.putChild("data.json", static.Data(json.dumps(data).encode("utf-8"), + "application/json")) + root.putChild("lib", static.File(lib_root)) class Shutdown(resource.Resource): def render_GET(self, request): - print("timeline ready, server shutting down") - reactor.stop() + if 0: + print("timeline ready, server shutting down") + reactor.stop() return "shutting down" root.putChild("done", Shutdown()) site = server.Site(root) diff --git a/misc/web/timeline.css b/misc/web/timeline.css index c49565d..ea90e15 100644 --- a/misc/web/timeline.css +++ b/misc/web/timeline.css @@ -1,20 +1,51 @@ -div.item-group-0 { - background-color: #fcc; +rect.bar { + stroke: black; } -div.item-group-1 { - background-color: #cfc; +.lane-0 { + fill: #fcc; } -div.item-group-2 { - background-color: #ccf; +.lane-1 { + fill: #cfc; } -div.wait-user { - background-color: #ccc; +.lane-2 { + fill: #ccf; +} + +.lane-3 { + fill: #ccf; +} + +.lane-4 { + fill: #ccf; +} + +.lane-5 { + fill: #ccf; +} + +.lane-6 { + fill: #ccf; +} + +rect.wait-user { + fill: #ccc; +} + +rect.wait-crypto { + fill: #bbe; } .vis-item .vis-item-overflow { overflow: visible; } + +.d3-tip { + margin: 4px; + padding: 2px; + background: #111; + color: #fff; +} diff --git a/misc/web/timeline.html b/misc/web/timeline.html index 190fdd9..41e92fe 100644 --- a/misc/web/timeline.html +++ b/misc/web/timeline.html @@ -2,9 +2,9 @@ Timeline Visualizer - - - + + + diff --git a/misc/web/timeline.js b/misc/web/timeline.js index 1554f29..0c9978f 100644 --- a/misc/web/timeline.js +++ b/misc/web/timeline.js @@ -1,59 +1,356 @@ -var vis, $; // hush +var d3, $; // hush -var container = document.getElementById("viz"); -var options = {editable: false, - showCurrentTime: false, - snap: null, - order: function(a,b) { return a.id - b.id; } - }; -var timeline = new vis.Timeline(container, options); +var container = d3.select("#viz"); +var data; var items; +var globals = {}; -$.getJSON("data.json", function(data) { - items = new vis.DataSet(data.items); - timeline.setData({groups: new vis.DataSet(data.groups), - items: items}); - var start = data.items[0].start; - var end = data.items[data.items.length-1].start; - var span = end - start; - timeline.setWindow(start - (span/10), end + (span/10)); - //timeline.fit(); // doesn't leave space on the ends - timeline.setOptions({min: start - (span/10), - max: end + (span/10), - zoomMin: 50, - zoomMax: 1.2*span}); - var bar = timeline.addCustomTime(start, "cursor"); - timeline.on("timechange", update_cursor); - update_cursor({time: new Date(start)}); - timeline.on("doubleClick", zoom); - timeline.on("select", select_item); +var zoom = d3.behavior.zoom().scaleExtent([1, Infinity]); +function zoomin() { + //var w = Number(container.style("width").slice(0,-2)); + //console.log("zoomin", w); + //zoom.center([w/2, 20]); // doesn't work yet + zoom.scale(zoom.scale() * 2); + globals.redraw(); +} +function zoomout() { + zoom.scale(zoom.scale() * 0.5); + globals.redraw(); +} + +$.getJSON("data.json", function(d) { + data = d; + + const LANE_HEIGHT = 30; + const RECT_HEIGHT = 20; + + // the Y axis is as follows: + // * each lane is LANE_HEIGHT tall (e.g. 30px) + // * the actual rects are RECT_HEIGHT tall (e.g. 20px) + // * there is a one-lane-tall gap at the top of the chart + // * there are data.sides.length sides (e.g. one tx, one rx) + // * there are data.lanes.length lanes (e.g. 6), each with a name + // * there is a one-lane-tall gap between each side + // * there is a one-lane-tall gap after the last lane + // * the horizontal scale markers begin after that gap + // * the tick marks extend another 6 pixels down + + var w = Number(container.style("width").slice(0,-2)); + + function y_off(d) { + return (LANE_HEIGHT * (d.side*(data.lanes.length+1) + d.lane) + + d.wiggle); + } + var bottom_rule_y = LANE_HEIGHT * data.sides.length * (data.lanes.length+1); + var bottom_y = bottom_rule_y + 45; + + var tip = d3.tip() + .attr("class", "d3-tip") + .html(function(d) { return "" + d.details_str + ""; }) + .direction("s") + ; + + var chart = container.append("svg:svg") + .attr("id", "outer_chart") + .attr("width", w) + .attr("pointer-events", "all") + .call(zoom) + .call(tip) + ; + //var chart_g = chart.append("svg:g"); + + // this "backboard" rect lets us catch mouse events anywhere in the + // chart, even between the bars. Without it, we only see events on solid + // objects like bars and text, but not in the gaps between. + chart.append("svg:rect") + .attr("id", "outer_rect") + .attr("width", w).attr("height", bottom_y).attr("fill", "none"); + + // but the stuff we put inside it should have some room + w = w-50; + + chart.selectAll("text.sides-label").data(data.sides).enter() + .append("svg:text") + .attr("class", "sides-label") + .attr("x", "0px") + .attr("y", function(d,idx) { + return y_off({side: idx, lane: data.lanes.length/2, + wiggle: 0}) ;}) + .attr("text-anchor", "start") // anchor at top-left + .attr("dy", ".71em") + .attr("fill", "black") + .text(function(d) { return d; }) + ; + + var lanes_by_sides = []; + data.sides.forEach(function(side, side_index) { + data.lanes.forEach(function(lane, lane_index) { + lanes_by_sides.push({side: side, side_index: side_index, + lane: lane, lane_index: lane_index}); + }); + }); + + chart.selectAll("text.lanes-label").data(lanes_by_sides).enter() + .append("svg:text") + .attr("class", "lanes-label") + .attr("x", "50px") + .attr("y", function(d) { + return y_off({side: d.side_index, lane: d.lane_index, + wiggle: 0}) ;}) + .attr("text-anchor", "start") // anchor at top-left + .attr("dy", ".91em") + .attr("fill", "#f88") + .text(function(d) { return d.lane; }) + ; + + chart.append("svg:text") + .attr("class", "seconds-label") + //.attr("x", w/2).attr("y", y + 35) + .attr("text-anchor", "middle") + .attr("fill", "black") + .text("seconds"); + + d3.select("#outer_chart").attr("height", bottom_y); + d3.select("#outer_rect").attr("height", bottom_y); + d3.select("#zoom").attr("transform", "translate("+(w-10)+","+10+")"); + + function reltime(t) {return t-data.bounds.min;} + var last = data.bounds.max - data.bounds.min; + //last = reltime(d3.max(data.dyhb, function(d){return d.finish_time;})); + last = last * 1.05; + // long downloads are likely to have too much info, start small + if (last > 10.0) + last = 10.0; + // d3.time.scale() has no support for ms or us. + var xOFF = d3.time.scale().domain([data.bounds.min, data.bounds.max]) + .range([0,w]); + var x = d3.scale.linear().domain([-last*0.05, last]) + .range([0,w]); + zoom.x(x); + function tx(d) { return "translate(" +x(d) + ",0)"; } + function left(d) { return x(reltime(d.start_time)); } + function left_server(d) { return x(reltime(d.server_sent)); } + function right(d) { + return d.finish_time ? x(reltime(d.finish_time)) : "1px"; + } + function width(d) { + return d.finish_time ? x(reltime(d.finish_time))-x(reltime(d.start_time)) : "1px"; + } + function halfwidth(d) { + if (d.finish_time) + return (x(reltime(d.finish_time))-x(reltime(d.start_time)))/2; + return "1px"; + } + function middle(d) { + if (d.finish_time) + return (x(reltime(d.start_time))+x(reltime(d.finish_time)))/2; + else + return x(reltime(d.start_time)) + 1; + } + function color(d) { return data.server_info[d.serverid].color; } + function servername(d) { return data.server_info[d.serverid].short; } + function timeformat(duration) { + // TODO: trim to microseconds, maybe humanize + return duration; + } + + function redraw() { + // at this point zoom/pan must be fixed + var min = data.bounds.min + x.domain()[0]; + var max = data.bounds.min + x.domain()[1]; + function inside(d) { + var finish_time = d.finish_time || d.start_time; + if (Math.max(d.start_time, min) <= Math.min(finish_time, max)) + return true; + return false; + } + + // from the data, build a list of bars, dots, and lines + var clipped = {bars: [], dots: [], lines: []}; + data.items.filter(inside).forEach(function(d) { + if (!d.finish_time) { + clipped.dots.push(d); + } else { + clipped.bars.push(d); + if (!!d.server_sent) { + clipped.lines.push(d); + } + } + }); + globals.clipped = clipped; + + //chart.select(".dyhb-label") + // .attr("x", x(0))//"20px") + // .attr("y", y); + + // Panning and zooming will re-run this function multiple times, and + // bars will come and go, so we must process all three selections + // (including enter() and exit()). + + // TODO: add dots for events that have only start, not finish. Add + // the server-sent bar (a vertical line, half height, centered + // vertically) for events that have server-sent as well as finish. + // This probably requires creating a dot for everything, but making + // it invisible if finished is non-null, likewise for the server-sent + // bar. + + // each item gets an SVG group (g.bars), translated left and down + // to match the start time and side/lane of the event + var bars = chart.selectAll("g.bars") + .data(clipped.bars, function(d) { return d.start_time; }) + .attr("transform", function(d) { + return "translate("+left(d)+","+y_off(d)+")"; }) + ; + // update the variable parts of each bar, which depends upon the + // current pan/zoom values + bars.select("rect") + .attr("width", width); + bars.select("text") + .attr("x", halfwidth); + bars.exit().remove(); + var new_bars = bars.enter() + .append("svg:g") + .attr("class", "bars") + .attr("transform", function(d) { + return "translate("+left(d)+","+y_off(d)+")"; }) + ; + + // inside the group, we have a rect with a width for the duration of + // the event, and a fixed height. The fill and stroke color depend + // upon the event, and the title has the details. We append the rects + // first, so the text is drawn on top (higher z-order) + //y += 30*(1+d3.max(data.bars, function(d){return d.row;})); + new_bars.append("svg:rect") + .attr("width", width) + .attr("height", RECT_HEIGHT) + .attr("class", function(d) { + var c = ["bar", "lane-" + d.lane]; + if (d.details.waiting) + c.push("wait-" + d.details.waiting); + return c.join(" "); + }) + .on("mouseover", function(d) {if (d.details_str) tip.show(d);}) + .on("mouseout", tip.hide) + //.attr("title", function(d) {return d.details_str;}) + ; + + // each group also has a text, with 'x' set to place it in the middle + // of the rect, and text contents that are drawn in the rect + new_bars.append("svg:text") + .attr("x", halfwidth) + .attr("text-anchor", "middle") + .attr("dy", "0.9em") + //.attr("fill", "black") + .text((d) => d.what) + .on("mouseover", function(d) {if (d.details_str) tip.show(d);}) + .on("mouseout", tip.hide) + ; + + // dots: events that have a single timestamp, rather than a range. + // These get an SVG group, and a circle and some text. + var dots = chart.selectAll("g.dots") + .data(clipped.dots, (d) => d.start_time) + .attr("transform", + (d) => "translate("+left(d)+","+(y_off(d)+LANE_HEIGHT/3)+")") + ; + dots.exit().remove(); + var new_dots = dots.enter() + .append("svg:g") + .attr("class", "dots") + .attr("transform", + (d) => "translate("+left(d)+","+(y_off(d)+LANE_HEIGHT/3)+")") + ; + new_dots.append("svg:circle") + .attr("r", "5") + .attr("class", (d) => "dot lane-"+d.lane) + .attr("fill", "#888") + .attr("stroke", "black") + .on("mouseover", function(d) {if (d.details_str) tip.show(d);}) + .on("mouseout", tip.hide) + ; + new_dots.append("svg:text") + .attr("x", "5px") + .attr("text-anchor", "start") + .attr("dy", "0.2em") + .text((d) => d.what) + .on("mouseover", function(d) {if (d.details_str) tip.show(d);}) + .on("mouseout", tip.hide) + ; + + // lines: these represent the time at which the server sent a message + // which finished a bar. These get an SVG group, and a line + var lines = chart.selectAll("g.lines") + .data(clipped.lines, (d) => d.start_time) + .attr("transform", + (d) => "translate("+left_server(d)+","+y_off(d)+")") + ; + lines.exit().remove(); + var new_lines = lines.enter() + .append("svg:g") + .attr("class", "lines") + .attr("transform", + (d) => "translate("+left_server(d)+","+(y_off(d))+")") + ; + new_lines.append("svg:line") + .attr("x1", 0).attr("y1", -5).attr("x2", "0").attr("y2", LANE_HEIGHT) + .attr("class", (d) => "line lane-"+d.lane) + .attr("stroke", "red") + ; + + + + + // horizontal scale markers: vertical lines at rational timestamps + var rules = chart.selectAll("g.rule") + .data(x.ticks(10)) + .attr("transform", tx); + rules.select("text").text(x.tickFormat(10)); + + var newrules = rules.enter().insert("svg:g") + .attr("class", "rule") + .attr("transform", tx) + ; + + newrules.append("svg:line") + .attr("class", "rule-tick") + .attr("stroke", "black"); + chart.selectAll("line.rule-tick") + .attr("y1", bottom_rule_y) + .attr("y2", bottom_rule_y + 6); + newrules.append("svg:line") + .attr("class", "rule-red") + .attr("stroke", "red") + .attr("stroke-opacity", .3); + chart.selectAll("line.rule-red") + .attr("y1", 0) + .attr("y2", bottom_rule_y); + newrules.append("svg:text") + .attr("class", "rule-text") + .attr("dy", ".71em") + .attr("text-anchor", "middle") + .attr("fill", "black") + .text(x.tickFormat(10)); + chart.selectAll("text.rule-text") + .attr("y", bottom_rule_y + 9); + rules.exit().remove(); + chart.select(".seconds-label") + .attr("x", w/2) + .attr("y", bottom_rule_y + 35); + + } + globals.x = x; + globals.redraw = redraw; + + zoom.on("zoom", redraw); + + d3.select("#zoom_in_button").on("click", zoomin); + d3.select("#zoom_out_button").on("click", zoomout); + d3.select("#reset_button").on("click", + function() { + x.domain([-last*0.05, last]).range([0,w]); + redraw(); + }); + + redraw(); $.get("done", function(_) {}); }); - -function zoom(properties) { - var target = properties.time.valueOf(); - var w = timeline.getWindow(); - var span = w.end - w.start; - var new_span = span / 2; - var new_start = target - new_span/2; - var new_end = target + new_span/2; - timeline.setWindow(new_start, new_end, {animation: true}); -} - -function update_cursor(properties) { - var t = properties.time; - document.getElementById("cursor_date").innerText = t; - var m = vis.moment(t); - document.getElementById("cursor_time").innerText = m.format("ss.SSSSSS"); -} - -function select_item(properties) { - var item_id = properties.items[0]; - var i = items.get(item_id); - if (i.end) { - var elapsed = (i.end - i.start) / 1000; - $("div#elapsed").text("elapsed: " + elapsed + " s"); - } else { - $("div#elapsed").text(""); - } -}