overhaul dump-timing JS

Do all the work in JS.

misc/dump-timing.py

@@ -7,106 +7,18 @@ import os, sys, time, json, random
 
 
 streams = sys.argv[1:]
-if not streams:
+if len(streams) != 2:
     print("run like: python dump-timing.py tx.json rx.json")
     sys.exit(1)
-num_streams = len(streams)
+# for now, require sender as first file, receiver as second
-labels = dict([(num, " "*num + "[%d]" % (num+1) + " "*(num_streams-1-num))
+# later, allow use of only one file.
-               for num in range(num_streams)])
-abs_timeline = []
-sides = []
 
-for side,fn in enumerate(streams):
-    with open(fn, "rb") as f:
-        for (start, sent, finish, what, details) in json.load(f):
-            abs_timeline.append( (start, sent, finish, side, what, details) )
-    print("%s is %s" % (labels[side], fn))
-    sides.append(os.path.basename(fn))
-
-# relativize all timestamps
-all_times = [e[0] for e in abs_timeline] + [e[2] for e in abs_timeline if e[2]]
-all_times.sort()
-earliest = all_times[0]
-def rel(t):
-    if t is None: return None
-    return t - earliest
-timeline = [ (rel(start), rel(sent), rel(finish), side, what, details)
-             for (start, sent, finish, side, what, details)
-             in abs_timeline ]
 data = {}
-
+for i,fn in enumerate(streams):
-# we pre-calculate the "lane" that each item uses, here in python, rather
+    name = ["send", "receive"][i]
-# than leaving that up to the javascript.
+    with open(fn, "rb") as f:
-data["lanes"] = ["proc", # 0 gets high-level events and spans: process start,
+        events = json.load(f)
-                         # imports, command dispatch, code established, key
+    data[name] = {"fn": os.path.basename(fn), "events": events}
-                         # established, transit connected, process exit
-                 "API", # 1 gets API call spans (apps usually only wait for
-                        # one at a time, so they won't overlap): get_code,
-                        # input_code, get_verifier, get_data, send_data,
-                        # close
-                 "wait", # 2 shows waiting-for-human: input code, get
-                         # permission
-                 "app", # 3: file-xfer events
-                 "skt", # 4: websocket message send/receives
-                 "misc", # 5: anything else
-                 ]
-data["bounds"] = {"min": rel(all_times[0]), "max": rel(all_times[-1]),
-                  }
-data["sides"] = sides
-print("started at %s" % time.ctime(earliest))
-print("duration %s seconds" % data["bounds"]["max"])
-items = data["items"] = []
-
-for num, (start, sent, finish, side, what, details) in enumerate(timeline):
-    background = False
-    if what in ["wormhole",]:
-        # background region for wormhole lifetime
-        lane = 0
-        background = True
-    elif what in ["process start", "import", "command dispatch",
-                  "code established", "key established", "transit connected",
-                  "exit"]:
-        lane = 0
-    elif what in ["API get_code", "API input_code", "API set_code",
-                  "API get_verifier", "API get_data", "API send_data",
-                  #"API get data", "API send data",
-                  "API close"]:
-        lane = 1
-    elif details.get("waiting") in ["user", "crypto"]: # permission or math
-        lane = 2
-    elif what in ["tx file", "get ack", "rx file", "unpack zip", "send ack"]:
-        lane = 3
-    elif what in ["websocket"]:
-        # connection establishment
-        lane = 4
-        background = True
-    elif (what in ["welcome", "error"] # rendezvous message receives
-          or what in ["allocate", "list", "get", "add", "deallocate"]
-          # rendezvous message sends
-          ):
-        lane = 4
-    else:
-        lane = 5 # unknown
-
-    if background:
-        continue # disable until I figure out how to draw these better
-
-    details_str = ", ".join(["%s=%s" % (name, details[name])
-                             for name in sorted(details)])
-
-    items.append({"side": side,
-                  "lane": lane,
-                  "start_time": start,
-                  "server_sent": sent,
-                  "finish_time": finish, # maybe None
-                  "what": what,
-                  "details": details,
-                  "details_str": details_str,
-                  "wiggle": random.randint(0,4),
-                  })
-
-    #if "waiting" in details:
-    #    viz_className += " wait-%s" % details["waiting"]
 
 from pprint import pprint
 pprint(data)

misc/web/timeline.css

@@ -1,4 +1,57 @@
 
+line.client_tx {
+    stroke: red;
+    stroke-dasharray: 5,5;
+}
+line.client_rx {
+    stroke: blue;
+    stroke-dasharray: 5,5;
+}
+line.c2c_column {
+    stroke: black;
+    stroke-dasharray: 1,5;
+}
+
+line.c2c {
+    stroke-width: 2.0;
+}
+line.c2c.active {
+    stroke-width: 4.0;
+}
+
+line.y_axis {
+    stroke: gray;
+}
+
+/* putting these in a .css file doesn't work, for some reason I have to add
+the markers as a style= attribute directly. */
+line.circle-arrow-circle {
+    /*marker-start: url(#markerCircle);*/
+    marker-mid: url(#markerArrow);
+    /*marker-end: url(#markerCircle);*/
+}
+
+rect.wait-crypto {
+    stroke: black;
+    fill: #ccc;
+}
+
+rect.wait-user {
+    stroke: #00f;
+    fill: #bbe;
+}
+text.wait-text-user {
+    fill: #00f;
+}
+
+rect.proc-span-import {
+    fill: #fcc;
+}
+
+rect.api {
+    fill: #cfc;
+}
+
 rect.bar {
     stroke: black;
 }
@@ -31,14 +84,6 @@ rect.bar {
     fill: #ccf;
 }
 
-rect.wait-user {
-    fill: #ccc;
-}
-
-rect.wait-crypto {
-    fill: #bbe;
-}
-
 .vis-item .vis-item-overflow {
     overflow: visible;
 }

misc/web/timeline.js

@@ -1,4 +1,4 @@
-var d3, $; // hush
+var d3; // hush
 
 var container = d3.select("#viz");
 var data;
@@ -18,24 +18,693 @@ function zoomout() {
     globals.redraw();
 }
 
-$.getJSON("data.json", function(d) {
+function is_span(ev, category) {
+    if (ev.category === category && !!ev.stop)
+        return true;
+    return false;
+}
+function is_event(ev, category) {
+    if (ev.category === category && !ev.stop)
+        return true;
+    return false;
+}
+
+const server_message_color = {
+    "welcome": 0, // receive
+
+    "bind": 0, // send
+
+    "allocate": 1, // send
+    "allocated": 1, // receive
+
+    "list": 2, // send
+    "channelids": 2, // receive
+
+    "claim": 3, // send
+    "watch": 4, // send
+
+    "deallocate": 5, // send
+    "deallocated": 5, // receive
+
+    "error": 6, // receive
+
+    //"add": 8, // send (client message)
+    //"message": 8, // receive (client message)
+
+    "ping": 7, // send
+    "pong": 7 // receive
+};
+
+const proc_map = {
+    "command dispatch": "dispatch",
+    "code established": "code-established",
+    "key established": "key-established",
+    "transit connected": "transit-connected",
+    "exit": "exit",
+    "transit connect": "transit-connect",
+    "import": "import"
+ };
+
+const TX_COLUMN = 14;
+const RX_COLUMN = 18;
+const SERVER_COLUMN0 = 20;
+const SERVER_COLUMNS = [20,21,22,23,24,25];
+const NUM_SERVER_COLUMNS = 6;
+const MAX_COLUMN = 45;
+
+function x_offset(offset, side_name) {
+    if (side_name === "send")
+        return offset;
+    return MAX_COLUMN - offset;
+}
+function side_text_anchor(side_name) {
+    if (side_name === "send")
+        return "end";
+    return "start";
+}
+function side_text_dx(side_name) {
+    if (side_name === "send")
+        return "-5px";
+    return "5px";
+}
+
+d3.json("data.json", function(d) {
     data = d;
 
-    const LANE_HEIGHT = 30;
+    // data is {send,receive}{fn,events}
-    const RECT_HEIGHT = 20;
+    // each event has: {name, start, [stop], [server_rx], [server_tx],
+    //                  [id], details={} }
 
-    // the Y axis is as follows:
+    // Display all timestamps relative to the sender's startup event. If all
-    // * each lane is LANE_HEIGHT tall (e.g. 30px)
+    // clocks are in sync, this will be the same as first_timestamp, but in
-    // * the actual rects are RECT_HEIGHT tall (e.g. 20px)
+    // case they aren't, I'd rather use the sender as the reference point.
-    // * there is a one-lane-tall gap at the top of the chart
+    var first = data.send.events[0].start;
-    // * there are data.sides.length sides (e.g. one tx, one rx)
+
-    // * there are data.lanes.length lanes (e.g. 6), each with a name
+    // The X axis is divided up into 50 slots, and then scaled to the screen
-    // * there is a one-lane-tall gap between each side
+    // later. The left portion represents the "wormhole send" side, the
-    // * there is a one-lane-tall gap after the last lane
+    // middle is the rendezvous server, and the right portion is the
-    // * the horizontal scale markers begin after that gap
+    // "wormhole receive" side.
-    // * the tick marks extend another 6 pixels down
+    //
+    // 0: time axis, tick marks
+    // 3: sender process events: import, dispatch, exit
+    // 4: sender major application-level events: code/key establishment,
+    //    transit-connect
+    // 8: sender stalls: waiting for user, waiting for permission
+    // 10: sender websocket transmits originate from here
+    // 15: sender websocket receives terminate here
+
+    // 20-25: rendezvous-server message lanes
+
+    // 30: receiver websocket receives
+    // 35: receiver websocket transmits
+    // 37: receiver stalls
+    // 41: receiver app-level events
+    // 42: receiver process events
+
+    var first_timestamp = Infinity;
+    var last_timestamp = 0;
+    function prepare_data(e, side_name) {
+        var rel_e = {side_name: side_name, // send or receive
+                     name: e.name,
+                     start: e.start - first,
+                     details: e.details
+                    };
+        if (e.stop) rel_e.stop = e.stop - first;
+
+        // sort events into categories, assign X coordinates to some
+        if (proc_map[e.name]) {
+            rel_e.category = "proc";
+            rel_e.x = x_offset(3, side_name);
+            rel_e.text = proc_map[e.name];
+            if (e.name === "import")
+                rel_e.text += " " + e.details.which;
+        }
+        if (e.details.waiting) {
+            rel_e.category = "wait";
+            var off = 8;
+            if (e.details.waiting === "user")
+                off += 0.5;
+            rel_e.x = x_offset(off, side_name);
+        }
+
+        // also, calculate the overall time domain while we're at it
+        [rel_e.start, rel_e.stop].forEach(v => {
+            if (v) {
+                if (v > last_timestamp) last_timestamp = v;
+                if (v < first_timestamp) first_timestamp = v;
+            }
+        });
+        return rel_e;
+    }
+    var events = data.send.events.map(e => prepare_data(e, "send"));
+    events = events.concat(data.receive.events.map(e => prepare_data(e, "receive")));
+
+    /* "Client messages" are ones that go all the way from one client to the
+     other, through the rendezvous channel (and get echoed back to the sender
+     too). We can correlate three websocket messages for each (the send, the
+     local receive, and the remote receive) by comparing their "id" strings.
+
+     Scan for all client messages, to build a list of central columns. For
+     each message, we'll have tx/server_rx/server_tx/rx for the sending side,
+     and server_rx/server_tx/rx for the receiving side. The "add" event
+     contributes tx, the sender's echo contributes and the "message" event
+     contributes server_rx, server_tx, and rx.
+     */
+
+    var side_map = new Map(); // side -> "send"/"receive"
+    var c2c = new Map(); // msgid => {send,receive}{tx,server_rx,server_tx,rx}
+    events.forEach(ev => {
+        var id, phase;
+        if (ev.name === "ws_send") {
+            if (ev.details.type !== "add")
+                return;
+            id = ev.details.id;
+            phase = ev.details.phase;
+            side_map.set(ev.details._side, ev.side_name);
+        } else if (ev.name === "ws_receive") {
+            if (ev.details.message.type !== "message")
+                return;
+            id = ev.details.message.message.id;
+            phase = ev.details.message.message.phase;
+        } else
+            return;
+
+        if (!c2c.has(id)) {
+            c2c.set(id, {phase: phase,
+                         side_id: ev.details._side,
+                         //tx_side_name: assigned when we see 'add'
+                         id: id,
+                         arrivals: []
+                         //col, server_x: assigned later
+                         //server_rx: assigned when we see 'message'
+                        });
+        }
+        var cm = c2c.get(id);
+        if (ev.name === "ws_send") { // add
+            cm.tx = ev.start;
+            cm.tx_x = x_offset(TX_COLUMN, ev.side_name);
+            cm.tx_side_name = ev.side_name;
+        } else { // message
+            cm.server_rx = ev.details.message.message.server_rx - first;
+            cm.arrivals.push({server_tx: ev.details.message.server_tx - first,
+                              rx: ev.start,
+                              rx_x: x_offset(RX_COLUMN, ev.side_name)});
+        }
+    });
+
+    // sort c2c messages by initial sending time
+    var client_messages = Array.from(c2c.values());
+    client_messages.sort( (a,b) => (a.tx - b.tx) );
+
+    // assign columns
+    // TODO: identify overlaps between the c2c messages, share columns
+    // between messages which don't overlap
+
+    client_messages.forEach((cm,index) => {
+        cm.col = index % 6;
+        cm.server_x = 20 + cm.col;
+    });
+
+    console.log("client_messages", client_messages);
+    console.log(side_map);
+    console.log(first_timestamp, last_timestamp);
+
+    /* "Server messages" are ones that stop or originate at the rendezvous
+     server. These are of types other than "add" or "message". Although many
+     of these provoke responses, we do not attempt to correlate these with
+     any other message. For outbound ws_send messages, we know the send
+     timestamp, but not the server receipt timestamp. For inbound ws_receive
+     messages, we know both.
+     */
+    var outbound_sm = new Map();
+    globals.outbound_sm = outbound_sm;
+    events
+        .filter(ev => ev.name === "ws_send")
+        .forEach(ev => {
+            // we don't know the server receipt time, so draw a horizontal
+            // line by setting stop_timestamp=start_timestamp
+            var sm = {side_name: ev.side_name,
+                      start_timestamp: ev.start,
+                      stop_timestamp: ev.start,
+                      start_x: x_offset(TX_COLUMN, ev.side_name),
+                      end_x: x_offset(20, ev.side_name),
+                      text_x: x_offset(TX_COLUMN, ev.side_name),
+                      text_timestamp: ev.start,
+                      text_dy: "-5px",
+                      type: ev.details.type,
+                      tip: ev.details.type,
+                      ev: ev
+                     };
+            outbound_sm.set(ev.details.id, sm);
+        });
+
+    events
+        .filter(ev => ev.name === "ws_receive")
+        .filter(ev => ev.details.message.type === "ack")
+        .forEach(ev => {
+            var id = ev.details.message.id;
+            var server_tx = ev.details.message.server_tx;
+            var sm = outbound_sm.get(id);
+            sm.stop_timestamp = server_tx - first;
+        });
+
+    var server_messages = [];
+    events
+        .filter(ev => ev.name === "ws_receive")
+        .filter(ev => ev.details.message.type !== "message")
+        .filter(ev => ev.details.message.type !== "ack")
+        .forEach(ev => {
+            var sm = {side_name: ev.side_name,
+                      start_timestamp: ev.details.message.server_tx - first,
+                      stop_timestamp: ev.start,
+                      start_x: x_offset(20, ev.side_name),
+                      end_x: x_offset(RX_COLUMN, ev.side_name),
+                      text_x: x_offset(RX_COLUMN, ev.side_name),
+                      text_timestamp: ev.start,
+                      text_dy: "8px",
+                      type: ev.details.message.type,
+                      tip: ev.details.message.type,
+                      ev: ev
+                     };
+            server_messages.push(sm);
+        });
+    server_messages = server_messages.concat(
+        Array.from(outbound_sm.values())
+            .filter(sm => sm.type !== "add"));
+    console.log("server_messages", server_messages);
+
+    // TODO: this goes off the edge of the screen, use the viewport instead
+    var container_width = Number(container.style("width").slice(0,-2));
+    var container_height = Number(container.style("height").slice(0,-2));
+    container_height = 700; // no contents, so no height is allocated yet
+    // scale the X axis to the full width of our container
+    var x = d3.scale.linear().domain([0, 50]).range([0, container_width]);
+
+    // scale the Y axis later
+    var y = d3.scale.linear().domain([first_timestamp, last_timestamp])
+            .range([0, container_height]);
+    zoom.y(y);
+    zoom.on("zoom", redraw);
+
+
+    var tip = d3.tip()
+            .attr("class", "d3-tip")
+            .html(function(d) { return "<span>" + d + "</span>"; })
+            .direction("s")
+    ;
+
+    var chart = container.append("svg:svg")
+            .attr("id", "outer_chart")
+            .attr("width", container_width)
+            .attr("height", container_height)
+            .attr("pointer-events", "all")
+            .call(zoom)
+            .call(tip)
+    ;
+
+    var defs = chart.append("svg:defs");
+    defs.append("svg:marker")
+        .attr("id", "markerCircle")
+        .attr("markerWidth", 8)
+        .attr("markerHeight", 8)
+        .attr("refX", 5)
+        .attr("refY", 5)
+        .append("circle")
+        .attr("cx", 5)
+        .attr("cy", 5)
+        .attr("r", 3)
+        .attr("style", "stroke: none; fill: #000000;")
+    ;
+    defs.append("svg:marker")
+        .attr("id", "markerArrow")
+        .attr("markerWidth", 26)
+        .attr("markerHeight", 26)
+        .attr("refX", 26)
+        .attr("refY", 12)
+        .attr("orient", "auto")
+        .attr("markerUnits", "userSpaceOnUse") // don't scale to stroke-width
+        .append("path")
+        .attr("d", "M8,20 L20,12 L8,4")
+        .attr("style", "stroke: #000000; fill: none")
+    ;
+
+    chart.append("svg:line")
+        .attr("x1", x(0.5)).attr("y1", 0)
+        .attr("x2", x(0.5)).attr("y2", container_height)
+        .attr("class", "y_axis")
+    ;
+    chart.append("svg:g")
+        .attr("class", "seconds_g")
+        .attr("transform", "translate("+(x(0.5)+5)+","+(container_height-10)+")")
+        .append("svg:text")
+        .text("seconds")
+    ;
+
+    chart.append("svg:line")
+        .attr("x1", x(TX_COLUMN)).attr("y1", y(first_timestamp))
+        .attr("x2", x(TX_COLUMN)).attr("y2", y(last_timestamp))
+        .attr("class", "client_tx")
+    ;
+    chart.append("svg:text")
+        .attr("x", x(TX_COLUMN)).attr("y", 10)
+        .attr("text-anchor", "middle")
+        .text("sender tx");
+
+    chart.append("svg:line")
+        .attr("x1", x(RX_COLUMN)).attr("y1", y(first_timestamp))
+        .attr("x2", x(RX_COLUMN)).attr("y2", y(last_timestamp))
+        .attr("class", "client_rx")
+    ;
+    chart.append("svg:text")
+        .attr("x", x(RX_COLUMN)).attr("y", 10)
+        .attr("text-anchor", "middle")
+        .text("sender rx");
+
+    chart.selectAll("line.c2c_column").data(SERVER_COLUMNS)
+        .enter().append("svg:line")
+        .attr("class", "c2c_column")
+        .attr("x1", d => x(d)).attr("y1", y(first_timestamp))
+        .attr("x2", d => x(d)).attr("y2", y(last_timestamp))
+    ;
+
+    chart.append("svg:line")
+        .attr("x1", x(MAX_COLUMN-RX_COLUMN)).attr("y1", y(first_timestamp))
+        .attr("x2", x(MAX_COLUMN-RX_COLUMN)).attr("y2", y(last_timestamp))
+        .attr("class", "client_rx")
+    ;
+    chart.append("svg:text")
+        .attr("x", x(MAX_COLUMN-RX_COLUMN)).attr("y", 10)
+        .attr("text-anchor", "middle")
+        .text("receiver rx");
+
+    chart.append("svg:line")
+        .attr("x1", x(MAX_COLUMN-TX_COLUMN)).attr("y1", y(first_timestamp))
+        .attr("x2", x(MAX_COLUMN-TX_COLUMN)).attr("y2", y(last_timestamp))
+        .attr("class", "client_tx")
+    ;
+    chart.append("svg:text")
+        .attr("x", x(MAX_COLUMN-TX_COLUMN)).attr("y", 10)
+        .attr("text-anchor", "middle")
+        .text("receiver tx");
+
+    // produces list of {p_from, p_to, col, add_arrow, tip}
+    function cm_line(cm) {
+        // We draw a bunch of two-point lines
+        var lines = [];
+        function push(p_from, p_to, add_arrow) {
+            lines.push({p_from: p_from, p_to: p_to,
+                        col: cm.col, tip: cm.tip,
+                        add_arrow: add_arrow});
+        }
+        // the first goes from the sender to the server_rx, if we know it
+        // TODO: tolerate not knowing it
+        var sender_point = [cm.tx_x, cm.tx];
+        var server_rx_point = [cm.server_x, cm.server_rx];
+        push(sender_point, server_rx_point, true);
+
+        // the second goes from the server_rx to the last server_tx
+        var last_server_tx = Math.max.apply(null,
+                                            cm.arrivals.map(a => a.server_tx));
+        var last_server_tx_point = [cm.server_x, last_server_tx];
+        push(server_rx_point, last_server_tx_point, false);
+
+        cm.arrivals.forEach(ar => {
+            var delivery_tx_point = [cm.server_x, ar.server_tx];
+            var delivery_rx_point = [ar.rx_x, ar.rx];
+            push(delivery_tx_point, delivery_rx_point, true);
+        });
+
+        return lines;
+    }
+
+    var all_cm_lines = [];
+    client_messages.forEach(v => {
+        all_cm_lines = all_cm_lines.concat(cm_line(v));
+    });
+    console.log(all_cm_lines);
+    var cm_colors = d3.scale.category10();
+    chart.selectAll("line.c2c").data(all_cm_lines)
+        .enter()
+        .append("svg:line")
+        .attr("class", "c2c") // circle-arrow-circle")
+        .attr("stroke", ls => cm_colors(ls.col))
+        .attr("style", ls => {
+            if (ls.add_arrow) return "marker-end: url(#markerArrow);";
+            return "";
+        })
+        .on("mouseover", ls => {
+            if (ls.tip)
+                tip.show(ls.tip);
+            chart.selectAll("circle.c2c").filter(d => d.col == ls.col)
+                .attr("r", 10);
+            chart.selectAll("line.c2c")
+                .classed("active", d => d.col == ls.col);
+        })
+        .on("mouseout", ls => {
+            tip.hide(ls);
+            chart.selectAll("circle.c2c")
+                .attr("r", 5);
+            chart.selectAll("line.c2c")
+                .classed("active", false);
+        })
+    ;
+
+    chart.selectAll("g.c2c").data(client_messages)
+        .enter()
+        .append("svg:g")
+        .attr("class", "c2c")
+        .append("svg:text")
+        .attr("class", "c2c")
+        .attr("text-anchor", cm => side_text_anchor(cm.tx_side_name))
+        .attr("dx", cm => side_text_dx(cm.tx_side_name))
+        .attr("dy", "10px")
+        .attr("fill", cm => cm_colors(cm.col))
+        .text(cm => cm.phase);
+
+    function cm_dot(cm) {
+        var dots = [];
+        var color = cm_colors(cm.col);
+        var tip = cm.phase;
+        function push(x,y) {
+            dots.push({x: x, y: y, col: cm.col, color: color, tip: tip});
+        }
+        push(cm.tx_x, cm.tx);
+        cm.arrivals.forEach(ar => push(ar.rx_x, ar.rx));
+        return dots;
+    }
+    var all_cm_dots = [];
+    client_messages.forEach(cm => {
+        all_cm_dots = all_cm_dots.concat(cm_dot(cm));
+    });
+    chart.selectAll("circle.c2c").data(all_cm_dots)
+        .enter()
+        .append("svg:circle")
+        .attr("class", "c2c")
+        .attr("r", 5)
+        .attr("fill", dot => dot.color)
+        .on("mouseover", dot => {
+            if (dot.tip)
+                tip.show(dot.tip);
+            chart.selectAll("circle.c2c").filter(d => d.col == dot.col)
+                .attr("r", 10);
+            chart.selectAll("line.c2c")
+                .classed("active", d => d[2] == dot.col);
+        })
+        .on("mouseout", dot => {
+            tip.hide(dot);
+            chart.selectAll("circle.c2c")
+                .attr("r", 5);
+            chart.selectAll("line.c2c")
+                .classed("active", false);
+        })
+    ;
+
+    // server messages
+    chart.selectAll("line.server-message").data(server_messages)
+        .enter()
+        .append("svg:line")
+        .attr("class", "server-message")
+        .attr("stroke", sm => cm_colors(server_message_color[sm.type] || 0))
+        .attr("style", "marker-end: url(#markerArrow)")
+        .on("mouseover", sm => {
+            if (sm.tip)
+                tip.show(sm.tip);
+        })
+        .on("mouseout", sm => {
+            tip.hide(sm);
+        })
+    ;
+    chart.selectAll("g.server-message").data(server_messages)
+        .enter()
+        .append("svg:g")
+        .attr("class", "server-message")
+    .append("svg:text")
+        .attr("class", "server-message")
+        .attr("text-anchor", sm => side_text_anchor(sm.side_name))
+        .attr("dx", sm => side_text_dx(sm.side_name))
+        .attr("dy", sm => sm.text_dy)
+        .attr("fill", sm => cm_colors(server_message_color[sm.type] || 0))
+        .text(sm => sm.type);
+    // TODO: add dots on the known send/receive time points
+
+    var w = chart.selectAll("g.wait")
+        .data(events.filter(ev => ev.category === "wait"))
+        .enter().append("svg:g")
+        .attr("class", "wait");
+    w.append("svg:rect")
+        .attr("class", ev => "wait wait-"+ev.details.waiting)
+        .attr("width", 10);
+    var wt = chart.selectAll("g.wait-text")
+        .data(events.filter(ev => ev.category === "wait"))
+        .enter().append("svg:g")
+        .attr("class", "wait-text");
+    wt.append("svg:text")
+        .attr("class", ev => "wait-text wait-text-"+ev.details.waiting)
+        .attr("text-anchor", ev => ev.side_name === "send" ? "end" : "start")
+        .attr("dx", ev => ev.side_name === "send" ? "-5px" : "15px")
+        .attr("dy", "5px")
+        .text(v => v.name+" ("+v.details.waiting+")");
+
+    // process-related events
+    var pe = chart.selectAll("g.proc-event")
+        .data(events.filter(ev => is_event(ev, "proc")))
+        .enter().append("svg:g")
+        .attr("class", "proc-event");
+    pe.append("svg:circle")
+        .attr("class", ev => "proc-event proc-event-"+proc_map[ev.name])
+        .attr("cx", ev => ev.side_name === "send" ? "12px" : "-2px")
+        .attr("r", 5)
+        .attr("fill", "red")
+        .attr("width", 10);
+    pe.append("svg:text")
+        .attr("class", ev => "proc-event proc-event-"+proc_map[ev.name])
+        .attr("text-anchor", ev => ev.side_name === "send" ? "start" : "end")
+        .attr("dx", ev => ev.side_name === "send" ? "15px" : "-5px")
+        .attr("dy", "5px")
+        .attr("transform", "rotate(-30)")
+        .text(ev => proc_map[ev.name]);
+
+    // process-related spans
+    var ps = chart.selectAll("g.proc-span")
+        .data(events.filter(ev => is_span(ev, "proc")))
+        .enter().append("svg:g")
+        .attr("class", "proc-span");
+    ps.append("svg:rect")
+        .attr("class", ev => "proc-span proc-span-"+proc_map[ev.name])
+        .attr("width", 10);
+    var pst = chart.selectAll("g.proc-span-text")
+        .data(events.filter(ev => is_span(ev, "proc")))
+        .enter().append("svg:g")
+        .attr("class", "proc-span-text");
+    pst.append("svg:text")
+        .attr("class", ev => "proc-span-text proc-span-text-"+proc_map[ev.name])
+        .attr("text-anchor", ev => ev.side_name === "send" ? "start" : "end")
+        .attr("dx", ev => ev.side_name === "send" ? "15px" : "-5px")
+        .attr("dy", "5px")
+        .text(ev => ev.text);
+
+    function ty(d) { return "translate(0,"+y(d)+")"; }
+
+    function redraw() {
+        chart.selectAll("line.c2c")
+            .attr("x1", ls => x(ls.p_from[0]))
+            .attr("y1", ls => y(ls.p_from[1]))
+            .attr("x2", ls => x(ls.p_to[0]))
+            .attr("y2", ls => y(ls.p_to[1]))
+        ;
+        chart.selectAll("g.c2c")
+            .attr("transform", cm =>
+                  "translate("+x(cm.tx_x)+","+y(cm.tx)+")")
+        ;
+        chart.selectAll("circle.c2c")
+            .attr("cx", d => x(d.x))
+            .attr("cy", d => y(d.y))
+        ;
+        chart.selectAll("line.server-message")
+            .attr("x1", sm => x(sm.start_x))
+            .attr("y1", sm => y(sm.start_timestamp))
+            .attr("x2", sm => x(sm.end_x))
+            .attr("y2", sm => y(sm.stop_timestamp));
+        chart.selectAll("g.server-message")
+            .attr("transform", sm => {
+                return "translate("+x(sm.text_x)+","+y(sm.text_timestamp)+")";
+            })
+        ;
+
+
+        chart.selectAll("g.wait")
+            .attr("transform", ev => {
+                return "translate("+x(ev.x)+","+y(ev.start)+")";
+            });
+        chart.selectAll("rect.wait")
+            .attr("height", ev => y(ev.stop)-y(ev.start));
+
+        chart.selectAll("g.wait-text")
+            .attr("transform", ev => {
+                return "translate("+x(ev.x)+","+y((ev.start+ev.stop)/2)+")";
+            });
+
+        chart.selectAll("g.proc-event")
+            .attr("transform", ev => {
+                return "translate("+x(ev.x)+","+y(ev.start)+")";
+            })
+        ;
+
+        chart.selectAll("g.proc-span")
+            .attr("transform", ev => {
+                return "translate("+x(ev.x)+","+y(ev.start)+")";
+            })
+        ;
+        chart.selectAll("rect.proc-span")
+            .attr("height", ev => y(ev.stop)-y(ev.start));
+        chart.selectAll("g.proc-span-text")
+            .attr("transform", ev => {
+                return "translate("+x(ev.x)+","+y((ev.start+ev.stop)/2)+")";
+            });
+
+
+        // vertical scale markers: horizontal tick lines at rational
+        // timestamps
+
+        // TODO: clicking on a dot should set the new zero time
+        var rules = chart.selectAll("g.rule")
+                .data(y.ticks(10))
+                .attr("transform", ty);
+        rules.select("text")
+            .text(t => y.tickFormat(10, "s")(t)+"s");
+        var newrules = rules.enter().insert("svg:g")
+                .attr("class", "rule")
+                .attr("transform", ty)
+        ;
+        newrules.append("svg:line")
+            .attr("class", "rule-tick")
+            .attr("stroke", "black");
+        chart.selectAll("line.rule-tick")
+            .attr("x1", x(0.5)-5)
+            .attr("x2", x(0.5));
+        newrules.append("svg:line")
+            .attr("class", "rule-red")
+            .attr("stroke", "red")
+            .attr("stroke-opacity", .3);
+        chart.selectAll("line.rule-red")
+            .attr("x1", x(0.5))
+            .attr("x2", x(MAX_COLUMN));
+        newrules.append("svg:text")
+            .attr("class", "rule-text")
+            .attr("dx", ".1em")
+            .attr("dy", "-0.2em")
+            .attr("text-anchor", "start")
+            .attr("fill", "black")
+            .text(t => y.tickFormat(10, "s")(t)+"s");
+        chart.selectAll("text.rule-text")
+            .attr("x", 6 + 9);
+        rules.exit().remove();
+    }
+
+
+    redraw();
+    return;
 
-    var w = Number(container.style("width").slice(0,-2));
 
     function y_off(d) {
         return (LANE_HEIGHT * (d.side*(data.lanes.length+1) + d.lane)
@@ -43,20 +712,6 @@ $.getJSON("data.json", function(d) {
     }
     var bottom_rule_y = LANE_HEIGHT * data.sides.length * (data.lanes.length+1);
     var bottom_y = bottom_rule_y + 45;
-
-    var tip = d3.tip()
-            .attr("class", "d3-tip")
-            .html(function(d) { return "<span>" + d.details_str + "</span>"; })
-            .direction("s")
-    ;
-
-    var chart = container.append("svg:svg")
-            .attr("id", "outer_chart")
-            .attr("width", w)
-            .attr("pointer-events", "all")
-            .call(zoom)
-            .call(tip)
-    ;
     //var chart_g = chart.append("svg:g");
 
     // this "backboard" rect lets us catch mouse events anywhere in the
@@ -154,7 +809,7 @@ $.getJSON("data.json", function(d) {
         return duration;
     }
 
-    function redraw() {
+    function oldredraw() {
         // at this point zoom/pan must be fixed
         var min = data.bounds.min + x.domain()[0];
         var max = data.bounds.min + x.domain()[1];
@@ -279,10 +934,28 @@ $.getJSON("data.json", function(d) {
 
         // lines: these represent the time at which the server sent a message
         // which finished a bar. These get an SVG group, and a line
-        var lines = chart.selectAll("g.lines")
+        var linedata = clipped.lines.map(d => [
-                .data(clipped.lines, (d) => d.start_time)
+            [d.server_sent, 0],
+            [d.server_sent, LANE_HEIGHT],
+            [d.finish_time, 0],
+        ]);
+
+        function lineshape(d) {
+            var l = d3.svg.line()
+                    .x(d => x(d[0]))
+                    .y(d => y_off(d) + 12345);
+        }
+        function update_line(sel) {
+            sel.attr("d", lineshape)
+                .attr("class", d => "line lane-"+d.lane)
+            ;
+        }
+
+        var lines = chart.selectAll("polyline.lines")
+                .data(linedata)
+
                 .attr("transform",
-                      (d) => "translate("+left_server(d)+","+y_off(d)+")")
+                      (d) => "translate("+left(d)+","+y_off(d)+")")
         ;
         lines.exit().remove();
         var new_lines = lines.enter()
@@ -292,7 +965,17 @@ $.getJSON("data.json", function(d) {
                       (d) => "translate("+left_server(d)+","+(y_off(d))+")")
         ;
         new_lines.append("svg:line")
-            .attr("x1", 0).attr("y1", -5).attr("x2", "0").attr("y2", LANE_HEIGHT)
+            .attr("x1", 0)
+            .attr("y1", -5)
+            .attr("x2", "0")
+            .attr("y2", LANE_HEIGHT)
+            .attr("class", (d) => "line lane-"+d.lane)
+            .attr("stroke", "red")
+        ;
+        new_lines.append("svg:line")
+            .attr("x1", 0).attr("y1", -5)
+            .attr("x2", (d) => x(d.finish_time - d.server_sent))
+            .attr("y2", 0)
             .attr("class", (d) => "line lane-"+d.lane)
             .attr("stroke", "red")
         ;
@@ -354,3 +1037,17 @@ $.getJSON("data.json", function(d) {
     redraw();
     $.get("done", function(_) {});
 });
+
+/*
+TODO
+
+* identify the largest gaps in the timeline (biggest is probably waiting for
+  the recipient to start the program, followed by waiting for recipient to
+  type in code, followed by waiting for recipient to approve transfer, with
+  the time of actual transfer being anywhere among the others).
+* identify groups of events that are separated by those gaps
+* put a [1 2 3 4 all] set of buttons at the top of the page
+* clicking on each button will zoom the display to 10% beyond the span of
+  events in the given group, or reset the zoom to include all events
+
+*/