#!/usr/bin/env python3 """The frame on the 4K stage: four columns of 880, the context at mono 20, four times what 1080 held. python3 -B dense.py # writes 15-marks.svg, 16-none.svg, 17-subject.svg, 18-loupes.svg The blocks are context.py's, placed on the 4K stage's columns; what is new here is the stage, the thread across the gutter, the `loupe`, and the marks sheet. The moment is real: a fresh recording of the same `targets/list`, `ztrace dbg --stack 1024` with a breakpoint at `reverse+22` (the load at line 21), its second stop and the instruction after (`list-dbg.txt`, kept beside this file), and the program's disassembly with its lines (`objdump -dl`, `list.dis`). """ import re import gen gen.W, gen.H = 3840, 2160 # the Svg's canvas: the 4K stage, in units import context as C from context import (PAGE, RAISED, VELLUM, BRIGHT, DIM, FAINT, INK, GOLD, VERM, READ, HEAP, STACK, A, S, ROW, TAG, PAD, MONO, BOLD, b, ph, tw, mix, bracket, chain, Svg, S_, H_, C_, D_, P) W4, H4 = 3840, 2160 COL = [112, 1024, 1936, 2848] # four columns of 880, gutters of 32 CW = 880 TOP4, BOT4 = 96, 1784 # the stage: 1688, a single panel of 69 rows C.W = W4 C.TOP, C.BOTTOM = TOP4, BOT4 C.LANE_LINK, C.LANE_VIEW = 1912, 1924 # the gutter between the program's columns and the state's THREAD_X = 1008 # the gutter between the source and the disassembly LIFT, LIFT_TAG, SHADE = "#121c52", "#22317a", "#02061a" # a loupe's surface, its tag, its shadow def put(x): """Place the next block in the column at x.""" C.LX = C.RX = x # ---- the recording ---------------------------------------------------------------------------------- DBG = open(__file__.replace("dense.py", "list-dbg.txt")).read() MEM = {} for line in DBG.split("\n"): m = re.match(r"^([0-9a-f]{12}) ((?:[0-9a-f?]{2} ){16})", line) if m and "??" not in m.group(2): for i, v in enumerate(m.group(2).split()): MEM[int(m.group(1), 16) + i] = int(v, 16) NAMES = {0x1001730: "_start", 0x1001830: "list_done", 0x1001860: "main", 0x10018e0: "push", 0x1001910: "reverse", 0x1001938: "_init", 0x1001954: "_fini"} LIBNAMES = {0x7ffff7c27781: "libc+27781", 0x7ffff7c278b9: "__libc_start_main+137", 0x7ffff7fe0c60: "ld.so+22c60", 0x7ffff7fbc000: "vdso+0"} def region(v): """The ink of what v points into, by the run's mappings (None: no mapping).""" for lo, hi, ink in [(0x7ffffffde000, 0x7ffffffff000, S_), (0x1004000, 0x1025000, H_), (0x1001000, 0x1002000, C_), (0x1000000, 0x1001000, VELLUM), (0x1002000, 0x1003000, VELLUM), (0x1003000, 0x1004000, D_), (0x7ffff7c24000, 0x7ffff7d9f000, C_), (0x7ffff7c00000, 0x7ffff7c24000, VELLUM), (0x7ffff7e15000, 0x7ffff7e23000, D_), (0x7ffff7fbc000, 0x7ffff7fbe000, C_), (0x7ffff7fbf000, 0x7ffff7fec000, C_), (0x7ffff7ffb000, 0x7ffff7fff000, D_)]: if lo <= v < hi: return ink return None def name(v): if v in LIBNAMES: return LIBNAMES[v] f = max((a for a in NAMES if a <= v), default=None) return "%s+%d" % (NAMES[f], v - f) if f is not None else "%x" % v def word(a): if all(a + i in MEM for i in range(8)): return sum(MEM[a + i] << (8 * i) for i in range(8)) return None def text(a): """The C string at a, if there is one of two characters or more.""" out = [] while a + len(out) in MEM and MEM[a + len(out)] and 32 <= MEM[a + len(out)] < 127: out.append(chr(MEM[a + len(out)])) return "".join(out) if len(out) >= 2 and MEM.get(a + len(out)) == 0 else None def follow(v, depth=5): """A chain from the address v, as far as the recording holds the memory: (links, end) for context.chain.""" links = [] while True: ink = region(v) if ink == C_: return links + [(v, C_, name(v))], None links.append((v, ink)) s = text(v) if s: return links, ("str", s) w = word(v) if w is None or len(links) > depth: return links, None if w and region(w): v = w continue return links, ("int", w) def value(v): return follow(v) if v and region(v) else ([], ("int", v)) RSP = 0x7fffffffece0 FRAMES = [("reverse", 0, 1), ("main", 2, 5), ("libc+27781", 6, 27), ("__libc_start_main", 28, 39), ("_start", 40, 99)] # What a slot is, where the recording can know it: a saved register (the frame-pointer chain; the prologue's pushes, # from the call frame information), the process's start (argc, argv, envp, auxv, from the first stack pointer). A # return address needs no word: its chain is code, named, and its frame's bracket ends on it. ROLES = {0: "saved rbp", 3: "saved rbx", 4: "saved rbp", 26: "saved rbp", 38: "saved rbp", 42: "argc", 43: "argv[0]", 44: "argv end", 45: "envp end", 46: "auxv"} def stack_rows(n, rbp_rsp=True): rows = [] for i in range(n): a = RSP + 8 * i links, end = value(word(a)) rows.append((a, (["rbp", "rsp"] if i == 0 and rbp_rsp else []), links, end)) return rows def roles(n): return {i: (t, DIM) for i, t in ROLES.items() if i < n} def regs(step): rdi = 0x1004070 if step else 0x1004090 out = [] for r, v in [("rax", 0x1004090), ("rbx", 0), ("rcx", 0x10040b0), ("rdx", 0), ("rsi", 0x10040c0), ("rdi", rdi), ("rbp", RSP), ("rsp", RSP), ("r8", 0x7ffff7e19ac0), ("r9", 0x20), ("r10", 0x10), ("r11", 0x7ffff7e19ac8), ("r12", 0x7fffffffee38), ("r13", 1), ("r14", 0x7ffff7ffd000), ("r15", 0x10029c0)]: links, end = value(v) if r == "r14": # ld.so's data: the recording followed it (ztrace's own telescope) links, end = [(0x7ffff7ffd000, D_), (0x7ffff7ffe3c0, D_)], ("int", 0) if r == "r15": links, end = [(0x10029c0, VELLUM)], ("int", 1) out.append((r, links, end)) out.append(("rip", [(0x1001926, C_, "reverse+22")], None)) return out FLAGS = [("CF", 0, 0), ("PF", 1, 0), ("AF", 0, 0), ("ZF", 0, 0), ("SF", 0, 0), ("IF", 1, 0), ("DF", 0, 0), ("OF", 0, 0)] BT = [("reverse+22", "", "list.c", 21), ("main+80", "", "list.c", 40), ("libc+27781", "", None, 0), ("__libc_start_main+137", "", None, 0), ("_start+37", "", None, 0)] NODES = [(0x10040b0, 1, 0), (0x1004090, 4, 0x1004070), (0x1004070, 9, 0x1004050), (0x1004050, 16, 0x1004030), (0x1004030, 25, 0x1004010), (0x1004010, 36, 0)] SRC = C.SRC[:42] + [""] * 40 def heap_fields(base, length, read_at=None): """The heap's words by what the program and malloc made of them: each node a value and a next, each chunk's size word malloc's, the top chunk's size word, and the rest loose. The word at read_at carries mark 9.""" out = [] for a, _, _ in NODES: out += [(a - base, 8, "number", False, None), (a + 8 - base, 8, "pointer", False, 9 if a + 8 == read_at else None), (a - 8 - base, 8, "malloc", False, None)] out.append((0x10040c8 - base, 8, "malloc", False, None)) return [f for f in out if 0 <= f[0] < length] def conv(mn, ops): """objdump's Intel syntax as the recording's disassembler writes it.""" if mn.startswith("data16") or mn == "cs": return "nop", "word ptr cs:[rax + rax]" ops = ops.lower().replace(",", ", ") ops = re.sub(r"\[(.*?)\]", lambda m: "[" + re.sub(r"([+*-])", r" \1 ", m.group(1)).replace(" * ", "*") + "]", ops) ops = ops.replace("*1 + 0x0", "").replace(" + 0x0]", "]") def num(m): v = int(m.group(1), 16) return str(v) if v < 10 else "0x%x" % v ops = re.sub(r"rip \+ 0x(f{8}[0-9a-f]+)", lambda m: "rip - 0x%x" % ((1 << 64) - int(m.group(1), 16)), ops) ops = re.sub(r"(?:", l) if m: func = m.group(2) continue m = re.match(r"^targets/list\.c:(\d+)", l) if m: line = int(m.group(1)) continue m = re.match(r"^\s+([0-9a-f]+):\s+(\S+)\s*(.*)$", l) if m: ops = re.sub(r"\s*<[^>]+>", "", re.sub(r"\s+#.*", "", m.group(3))).strip() mn, ops = conv(m.group(2), ops) own = func in ("main", "push", "reverse", "list_done") rows.append((int(m.group(1), 16), func, line if own else None, mn, ops)) return rows CALLEES = {0x1001990: "printf", 0x10019a0: "malloc", 0x1001980: "strtol"} def window(n, step): """n instructions round the pc: the function's and its neighbours', each neighbour under a rule and its name.""" rows = disasm_rows() i = [r[0] for r in rows].index(0x1001926) lo = max(0, min(i - n // 2, len(rows) - n)) out, last_func = [], None for a, func, line, mn, ops in rows[lo:lo + n]: target = re.fullmatch(r"([0-9a-f]{6,})", ops) parts = [(int(ops, 16), C_)] if target else ([(ops, P)] if ops else []) st = "cur" if a == 0x1001926 else None note = None if func != "reverse": st = "other_first" if func != last_func else "other" if func != last_func: note = ("text", func, DIM) if mn == "call" and target and int(ops, 16) in CALLEES: note = ("text", CALLEES[int(ops, 16)], VELLUM) if a == 0x1001926: note = ("chain", [(0x1004098, H_), (0x1004070, H_)], ("int", 9), {0: "read"} if step else None) out.append((a, line, mn, parts, st, note)) last_func = func return out # ---- the stage's blocks ------------------------------------------------------------------------------- def thread_across(g, ys, spans): """The thread on the 4K stage: the source and the disassembly side by side, the gutter between them its channel. A bracket on the line at the source's right edge, one on each run at the disassembly's left, a line down the gutter between.""" x0, x1 = COL[0] + CW + 6, COL[1] - 6 bracket(g, x0, ys + 2, ys + ROW - 2, GOLD, 4, 7, side=-1) tops = [ys + 12] for ya, yb in spans: bracket(g, x1, ya + 2, yb - 2, GOLD, 4, 7) tops.append((ya + yb) / 2) g.rect(THREAD_X, (ya + yb) / 2 - 2, x1 - THREAD_X, 4, GOLD) g.rect(x0, ys + 10, THREAD_X - x0, 4, GOLD) g.rect(THREAD_X - 2, min(tops) - 2, 4, max(tops) - min(tops) + 4, GOLD) def clock4(g, moment, total): g.rect(COL[0], 1798, 3616, 4, C.RULE) g.rect(COL[0], 1796, 3616 * moment / total, 8, GOLD) g.t(COL[3] + CW, 1852, "moment %d of %d" % (moment, total), 40, FAINT, "mono", "end") def caption4(g, lines): """The caption, twice the 1080 frame's: Go 80 on 108, code mono 70; so it reads the same at 1080p.""" y = 1824 + (104 if len(lines) == 2 else 156) for parts in lines: g.rich(COL[0], y, [(p[0], VELLUM) + tuple(p[1:]) for p in parts], 80) y += 108 def loupe(g, x, y, title, note, base, flist, framed, depth): """`loupe`: the memory round a pointer's target, floating over the context, tied to the pointer it opens from. Nine rows, the target's in the middle; the target framed bright. Lifted: its own surface, a shadow under it, deeper for a loupe opened from another. Returns its rows' y.""" h = ph(9) sx, sy = (12, 16) if depth == 1 else (20, 26) g.rect(x + sx, y + sy, CW, h, SHADE) g.rect(x, y, CW, h, LIFT) g.rect(x, y, CW, TAG, LIFT_TAG) g.frame(x, y, CW, h, DIM, 2) xx = x + 12 for s_, ink in title: g.t(xx, y + 18, s_, S, ink, MONO) xx += len(s_) * A g.t(x + CW - 10, y + 18, note, 18, DIM, "sans", "end") return C.fields(g, y, h, "", base, 144, flist, MEM, HEAP, region, framed=framed, bare=True, x=x) def stem(g, pts): """The tie from a pointer to its loupe: a link, in the ink of what it points into, head on the loupe.""" gen.arrow(g, pts, HEAP, 4, r=8, hl=12, hw=12) # ---- the frames ---------------------------------------------------------------------------------------- def frame_none(name, step, loupes=False): """`none` on the 4K stage: source | disassembly | registers, backtrace, memory | stack.""" g = Svg() put(COL[0]) ys = C.source(g, TOP4, 69, SRC, 1, 21, [], "list.c", 42) put(COL[1]) at = C.disasm(g, TOP4, window(69, step), "reverse", 21) thread_across(g, ys, [(at[0x1001926], at[0x1001926] + ROW)]) put(COL[2]) marks = {"rax": {0: "tick"}, "rcx": {0: "tick"}, "rdi": {0: "strike" if step else "tick"}} where = C.registers(g, TOP4, regs(step), FLAGS, marks) C.backtrace(g, 548, 416, BT, x=COL[2], note="newest on top") C.fields(g, 976, 808, "the heap, round what the step reads", 0x1004000, 512, heap_fields(0x1004000, 512, 0x1004098), MEM, HEAP, region, marks=(9,) if step else (), title="memory") put(COL[3]) C.stack(g, TOP4, stack_rows(69), FRAMES, note="69 words from rsp", rsp=RSP, roles=roles(69)) if loupes: ry = where["rdi"] x1, y1 = 2728, ry r1 = loupe(g, x1, y1, [("rdi", DIM), (" → ", FAINT), ("1004070", HEAP)], "heap · struct node, 16 bytes", 0x1004030, heap_fields(0x1004030, 144), [(0x1004070 - 0x1004030, 16)], 1) stem(g, [(COL[2] + 72 + 7 * A + 30 + A + 16, ry + 12), (x1, ry + 12)]) # from the end of rdi's chain # the next word in it opens the next: the field's bytes bracketed, a stem down into the deeper loupe fy = r1[0x1004070] bx0 = x1 + 132 + 8 * 30 + 12 - 4 bx1 = x1 + 132 + 15 * 30 + 12 + 28 g.rect(bx0, fy + ROW + 1, bx1 - bx0, 3, HEAP) x2, y2 = x1 + 120, fy + ROW + 34 loupe(g, x2, y2, [("1004078", HEAP), (" → ", FAINT), ("1004050", HEAP)], "heap · struct node, 16 bytes", 0x1004010, heap_fields(0x1004010, 144), [(0x1004050 - 0x1004010, 16)], 2) stem(g, [((bx0 + bx1) / 2, fy + ROW + 4), ((bx0 + bx1) / 2, y2)]) clock4(g, 3 if step else 2, 4) if loupes: caption4(g, [[("The load is done: ",), ("rdi", "mono"), (" holds ",), ("1004070", "mono"), (", and beside it the heap round it opens.",)], [("Its ",), ("next", "mono"), (" word opens the node after it, one deeper.",)]]) else: caption4(g, [[("Stopped in ",), ("reverse", "mono"), (", about to load ",), ("head->next", "mono"), (".",)], [("rdi", "mono"), (" points at a node; the word after its value is the next node.",)]]) g.save(name) def frame_subject(name): """`subject` on the 4K stage, mid-step: the list drawn twice the size over the program's two columns, its bytes over the third, the context under and beside them.""" g = Svg() band = 604 put(COL[2]) view = C.fields(g, TOP4, 800, "the heap, 1004000 to 10041ff", 0x1004000, 512, heap_fields(0x1004000, 512, 0x1004098), MEM, HEAP, region, marks=(9,)) where = C.registers(g, 908, regs(True), FLAGS, {"rax": {0: "tick"}, "rcx": {0: "tick"}, "rdi": {0: "strike"}}) C.backtrace(g, 1360, 424, BT, x=COL[2], note="newest on top") put(COL[0]) pins = {0x10040b0: ["rcx"], 0x1004090: ["rax"], 0x1004070: ["rdi"]} C.RX = COL[2] # the link leaves the registers, the projection the view C.lens(g, TOP4, band, 800, NODES, pins, {"tick": [0x10040b8], "read": 0x1004098}, 0x1004000, 32, link_from=(where["rdi"], "rdi"), subject=True, dw=2 * CW + 32, k=2, view_rows=view) put(COL[0]) ys = C.source(g, TOP4 + band + 12, 43, SRC, 1, 21, [], "list.c", 42) put(COL[1]) at = C.disasm(g, TOP4 + band + 12, window(43, True), "reverse", 21) thread_across(g, ys, [(at[0x1001926], at[0x1001926] + ROW)]) put(COL[3]) C.stack(g, TOP4, stack_rows(69), FRAMES, note="69 words from rsp", rsp=RSP, roles=roles(69)) clock4(g, 2, 4) caption4(g, [[("The load read the word at ",), ("1004098", "mono"), (": the address of the next node.",)], [("rdi", "mono"), (" holds it now, and points at the node whose value is 9.",)]]) g.save(name) def frame_marks(name): """Mock 15: current, struck and held, as the first videos drew them (A), as the catalogue now does (B), and the other reading of "white, with less opacity" (C), on real rows of the same moment and the one after.""" g = Svg() xs = [112, 1376, 2640] titles = [("A", "the first videos: gold band, solid vermilion, a tick"), ("B", "recommended: a white wash, a softened strike, a turned corner"), ("C", "white: a white plaque, a white strike framed in vermilion, a bar under")] g.t(112, 120, "current, struck, held", 40, BRIGHT, "sansm") g.t(112, 164, "Rows of targets/list at the load in reverse (moment 3) and the store after it (moment 4), at the " "4K stage's density: mono 20, rows of 24. Look at it at full size.", 24, DIM, "sans") for (key, label), x in zip(titles, xs): C.STYLE.update(current=key, struck=key, held=key) g.t(x, 244, key, 56, GOLD if key == "B" else BRIGHT, "sansb") g.t(x + 56, 236, label, 22, DIM, "sans") put(x) y = 280 ys = C.source(g, y, 6, SRC, 18, 21, [20], "list.c", 42) y += ph(6) + 24 rows = [r for r in window(69, True) if 0x1001919 <= r[0] <= 0x100192e] rows = [(a, ln, mn, ops, "trail0" if a == 0x1001923 else st, note) for a, ln, mn, ops, st, note in rows] C.disasm(g, y, rows, "reverse", 21) y += ph(len(rows)) + 24 reg = [r for r in regs(True) if r[0] in ("rax", "rcx", "rsi", "rdi", "rbp")] + [regs(True)[-1]] y0 = C.panel(g, x, y, CW, len(reg), "registers") for i, (r, links, end) in enumerate(reg): yy = y0 + i * ROW if r == "rip": g.t(x + 12, b(yy), r, S, C.current_row(g, x + 2, yy, CW - 4, 54), BOLD) else: g.t(x + 12, b(yy), r, S, DIM, MONO) m = {"rax": {0: "tick"}, "rcx": {0: "tick"}, "rdi": {0: "strike"}}.get(r) chain(g, x + 72, yy, links, end, m, room=x + CW - 12) y += ph(len(reg)) + 24 mem4 = dict(MEM) # moment 4: the store wrote 10040b0 into the word at 1004098 for i, v in enumerate((0x10040b0).to_bytes(8, "little")): mem4[0x1004098 + i] = v flist = heap_fields(0x1004080, 64) flist = [(a, n, r, bg, 7 if a == 0x18 else m) for a, n, r, bg, m in flist] yr = C.fields(g, y, ph(4), "moment 4: the store, struck; 10040b8, held", 0x1004080, 64, flist, mem4, HEAP, region, marks=()) # struck on the run the store wrote, held on the one iteration 1 wrote (its own row, 10040b0's next) r_y = yr[0x1004090] x0 = x + 132 + 8 * 30 + 12 - 4 ink = C.struck(g, x0, r_y, 30) # only the byte that changed: 70 to b0; 40 00 01 kept their value g.t(x + 132 + 8 * 30 + 12, b(r_y), "b0", S, ink, BOLD) C.held(g, x + 132 + 8 * 30 + 12, yr[0x10040b0], 8 * 30 - 6) y += ph(4) + 36 # a lens's cells, at the drawing's scale on this stage (2): a node held, a node struck g.t(x, y, "in a lens's cell, twice the size", 20, DIM, "sansm") y += 16 for j, (lab, mark) in enumerate([("4", "held"), ("4", "struck")]): cx = x + j * 300 g.rect(cx, y, 152, 88, RAISED) g.rect(cx + 152, y, 96, 88, C.WELL) if mark == "struck": ink = C.struck(g, cx, y - 1, 152, 90) else: ink = VELLUM C.held_cell(g, cx + 152, y, 96, 88, 2) g.t(cx + 76, y + 44 + 18, lab, 50, ink, "mono", "middle") g.rect(cx + 190, y + 34, 20, 20, HEAP) g.t(cx + 124, y + 118, mark, 20, DIM, "sansm", "middle") # the same rows, closer: 1.4 times, for judging on a screen smaller than 4K g.t(x, 1236, "the same, 1.35 times", 20, DIM, "sansm") g.o.append('' % (x, 1256)) put(0) C.source(g, 0, 3, SRC, 19, 21, [20], "list.c", 42) y0 = C.panel(g, 0, ph(3) + 16, CW, 4, "registers") for i, r in enumerate([r for r in regs(True) if r[0] in ("rax", "rdi")] + [regs(True)[-1]]): yy = y0 + i * ROW nm, links, end = r if nm == "rip": g.t(12, b(yy), nm, S, C.current_row(g, 2, yy, CW - 4, 54), BOLD) else: g.t(12, b(yy), nm, S, DIM, MONO) chain(g, 72, yy, links, end, {"rax": {0: "tick"}, "rdi": {0: "strike"}}.get(nm), room=CW - 12) yy = y0 + 3 * ROW g.t(12, b(yy), "rcx", S, DIM, MONO) chain(g, 72, yy, [(0x10040b0, H_)], ("int", 1), {0: "tick"}, room=CW - 12) g.o.append("") put(x) C.STYLE.update(current="B", struck="B", held="B") g.save(name) if __name__ == "__main__": frame_marks("15-marks") frame_none("16-none", False) frame_subject("17-subject") frame_none("18-loupes", True, loupes=True)