//! pardes-replicas: one authority and several replicas of a session in one
//! process (src/replica.zig), each client typing at once over its own
//! delayed, jittered link, as monkey.zig's scripts type at a tty, each in a
//! view of its own (src/views.zig) over the text they share; `--shared`
//! puts them all in one, where they clash at every key: a stress of
//! correctness, not a picture of a session.
//!
//! Terminals cross as screen state (src/screen_state.zig): the session's
//! shell pane runs a full-screen program and then a `yes` flood (`--flood`
//! MB of it) while the clients type, its output stepped by the authority
//! and sent to each replica as deltas, once a frame and before any input
//! the authority steps, each replica acknowledging them over its link and
//! asking for the rows it scrolls back into and does not hold.
//!
//! zig build replicas -Dplatform=tui -- [--seeds N] [--steps M] [--clients C] [--rtt 30,80,207] [--first K] [--flood MB] [--shared]
//! zig build replicas -Dplatform=tui -- --echo [--seeds N] [--rtt 30,80,207] [--first K]
//! zig build replicas -Dplatform=tui -Doptimize=ReleaseFast -- --screen-state
//!
//! `--echo` types a corpus into a shell instead, with and without the
//! predicted echo (src/echo.zig), and gates what it draws: below, at
//! "predicted echo". `--screen-state` runs recorded pty streams through
//! screen_state's soak and measures its encoder: below, at screenState.
//!
//! Gates, per seed and RTT: no replica diverges (every confirmed unit's
//! digests agree as it is replayed), and at quiescence every replica's
//! digest and every file's text are the authority's; every delta a replica applies leaves its
//! terminal what the authority's was at that version, and at quiescence it
//! is the authority's, cell for cell. Reported per RTT: how inputs went
//! (applied at once, stopped at a host call, left to the authority), how
//! units were taken (fast path, rollback for a misprediction or another's
//! input), what a speculative apply, a copy and a rollback cost against a
//! frame, and the terminals': deltas a second, wire bytes a second under
//! the flood, encode a frame (every peer), apply a delta.
const std = @import("std");
const libc = std.c;
const pardes = @import("pardes");
const replica = pardes.replica;
const screen_state = pardes.screen_state;
const echo = pardes.echo;
const Key = pardes.Key;
const gpa = std.heap.c_allocator;
var trace = false;
const frame_us = 16_667;
/// The authority's host: no frames drawn (a replica draws its own), and a
/// pty that echoes what it is written, 2 ms later, as a shell's.
const Host = struct {
echo: std.ArrayListUnmanaged(struct { pane: u8, bytes: []u8 }) = .empty,
fn watched(_: ?*anyopaque) bool {
return false;
}
fn ptyWrite(ctx: ?*anyopaque, pane: u8, bytes: []const u8) void {
const h: *Host = @ptrCast(@alignCast(ctx.?));
h.echo.append(gpa, .{ .pane = pane, .bytes = gpa.dupe(u8, bytes) catch return }) catch {};
}
fn ignore(_: ?*anyopaque, _: []const u8) void {}
const vtable: pardes.Host.VTable = .{ .watched = watched, .pty_write = ptyWrite, .set_clipboard = ignore };
fn host(h: *Host) pardes.Host {
return .{ .ctx = h, .vtable = &vtable };
}
};
const Kind = enum { input, to_authority, to_replica, echo, frame, output, screen_down, screen_up, client_frame };
const Msg = struct {
at: u64,
n: u64,
kind: Kind,
who: u16,
bytes: []u8,
/// An echo's: the client's input that made it (--echo).
seq: u32 = 0,
fn order(_: void, a: Msg, b: Msg) std.math.Order {
if (a.at != b.at) return std.math.order(a.at, b.at);
return std.math.order(a.n, b.n);
}
};
const Link = struct {
last: u64 = 0,
/// When a message sent at `now` arrives: half the RTT, jitter on top,
/// and never before the one sent ahead of it (a stream keeps order).
fn arrive(l: *Link, random: std.Random, now: u64, one_way: u64, jitter: u64) u64 {
const at = @max(l.last + 1, now + one_way + random.uintLessThan(u64, jitter + 1));
l.last = at;
return at;
}
};
const Totals = struct {
inputs: u64 = 0,
speculated: u64 = 0,
stopped: u64 = 0,
unspeculated: u64 = 0,
units: u64 = 0,
fast: u64 = 0,
mispredicted: u64 = 0,
foreign: u64 = 0,
rebases: u64 = 0,
copies: u64 = 0,
copy_bytes: u64 = 0,
divergences: u64 = 0,
unequal: u64 = 0,
spec_ns: std.ArrayListUnmanaged(u64) = .empty,
copy_ns: std.ArrayListUnmanaged(u64) = .empty,
rollback_ns: std.ArrayListUnmanaged(u64) = .empty,
// Terminals.
apply_ns: std.ArrayListUnmanaged(u64) = .empty,
frame_ns: std.ArrayListUnmanaged(u64) = .empty,
deltas: u64 = 0,
delta_bytes: u64 = 0,
unit_bytes: u64 = 0,
/// Deltas the replicas applied while the flood ran.
flood_deltas: u64 = 0,
flood_us: u64 = 0,
flood_delta_bytes: u64 = 0,
flood_unit_bytes: u64 = 0,
fetched: u64 = 0,
/// Bytes of rows sent before a step's unit, the step having read them.
filled_bytes: u64 = 0,
resets: u64 = 0,
screens_wrong: u64 = 0,
screens_checked: u64 = 0,
fn add(t: *Totals, s: *const replica.Stats) !void {
t.inputs += s.inputs;
t.speculated += s.speculated;
t.stopped += s.stopped;
t.unspeculated += s.unspeculated;
t.units += s.units;
t.fast += s.fast;
t.mispredicted += s.mispredicted;
t.foreign += s.foreign;
t.rebases += s.rebases;
t.copies += s.copies;
t.copy_bytes += s.copy_bytes;
try t.spec_ns.appendSlice(gpa, s.spec_ns.items);
try t.copy_ns.appendSlice(gpa, s.copy_ns.items);
try t.rollback_ns.appendSlice(gpa, s.rollback_ns.items);
try t.apply_ns.appendSlice(gpa, s.apply_ns.items);
}
};
/// One client's next input: monkey.zig's mix, as events, and now and then
/// a wheel over the shell pane, which scrolls its history into view; a
/// file looked at (`looks`, its words) and a pane closed, the Panes and
/// Recent pickers (the store's and Recent's rows, the authority's), a jump.
fn nextEvent(random: std.Random, buf: *[64]u8, wheel_at: ?[2]u16, looks: []const []const u8) pardes.Event {
const typed = "abcdefghijklmnopqrstuvwxyz0123456789 -_=+,:;()";
const normal = "hjklwbe0$xpuyoOiaAIdcv.nN%sC";
switch (random.uintLessThan(u8, 43)) {
0...15 => {
buf[0] = typed[random.uintLessThan(usize, typed.len)];
return .{ .key = .{ .cp = buf[0], .text = buf[0..1] } };
},
16...27 => {
buf[0] = normal[random.uintLessThan(usize, normal.len)];
return .{ .key = .{ .cp = buf[0], .text = buf[0..1] } };
},
28...31 => return .{ .key = .{ .cp = Key.escape } },
32 => return .{ .key = .{ .cp = Key.enter } },
33 => return .{ .key = .{ .cp = Key.backspace } },
34 => return .{ .key = .{ .cp = switch (random.uintLessThan(u8, 4)) {
0 => Key.up,
1 => Key.down,
2 => Key.left,
else => Key.right,
} } },
35 => {
const n = random.intRangeAtMost(usize, 1, 24);
for (buf[0..n]) |*c| c.* = typed[random.uintLessThan(usize, typed.len)];
return .{ .paste = buf[0..n] };
},
36 => return .{ .command = switch (random.uintLessThan(u8, 10)) {
0 => "Newcol",
1 => "Zerox",
2 => "Undo",
3 => "Redo",
4 => "Search Panes",
5 => "Search Recent",
6, 7 => looks[random.uintLessThan(usize, looks.len)],
8 => "Del",
else => return .{ .key = .{ .cp = if (random.boolean()) 'o' else 'i', .ctrl = true } },
} },
37 => return .{ .resize = .{ .cols = @intCast(random.intRangeAtMost(u16, 60, 120)), .rows = @intCast(random.intRangeAtMost(u16, 16, 40)) } },
38 => return .{ .mouse = .{ .button = .left, .kind = .press, .col = random.uintLessThan(u16, 80), .row = random.uintLessThan(u16, 24) } },
39 => return .{ .mouse = .{ .button = .left, .kind = .release, .col = random.uintLessThan(u16, 80), .row = random.uintLessThan(u16, 24) } },
// Out of a shell's tty mode and back: its text, a cursor and
// selections over it.
40 => return .{ .key = .{ .cp = 'b', .ctrl = true } },
else => {
const at = wheel_at orelse return .{ .key = .{ .cp = Key.escape } };
return .{ .mouse = .{ .button = if (random.uintLessThan(u8, 3) == 0) .wheel_down else .wheel_up, .kind = .press, .col = at[0], .row = at[1] } };
},
}
}
fn writeFile(path: [:0]const u8, lines: usize) !void {
const fd = libc.open(path, .{ .ACCMODE = .WRONLY, .CREAT = true, .TRUNC = true }, @as(libc.mode_t, 0o644));
if (fd < 0) return error.OpenFailed;
defer _ = libc.close(fd);
var line: [80]u8 = undefined;
for (0..lines) |n| {
const text = try std.fmt.bufPrint(&line, "const value_{d:0>4} = {d}; // a line to edit\n", .{ n, n *% 7919 });
if (libc.write(fd, text.ptr, text.len) != @as(isize, @intCast(text.len))) return error.WriteFailed;
}
}
const Run = struct { seed: u64, rtt_ms: u64, steps: usize, clients: u16, shared: bool, flood_mb: u64 };
/// The terminal pane's screen as a hash of screen_state's dump of it.
fn screenHash(t: *pardes.panes.terminal.VtSlot) u64 {
var buf: [8192]u8 = undefined;
var h: std.Io.Writer.Hashing(std.hash.Wyhash) = .initHasher(.init(0), &buf);
screen_state.dumpScreen(t, &h.writer) catch {};
h.writer.flush() catch {};
return h.hasher.final();
}
fn run(r: Run, dir: []const u8, totals: *Totals) !void {
if (trace) std.debug.print("seed {d} rtt {d} ms\n", .{ r.seed, r.rtt_ms });
var prng = std.Random.DefaultPrng.init(r.seed *% 0x9e3779b97f4a7c15 +% r.rtt_ms);
const random = prng.random();
var host: Host = .{};
defer {
for (host.echo.items) |e| gpa.free(e.bytes);
host.echo.deinit(gpa);
}
var paths: [3][256:0]u8 = undefined;
var look_lines: [3][300]u8 = undefined;
var looks: [3][]const u8 = undefined;
for (&paths, &look_lines, &looks, 0..) |*p, *l, *look, i| {
const path = try std.fmt.bufPrintSentinel(p, "{s}/f{d}.zig", .{ dir, i }, 0);
try writeFile(path, 60 + 40 * i);
look.* = try std.fmt.bufPrint(l, "Look {s}", .{path});
}
// The tty's own boot (a raw shell, a text pane under it), then three
// files looked at: keys go to the last, a pane's shell echoes.
const a = try replica.Authority.init(gpa, .{ .cols = 100, .rows = 30, .boot = .tty }, host.host(), true);
defer a.deinit();
var screens: replica.Screens = .init(gpa);
defer screens.deinit(a.core);
const first = try gpa.dupe(u8, try a.unit(0, 0));
defer gpa.free(first);
var reps: [8]*replica.Replica = undefined;
var peers: [8]u16 = undefined;
for (reps[0..r.clients], peers[0..r.clients], 1..) |*rp, *peer, id| {
rp.* = try replica.Replica.init(gpa, @intCast(id), first, true);
peer.* = try screens.addPeer();
}
defer for (reps[0..r.clients]) |rp| rp.deinit();
const shell: u8 = for (a.core.panes, 0..) |slot, id| {
if (slot) |pane| if (pane.isTerminal()) break @intCast(id);
} else return error.NoShell;
for (&paths) |*path| {
var line: [300]u8 = undefined;
const unit = try a.local(.{ .command = try std.fmt.bufPrint(&line, "Look {s}", .{std.mem.sliceTo(path, 0)}) });
for (reps[0..r.clients]) |rp| rp.confirm(unit);
}
// Each client a view of its own (src/views.zig), the first the
// session's; or, --shared, every client in the one.
if (!r.shared) for (reps[1..r.clients], 2..) |rp, id| {
const opened = try a.openView(@intCast(id));
rp.view = opened.view;
for (reps[0..r.clients]) |each| each.confirm(opened.unit);
};
// Where a wheel over the shell lands in the session's view (the first
// client's); the others' views put it elsewhere, and their wheels hit
// whatever is there.
const sr = a.core.rects[shell];
const wheel_at: ?[2]u16 = if (sr.h > 2) .{ sr.x + sr.w / 2, sr.y + sr.h / 2 } else null;
// What the shell's program prints: a full-screen program, a shell's
// session (prompts, titles, directories, a clear now and then), then a
// flood of `yes`, read 4 KB at a time as a pty gives it.
const tui = try screen_state.tuiStream(gpa);
defer gpa.free(tui);
const session = try screen_state.shellStream(gpa);
defer gpa.free(session);
const yes = try screen_state.yesStream(gpa, @intCast(r.flood_mb * (1 << 20) / 3));
defer gpa.free(yes);
const tui_at: u64 = 400_000;
const session_at: u64 = 1_000_000;
const flood_at: u64 = 2_500_000;
var flood_end: u64 = 0;
var program: struct { bytes: []const u8 = &.{}, at: usize = 0 } = .{};
// The authority's version of each terminal, as its screen showed then.
var versions: std.AutoHashMapUnmanaged(u64, u64) = .empty;
defer versions.deinit(gpa);
var flood_deltas: [8]u64 = @splat(0);
var queue: std.PriorityQueue(Msg, void, Msg.order) = .empty;
defer {
while (queue.pop()) |m| if (m.bytes.len > 0) gpa.free(m.bytes);
queue.deinit(gpa);
}
var n: u64 = 0;
const one_way = r.rtt_ms * 1000 / 2;
const jitter = one_way / 10;
var up: [8]Link = @splat(.{});
var down: [8]Link = @splat(.{});
var left: [8]usize = @splat(r.steps);
var typing: usize = r.clients;
// Each client's first input, soon; the programs; the first frame.
for (0..r.clients) |c| try push(&queue, &n, .{ .at = random.uintLessThan(u64, 200_000), .n = 0, .kind = .input, .who = @intCast(c + 1), .bytes = &.{} });
try push(&queue, &n, .{ .at = tui_at, .n = 0, .kind = .output, .who = 0, .bytes = &.{} });
try push(&queue, &n, .{ .at = session_at, .n = 0, .kind = .output, .who = 2, .bytes = &.{} });
try push(&queue, &n, .{ .at = flood_at, .n = 0, .kind = .output, .who = 1, .bytes = &.{} });
try push(&queue, &n, .{ .at = frame_us, .n = 0, .kind = .frame, .who = 0, .bytes = &.{} });
var frames_left: u32 = 0;
var flood_units: u64 = 0;
// What each peer is owed of the terminals, onto its link: before the
// authority steps an input, and once a frame.
var owed: std.ArrayListUnmanaged([]u8) = .empty;
defer owed.deinit(gpa);
while (queue.pop()) |m| {
defer if (m.bytes.len > 0) gpa.free(m.bytes);
switch (m.kind) {
.input => {
const c = m.who - 1;
var buf: [64]u8 = undefined;
const ev = nextEvent(random, &buf, if (c == 0) wheel_at else null, &looks);
const before = reps[c].stats;
const msg = try gpa.dupe(u8, try reps[c].input(ev));
if (trace) std.debug.print("{d} ms client {d}: {t} {s} -> {s}\n", .{ m.at / 1000, m.who, ev, switch (ev) {
.key => |k| k.text,
.paste => |t| t,
.command => |t| t,
else => "",
}, if (reps[c].stats.speculated > before.speculated) "applied" else if (reps[c].stats.stopped > before.stopped) "STOPPED" else if (reps[c].stats.unspeculated > before.unspeculated) "view" else "waits" });
try push(&queue, &n, .{ .at = up[c].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .to_authority, .who = m.who, .bytes = msg });
left[c] -= 1;
if (left[c] > 0) {
// A typist: keys 50-200 ms apart, a pause now and then.
const gap: u64 = if (random.uintLessThan(u8, 12) == 0) random.intRangeAtMost(u64, 300_000, 1_500_000) else random.intRangeAtMost(u64, 50_000, 200_000);
try push(&queue, &n, .{ .at = m.at + gap, .n = 0, .kind = .input, .who = m.who, .bytes = &.{} });
} else typing -= 1;
},
.to_authority, .echo, .output => {
const unit = switch (m.kind) {
.echo => echo: {
a.output(.{ .output = .{ .pane = @intCast(m.who), .bytes = m.bytes } });
break :echo null;
},
.output => out: {
// The program's next read, 4 KB, then the next a
// fraction of a millisecond on (ghostty's pace).
if (program.at == program.bytes.len) {
program = .{ .bytes = switch (m.who) {
0 => tui,
1 => yes,
else => session,
} };
}
const k = @min(program.bytes.len - program.at, if (m.who == 1) @as(usize, 4096) else 700);
const chunk = program.bytes[program.at..][0..k];
program.at += k;
if (program.at < program.bytes.len)
try push(&queue, &n, .{ .at = m.at + if (m.who == 1) @as(u64, 450) else 3_000, .n = 0, .kind = .output, .who = m.who, .bytes = &.{} })
else if (m.who == 1) flood_end = m.at;
if (m.who == 1) flood_units += 1;
a.output(.{ .output = .{ .pane = shell, .bytes = chunk } });
break :out null;
},
else => in: {
// The authority's own steps since the last unit, and
// the terminals as they are: what the input's step
// reads, before it is stepped.
if (try a.pending()) |batch| try sendUnit(&queue, &n, batch, m.at, down[0..r.clients], random, one_way, jitter, totals, m.at >= flood_at and flood_end == 0, &screens, a.core, peers[0..r.clients]);
try flushAll(&screens, a.core, peers[0..r.clients], &owed, &queue, &n, m.at, &down, random, one_way, jitter, &versions, totals);
break :in try a.take(m.bytes);
},
};
if (unit) |u| try sendUnit(&queue, &n, u, m.at, down[0..r.clients], random, one_way, jitter, totals, false, &screens, a.core, peers[0..r.clients]);
for (host.echo.items) |e| try push(&queue, &n, .{ .at = m.at + 2_000, .n = 0, .kind = .echo, .who = e.pane, .bytes = e.bytes });
host.echo.clearRetainingCapacity();
},
.frame => {
if (try a.pending()) |batch| try sendUnit(&queue, &n, batch, m.at, down[0..r.clients], random, one_way, jitter, totals, m.at >= flood_at and flood_end == 0, &screens, a.core, peers[0..r.clients]);
const t0 = screens.encode_ns;
const before = totals.delta_bytes;
try flushAll(&screens, a.core, peers[0..r.clients], &owed, &queue, &n, m.at, &down, random, one_way, jitter, &versions, totals);
if (m.at >= flood_at and (flood_end == 0 or m.at <= flood_end)) {
totals.flood_delta_bytes += totals.delta_bytes - before;
try totals.frame_ns.append(gpa, screens.encode_ns - t0);
}
// Frames go on while anything moves, and a second after.
const busy = typing > 0 or flood_end == 0 or queue.count() > 0;
frames_left = if (busy) 60 else frames_left -| 1;
if (frames_left > 0) try push(&queue, &n, .{ .at = m.at + frame_us, .n = 0, .kind = .frame, .who = 0, .bytes = &.{} });
},
.to_replica, .screen_down => {
const c = m.who - 1;
const rp = reps[c];
if (m.kind == .to_replica) {
rp.confirm(m.bytes);
rp.settle();
} else if (rp.screen(m.bytes)) |reply| {
const pane = m.bytes[0];
if (reply[5] == 1) {
// Applied: the screen is the authority's at that version.
const sc = rp.screens[pane].?;
const vt = &rp.core().panes[pane].?.terminal.?.vt;
totals.screens_checked += 1;
if (versions.get(@as(u64, sc.serial) << 32 | sc.rep.seq)) |want| {
if (want != screenHash(vt)) {
totals.screens_wrong += 1;
std.debug.print("seed {d} rtt {d} ms: replica {d}'s terminal {d} at version {d} is not the authority's\n", .{ r.seed, r.rtt_ms, m.who, pane, sc.rep.seq });
}
} else totals.screens_wrong += 1;
// Deltas while the flood runs (an RTT/2 behind it).
if (m.at >= flood_at + one_way and (flood_end == 0 or m.at <= flood_end + one_way)) flood_deltas[c] += 1;
} else totals.resets += 1;
try push(&queue, &n, .{ .at = up[c].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .screen_up, .who = m.who, .bytes = try gpa.dupe(u8, reply) });
}
if (rp.screenRequest()) |req| try push(&queue, &n, .{ .at = up[c].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .screen_up, .who = m.who, .bytes = try gpa.dupe(u8, req) });
},
.client_frame => {},
.screen_up => {
const c = m.who - 1;
if (m.bytes[5] == 2) totals.fetched += std.mem.readInt(u64, m.bytes[14..22], .little) - std.mem.readInt(u64, m.bytes[6..14], .little);
if (try screens.take(a.core, peers[c], m.bytes)) |answer| try push(&queue, &n, .{ .at = down[c].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .screen_down, .who = m.who, .bytes = answer });
},
}
}
totals.flood_us += flood_end -| flood_at;
for (flood_deltas[0..r.clients]) |d| totals.flood_deltas += d;
const want = replica.digestOf(a);
for (reps[0..r.clients], 1..) |rp, id| {
try totals.add(&rp.stats);
if (rp.diverged) |why| {
totals.divergences += 1;
std.debug.print("seed {d} rtt {d} ms: replica {d} diverged: {s}\n", .{ r.seed, r.rtt_ms, id, why });
} else if (!rp.quiet() or rp.digest() != want) {
totals.unequal += 1;
std.debug.print("seed {d} rtt {d} ms: replica {d} at quiescence: {s}\n", .{ r.seed, r.rtt_ms, id, if (rp.quiet()) "another digest" else "inputs still pending" });
}
// At quiescence, every terminal the authority's, cell for cell.
for (a.core.panes, 0..) |slot, pid| {
const pane = slot orelse continue;
const state = pane.terminal orelse continue;
const mine = (rp.core().panes[pid] orelse {
totals.unequal += 1;
continue;
}).terminal orelse continue;
if (screenHash(&state.vt) != screenHash(&mine.vt)) {
totals.unequal += 1;
std.debug.print("seed {d} rtt {d} ms: replica {d}'s terminal {d} at quiescence is not the authority's\n", .{ r.seed, r.rtt_ms, id, pid });
}
}
// ...and every file's text the authority's, byte for byte: a digest
// takes a text of 4 KiB and up by its length (src/walk.zig).
for (a.core.panes, 0..) |slot, pid| {
const f = (slot orelse continue).file orelse continue;
const mine = if (rp.core().panes[pid]) |pane| pane.file else null;
if (mine == null or !std.mem.eql(u8, f.content, mine.?.content)) {
totals.unequal += 1;
std.debug.print("seed {d} rtt {d} ms: replica {d}'s text {d} at quiescence is not the authority's\n", .{ r.seed, r.rtt_ms, id, pid });
}
}
}
}
fn push(q: *std.PriorityQueue(Msg, void, Msg.order), nn: *u64, m: Msg) !void {
var mm = m;
mm.n = nn.*;
nn.* += 1;
try q.push(gpa, mm);
}
/// What each peer is owed of the terminals, onto its link (before the
/// authority steps an input, and once a frame); each version as the
/// authority's screen showed it.
fn flushAll(sc: *replica.Screens, core: *pardes.Pardes, ps: []const u16, list: *std.ArrayListUnmanaged([]u8), q: *std.PriorityQueue(Msg, void, Msg.order), nn: *u64, now: u64, dl: []Link, rnd: std.Random, ow: u64, jt: u64, vs: *std.AutoHashMapUnmanaged(u64, u64), tot: *Totals) !void {
for (ps, 0..) |peer, c| {
list.clearRetainingCapacity();
// No pane of a kind here: the view a Shadow is painted for is moot.
try sc.flush(core, peer, 0, list);
for (list.items) |bytes| {
tot.deltas += 1;
tot.delta_bytes += bytes.len;
try push(q, nn, .{ .at = dl[c].arrive(rnd, now, ow, jt), .n = 0, .kind = .screen_down, .who = @intCast(c + 1), .bytes = bytes });
}
}
// Each version as the authority's screen showed it.
for (&sc.tracks, 0..) |*slot, id| if (slot.*) |*tr| {
const key = @as(u64, tr.serial) << 32 | tr.track.version;
if (tr.track.version == 0 or vs.contains(key)) continue;
_ = core.panes[id] orelse continue;
try vs.put(gpa, key, screenHash(&tr.state.vt));
};
}
/// A unit of the authority's to every replica, down its link: first the
/// rows of its terminals its step read that a replica holds as holes.
fn sendUnit(q: *std.PriorityQueue(Msg, void, Msg.order), n: *u64, unit: []const u8, now: u64, down: []Link, random: std.Random, one_way: u64, jitter: u64, totals: *Totals, flooding: bool, screens: *replica.Screens, core: *pardes.Pardes, peers: []const u16) !void {
var rows: std.ArrayListUnmanaged([]u8) = .empty;
defer rows.deinit(gpa);
for (down, peers, 1..) |*l, peer, who| {
rows.clearRetainingCapacity();
try screens.fill(core, peer, &rows);
for (rows.items) |bytes| {
totals.filled_bytes += bytes.len;
try q.push(gpa, .{ .at = l.arrive(random, now, one_way, jitter), .n = n.*, .kind = .screen_down, .who = @intCast(who), .bytes = bytes });
n.* += 1;
}
}
screens.filled(core);
// The sizes its steps gave terminals, after their rows (Peers.unit).
pardes.panes.terminal.resizeDeferred(core);
totals.unit_bytes += unit.len * down.len;
if (flooding) totals.flood_unit_bytes += unit.len * down.len;
for (down, 1..) |*l, who| {
try q.push(gpa, .{ .at = l.arrive(random, now, one_way, jitter), .n = n.*, .kind = .to_replica, .who = @intCast(who), .bytes = try gpa.dupe(u8, unit) });
n.* += 1;
}
}
// ---- predicted echo (--echo): one client typing a corpus into a shell ----
//
// The authority's pty is a shell played here (`FakeShell`): a line editor at
// a prompt with pardes's OSC 133 marks and Tab completion, and the programs
// the corpus runs in it: a password prompt (no echo), vim (the alternate
// screen), a program that ignores what is typed, and `cat` (the kernel's
// plain echo, no marks). Each key's echo comes back 2 ms after the authority
// writes it, as a shell's. The client types at a typist's pace and in bursts;
// each of its frames (60 Hz, a phase of its own) is what its screen shows:
// the predicted echo where it is drawn (src/echo.zig), the screen state
// under it. Per key the shell echoed at a prompt or in cat: the time from
// the key to its character on that screen, with prediction and without. Per
// key predicted: whether the authority's screen, right after its echo,
// shows what was predicted where it was. Per key drawn wrong: its frames.
const ascii: [128][1]u8 = blk: {
var t: [128][1]u8 = undefined;
for (&t, 0..) |*c, i| c.* = .{@intCast(i)};
break :blk t;
};
const Step = struct { gap: u64, key: Key, section: u8 };
/// The corpus's parts, each a case of the gates.
const sections = [_][]const u8{ "typing", "a burst", "backspace", "left, right", "tab", "password", "vim", "ignores input", "cat", "typed ahead" };
const Corpus = struct {
steps: std.ArrayListUnmanaged(Step) = .empty,
random: std.Random,
extra: u64 = 0,
section: u8 = 0,
fn add(c: *Corpus, gap: u64, key: Key) !void {
try c.steps.append(gpa, .{ .gap = gap + c.extra, .key = key, .section = c.section });
c.extra = 0;
}
fn typing(c: *Corpus, text: []const u8, fast: bool) !void {
for (text) |ch| try c.add(if (fast) c.random.intRangeAtMost(u64, 12_000, 60_000) else c.random.intRangeAtMost(u64, 60_000, 180_000), .{ .cp = ch, .text = &ascii[ch] });
}
fn press(c: *Corpus, key: Key) !void {
try c.add(c.random.intRangeAtMost(u64, 80_000, 250_000), key);
}
fn pause(c: *Corpus) void {
c.extra += c.random.intRangeAtMost(u64, 500_000, 1_200_000);
}
fn build(c: *Corpus) !void {
const enter: Key = .{ .cp = Key.enter };
const bs: Key = .{ .cp = Key.backspace };
c.pause();
// At a prompt, at a typist's pace, then in a burst.
c.section = 0;
try c.typing("echo hello world", false);
try c.press(enter);
c.pause();
c.section = 1;
try c.typing("echo the quick brown fox jumps over", true);
try c.press(enter);
c.pause();
// Backspace.
c.section = 2;
try c.typing("ecoh", false);
try c.press(bs);
try c.press(bs);
try c.typing("ho fixed tpyo", false);
for (0..3) |_| try c.press(bs);
try c.typing("ypo", false);
try c.press(enter);
c.pause();
// Left and right over what was typed.
c.section = 3;
try c.typing("echo abc", false);
for ([_]u21{ Key.left, Key.left, Key.right, Key.right }) |k| try c.press(.{ .cp = k });
try c.typing("def", false);
try c.press(enter);
c.pause();
// Tab completion, typed straight on.
c.section = 4;
try c.typing("ec", false);
try c.press(.{ .cp = Key.tab });
try c.typing("tab completed", false);
try c.press(enter);
c.pause();
// A password prompt: nothing typed there may show.
c.section = 5;
try c.typing("sudo ls", false);
try c.press(enter);
c.pause();
try c.typing("hunter2", false);
try c.press(enter);
c.pause();
// vim: the alternate screen, never predicted.
c.section = 6;
try c.typing("vim", false);
try c.press(enter);
c.pause();
try c.typing("ihello from vim", false);
try c.press(.{ .cp = Key.escape, .shift = true });
try c.typing(":q", false);
try c.press(enter);
c.pause();
// A program that ignores what is typed.
c.section = 7;
try c.typing("sink", false);
try c.press(enter);
c.pause();
try c.typing("this goes nowhere", false);
try c.press(.{ .cp = 'c', .ctrl = true });
c.pause();
// cat: the kernel's plain echo.
c.section = 8;
try c.typing("cat", false);
try c.press(enter);
c.pause();
try c.typing("plain echo", false);
try c.press(enter);
try c.typing("a second line", false);
try c.press(enter);
try c.press(.{ .cp = 'd', .ctrl = true });
c.pause();
// Typed on before the next prompt.
c.section = 9;
try c.typing("echo one", false);
try c.press(enter);
try c.typing("echo two", false);
try c.press(enter);
c.pause();
}
};
const prompt_bytes = "\x1b]133;A;cl=line;aid=pardes\x07$ \x1b]133;B;aid=pardes\x07";
/// The shell on the authority's pty, a key a write (as forwardKey writes).
const FakeShell = struct {
mode: Mode = .prompt,
line: [256]u8 = undefined,
len: usize = 0,
cur: usize = 0,
vim_cmd: [8]u8 = undefined,
vim_n: usize = 0,
out: std.ArrayListUnmanaged(u8) = .empty,
const Mode = enum { prompt, password, vim, vim_insert, sink, cat };
fn put(sh: *FakeShell, bytes: []const u8) void {
sh.out.appendSlice(gpa, bytes) catch {};
}
fn print(sh: *FakeShell, comptime fmt: []const u8, args: anytype) void {
sh.out.print(gpa, fmt, args) catch {};
}
fn done(sh: *FakeShell, status: u8) void {
sh.print("\x1b]133;D;{d};aid=pardes\x07" ++ prompt_bytes, .{status});
sh.mode = .prompt;
sh.len = 0;
sh.cur = 0;
}
fn insert(sh: *FakeShell, c: u8) void {
if (sh.len + 1 >= sh.line.len) return;
sh.put(&.{c});
if (sh.cur < sh.len) {
sh.put(sh.line[sh.cur..sh.len]);
sh.print("\x1b[{d}D", .{sh.len - sh.cur});
}
std.mem.copyBackwards(u8, sh.line[sh.cur + 1 .. sh.len + 1], sh.line[sh.cur..sh.len]);
sh.line[sh.cur] = c;
sh.len += 1;
sh.cur += 1;
}
fn key(sh: *FakeShell, b: []const u8) void {
const printable = b.len == 1 and b[0] >= 0x20 and b[0] < 0x7f;
const eql = std.mem.eql;
switch (sh.mode) {
.prompt => if (printable) sh.insert(b[0]) else if (eql(u8, b, "\x7f")) {
if (sh.cur == 0) return;
if (sh.cur == sh.len) sh.put("\x08\x1b[K") else {
sh.put("\x08");
sh.put(sh.line[sh.cur..sh.len]);
sh.print(" \x1b[{d}D", .{sh.len - sh.cur + 1});
}
std.mem.copyForwards(u8, sh.line[sh.cur - 1 .. sh.len - 1], sh.line[sh.cur..sh.len]);
sh.len -= 1;
sh.cur -= 1;
} else if (eql(u8, b, "\x1b[D")) {
if (sh.cur == 0) return;
sh.put(b);
sh.cur -= 1;
} else if (eql(u8, b, "\x1b[C")) {
if (sh.cur == sh.len) return;
sh.put(b);
sh.cur += 1;
} else if (eql(u8, b, "\t")) {
if (sh.cur != sh.len or std.mem.indexOfScalar(u8, sh.line[0..sh.len], ' ') != null) return;
for ([_][]const u8{ "echo", "sudo", "sink", "cat", "vim" }) |cmd| {
if (sh.len == 0 or sh.len >= cmd.len or !std.mem.startsWith(u8, cmd, sh.line[0..sh.len])) continue;
for (cmd[sh.len..]) |ch| sh.insert(ch);
sh.insert(' ');
return;
}
} else if (eql(u8, b, "\r")) {
var cmd_buf: [256]u8 = undefined;
const cmd = cmd_buf[0..sh.len];
@memcpy(cmd, sh.line[0..sh.len]);
sh.len = 0;
sh.cur = 0;
sh.put("\r\n\x1b]133;C;aid=pardes\x07");
if (cmd.len == 0) return sh.done(0);
if (std.mem.startsWith(u8, cmd, "echo")) {
sh.print("{s}\r\n", .{std.mem.trimStart(u8, cmd[4..], " ")});
sh.done(0);
} else if (std.mem.startsWith(u8, cmd, "sudo")) {
sh.put("[sudo] password for you: ");
sh.mode = .password;
} else if (eql(u8, cmd, "vim")) {
sh.put("\x1b[?1049h\x1b[2J\x1b[H~\r\n~\r\n~\x1b[H");
sh.mode = .vim;
} else if (eql(u8, cmd, "sink")) {
sh.mode = .sink;
} else if (eql(u8, cmd, "cat")) {
sh.mode = .cat;
} else {
sh.print("fake: unknown command: {s}\r\n", .{cmd});
sh.done(127);
}
} else if (eql(u8, b, "\x03")) {
sh.put("^C\r\n");
sh.done(130);
},
.password => if (eql(u8, b, "\r")) {
sh.put("\r\nsudo: 1 incorrect password attempt\r\n");
sh.done(1);
} else if (eql(u8, b, "\x03")) {
sh.put("^C\r\n");
sh.done(130);
},
.vim => if (sh.vim_n > 0) {
if (eql(u8, b, "\r")) {
const quit = eql(u8, sh.vim_cmd[0..sh.vim_n], ":q");
sh.vim_n = 0;
if (quit) {
sh.put("\x1b[?1049l");
sh.done(0);
}
} else if (printable and sh.vim_n < sh.vim_cmd.len) {
sh.vim_cmd[sh.vim_n] = b[0];
sh.vim_n += 1;
}
} else if (eql(u8, b, "i")) {
sh.mode = .vim_insert;
} else if (eql(u8, b, ":")) {
sh.vim_cmd[0] = ':';
sh.vim_n = 1;
},
.vim_insert => if (eql(u8, b, "\x1b")) {
sh.mode = .vim;
} else if (printable) sh.put(b),
.sink => if (eql(u8, b, "\x03")) {
sh.put("^C\r\n");
sh.done(130);
},
.cat => if (printable) {
sh.put(b);
if (sh.len < sh.line.len) {
sh.line[sh.len] = b[0];
sh.len += 1;
}
} else if (eql(u8, b, "\x7f")) {
if (sh.len == 0) return;
sh.put("\x08 \x08");
sh.len -= 1;
} else if (eql(u8, b, "\r")) {
sh.print("\r\n{s}\r\n", .{sh.line[0..sh.len]});
sh.len = 0;
} else if (eql(u8, b, "\x04") and sh.len == 0) {
sh.done(0);
} else if (eql(u8, b, "\x03")) {
sh.put("^C\r\n");
sh.done(130);
},
}
}
};
/// The authority's host: each pty write tagged with the client input that
/// made it.
const EchoHost = struct {
writes: std.ArrayListUnmanaged(struct { seq: u32, bytes: []u8 }) = .empty,
current: u32 = 0,
fn watched(_: ?*anyopaque) bool {
return false;
}
fn ptyWrite(ctx: ?*anyopaque, _: u8, bytes: []const u8) void {
const h: *EchoHost = @ptrCast(@alignCast(ctx.?));
h.writes.append(gpa, .{ .seq = h.current, .bytes = gpa.dupe(u8, bytes) catch return }) catch {};
}
fn ignore(_: ?*anyopaque, _: []const u8) void {}
const vtable: pardes.Host.VTable = .{ .watched = watched, .pty_write = ptyWrite, .set_clipboard = ignore };
fn host(h: *EchoHost) pardes.Host {
return .{ .ctx = h, .vtable = &vtable };
}
};
const Typed = struct {
seq: u32,
section: u8,
at: u64,
printable: bool,
/// What was predicted: the cell (its row in the active area) and the
/// cursor after it.
pred: ?struct { x: u16, y: u16, cp: u21 } = null,
pcur: ?struct { x: u16, y: u16 } = null,
/// The shell's mode as it took the key; whether it printed anything for
/// it; the authority's version of the screen before that.
mode: ?FakeShell.Mode = null,
echoed: bool = false,
version_before: u32 = 0,
/// The authority's screen right after its echo: its cursor and the
/// cursor's row; whether that is what was predicted.
after: ?struct { x: u16, y: u16, row: [128]u21 } = null,
truth: ?bool = null,
/// Its prediction drawn: first, and in how many frames.
first_visible: ?u64 = null,
visible_frames: u32 = 0,
/// The screen state showing its echo, first.
seen: ?u64 = null,
};
const EchoTotals = struct {
keys: u64 = 0,
echo_keys: u64 = 0,
latency: std.ArrayListUnmanaged(u64) = .empty,
predicted: u64 = 0,
unpredicted: u64 = 0,
confirmed: u64 = 0,
mispredicted: u64 = 0,
cursor_missed: u64 = 0,
right: u64 = 0,
wrong: u64 = 0,
wrong_shown: u64 = 0,
wrong_frames: u64 = 0,
wrong_max: u64 = 0,
secret_frames: u64 = 0,
alt_predicted: u64 = 0,
divergences: u64 = 0,
unequal: u64 = 0,
screens_wrong: u64 = 0,
/// The replica's frame drawn while it shows a prediction (its core's
/// render, the tty's and the GUI's), and the overlay's part of it (ns);
/// cells drawn otherwise than predicted.
render_ns: std.ArrayListUnmanaged(u64) = .empty,
paint_ns: std.ArrayListUnmanaged(u64) = .empty,
drawn_frames: u64 = 0,
paint_wrong: u64 = 0,
sec: [sections.len]Section = @splat(.{}),
const Section = struct { keys: u64 = 0, predicted: u64 = 0, wrong_frames: u64 = 0, latency: std.ArrayListUnmanaged(u64) = .empty };
fn deinit(et: *EchoTotals) void {
for (&et.sec) |*x| x.latency.deinit(gpa);
et.latency.deinit(gpa);
et.render_ns.deinit(gpa);
et.paint_ns.deinit(gpa);
}
};
fn cellCp(state: *pardes.panes.terminal.State, x: u16, y: u16) u21 {
const ci = state.vt.screens.active.pages.getCell(.{ .active = .{ .x = x, .y = y } }) orelse return 0x10ffff;
const cp = ci.cell.codepoint();
return if (cp == 0) ' ' else cp;
}
/// A key's prediction against the authority's screen right after its echo,
/// once both are known (a key typed behind one not predicted is predicted
/// after the authority has it).
fn judgeTruth(k: *Typed) void {
if (k.truth != null) return;
const pc = k.pcur orelse return;
const after = k.after orelse return;
var ok = after.x == pc.x and after.y == pc.y;
if (k.pred) |cell| ok = ok and cell.y == after.y and cell.x < after.row.len and after.row[cell.x] == cell.cp;
k.truth = ok;
if (trace and !ok) std.debug.print("key {d} in {t} predicted cursor {d},{d} cell {any}; the authority's cursor {d},{d}\n", .{ k.seq, k.mode.?, pc.x, pc.y, k.pred, after.x, after.y });
}
fn runEcho(seed: u64, rtt_ms: u64, on: bool, et: *EchoTotals) !void {
var prng = std.Random.DefaultPrng.init(seed *% 0x9e3779b97f4a7c15 +% rtt_ms);
const random = prng.random();
var host: EchoHost = .{};
defer {
for (host.writes.items) |w| gpa.free(w.bytes);
host.writes.deinit(gpa);
}
var sh: FakeShell = .{};
defer sh.out.deinit(gpa);
const a = try replica.Authority.init(gpa, .{ .cols = 100, .rows = 30, .boot = .tty_shell }, host.host(), true);
defer a.deinit();
var screens: replica.Screens = .init(gpa);
defer screens.deinit(a.core);
const peer = try screens.addPeer();
const peers = [_]u16{peer};
const rp = try replica.Replica.init(gpa, 1, try a.unit(0, 0), true);
defer rp.deinit();
rp.echo.on = on;
const shell: u8 = for (a.core.panes, 0..) |slot, id| {
if (slot) |pane| if (pane.isTerminal()) break @intCast(id);
} else return error.NoShell;
var corpus: Corpus = .{ .random = random };
defer corpus.steps.deinit(gpa);
try corpus.build();
var typed: std.ArrayListUnmanaged(Typed) = .empty;
defer typed.deinit(gpa);
var t: Totals = .{};
defer {
t.spec_ns.deinit(gpa);
t.copy_ns.deinit(gpa);
t.rollback_ns.deinit(gpa);
t.apply_ns.deinit(gpa);
t.frame_ns.deinit(gpa);
}
var versions: std.AutoHashMapUnmanaged(u64, u64) = .empty;
defer versions.deinit(gpa);
var queue: std.PriorityQueue(Msg, void, Msg.order) = .empty;
defer {
while (queue.pop()) |m| if (m.bytes.len > 0) gpa.free(m.bytes);
queue.deinit(gpa);
}
var n: u64 = 0;
const one_way = rtt_ms * 1000 / 2;
const jitter = one_way / 10;
var up: [1]Link = @splat(.{});
var down: [1]Link = @splat(.{});
var owed: std.ArrayListUnmanaged([]u8) = .empty;
defer owed.deinit(gpa);
// The shell's first prompt.
sh.put(prompt_bytes);
a.output(.{ .output = .{ .pane = shell, .bytes = sh.out.items } });
sh.out.clearRetainingCapacity();
var next_step: usize = 0;
try push(&queue, &n, .{ .at = corpus.steps.items[0].gap, .n = 0, .kind = .input, .who = 1, .bytes = &.{} });
try push(&queue, &n, .{ .at = frame_us, .n = 0, .kind = .frame, .who = 0, .bytes = &.{} });
try push(&queue, &n, .{ .at = random.uintLessThan(u64, frame_us), .n = 0, .kind = .client_frame, .who = 1, .bytes = &.{} });
// Two clocks tick while anything else moves, and a second after.
var periodic: usize = 2;
var left: [2]u32 = .{ 60, 60 };
var frame_arena: std.heap.ArenaAllocator = .init(gpa);
defer frame_arena.deinit();
var made: std.ArrayListUnmanaged(echo.Echo.Predicted) = .empty;
defer made.deinit(gpa);
rp.echo.log = &made;
while (queue.pop()) |m| {
defer if (m.bytes.len > 0) gpa.free(m.bytes);
rp.echo.now = m.at * 1000;
// What it predicted in the last event's calls (a key typed, or the
// keys behind one it did not, at a unit, a delta or a frame).
defer {
const top = rp.echo.tops[shell];
for (made.items) |pr| {
const k = &typed.items[pr.seq - 1];
if (trace) {
const sc = rp.core().panes[shell].?.terminal.?.vt.screens.active.cursor;
std.debug.print("{d} ms ({t}): key {d} predicted {any} cursor {d}; acked {d} wait {d} nd {d}; the screen's cursor {d},{d}\n", .{ m.at / 1000, m.kind, pr.seq, pr.cell, pr.cursor.x, rp.echo.acked, rp.echo.wait, rp.echo.nd, sc.x, sc.y });
}
if (pr.cell) |cell| k.pred = .{ .x = cell.x, .y = @intCast(cell.row - top), .cp = cell.cp };
k.pcur = .{ .x = pr.cursor.x, .y = @intCast(pr.cursor.row - top) };
judgeTruth(k);
}
made.clearRetainingCapacity();
}
switch (m.kind) {
.input => {
const step = corpus.steps.items[next_step];
next_step += 1;
const msg = try gpa.dupe(u8, try rp.input(.{ .key = step.key }));
const seq = rp.next_seq - 1;
try typed.append(gpa, .{ .seq = seq, .section = step.section, .at = m.at, .printable = step.key.text.len == 1 });
if (trace) std.debug.print("{d} ms: key {d} typed: '{s}' cp {x}\n", .{ m.at / 1000, seq, step.key.text, step.key.cp });
try push(&queue, &n, .{ .at = up[0].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .to_authority, .who = 1, .bytes = msg });
if (next_step < corpus.steps.items.len) try push(&queue, &n, .{ .at = m.at + corpus.steps.items[next_step].gap, .n = 0, .kind = .input, .who = 1, .bytes = &.{} });
},
.to_authority => {
if (try a.pending()) |batch| try sendUnit(&queue, &n, batch, m.at, &down, random, one_way, jitter, &t, false, &screens, a.core, &peers);
try flushAll(&screens, a.core, &peers, &owed, &queue, &n, m.at, &down, random, one_way, jitter, &versions, &t);
host.current = (replica.parseInput(m.bytes) orelse return error.BadInput).seq;
const unit = try a.take(m.bytes);
host.current = 0;
try sendUnit(&queue, &n, unit, m.at, &down, random, one_way, jitter, &t, false, &screens, a.core, &peers);
for (host.writes.items) |w| try push(&queue, &n, .{ .at = m.at + 2_000, .n = 0, .kind = .echo, .who = shell, .bytes = w.bytes, .seq = w.seq });
host.writes.clearRetainingCapacity();
},
.echo => {
const ty: ?*Typed = if (m.seq > 0) &typed.items[m.seq - 1] else null;
const state = a.core.panes[shell].?.terminal.?;
if (ty) |k| {
k.mode = sh.mode;
k.version_before = if (screens.tracks[shell]) |*tr| tr.track.version else 0;
}
sh.key(m.bytes);
if (sh.out.items.len > 0) {
a.output(.{ .output = .{ .pane = shell, .bytes = sh.out.items } });
sh.out.clearRetainingCapacity();
if (ty) |k| k.echoed = true;
}
if (ty) |k| {
const cur = state.vt.screens.active.cursor;
k.after = .{ .x = @intCast(cur.x), .y = @intCast(cur.y), .row = undefined };
for (&k.after.?.row, 0..) |*cp, x| cp.* = cellCp(state, @intCast(x), @intCast(cur.y));
judgeTruth(k);
}
},
.frame => {
if (try a.pending()) |batch| try sendUnit(&queue, &n, batch, m.at, &down, random, one_way, jitter, &t, false, &screens, a.core, &peers);
try flushAll(&screens, a.core, &peers, &owed, &queue, &n, m.at, &down, random, one_way, jitter, &versions, &t);
const busy = next_step < corpus.steps.items.len or queue.count() + 1 > periodic;
left[0] = if (busy) 60 else left[0] -| 1;
if (left[0] > 0) try push(&queue, &n, .{ .at = m.at + frame_us, .n = 0, .kind = .frame, .who = 0, .bytes = &.{} }) else periodic -= 1;
},
.client_frame => {
_ = rp.echoFrame();
const core = rp.core();
const drawn = rp.echo.visible(core) and core.panes[shell].?.body.mode == .tty;
if (drawn and rp.echo.n > 0) {
// Drawn as the client draws it: each cell where it was
// predicted.
_ = frame_arena.reset(.retain_capacity);
core.needs_frame = true;
const t0 = pardes.FrameStats.nowNs();
const surf = try core.render(frame_arena.allocator());
try et.render_ns.append(gpa, pardes.FrameStats.nowNs() - t0);
const body = surf.region(.body, shell).?.rect;
const top = rp.echo.tops[shell];
for (rp.echo.cells[0..rp.echo.n]) |cell| {
if (cell.row < top or surf.at(body.x + cell.x, body.y + @as(u16, @intCast(cell.row - top))).text[0] != cell.cp) et.paint_wrong += 1;
}
const t1 = pardes.FrameStats.nowNs();
rp.echo.paint(core, surf, core.panes[shell].?, shell, body.x, body.w, body.y, body.h, true);
try et.paint_ns.append(gpa, pardes.FrameStats.nowNs() - t1);
et.drawn_frames += 1;
}
if (drawn) for (rp.echo.cells[0..rp.echo.n]) |cell| {
const k = &typed.items[cell.seq - 1];
k.visible_frames += 1;
if (k.first_visible == null) {
k.first_visible = m.at;
if (trace) {
const st = core.panes[shell].?.terminal.?;
std.debug.print("{d} ms: key {d} ('{u}') drawn: trusted {} doubt {} phase {t} sem {t}\n", .{ m.at / 1000, cell.seq, cell.cp, rp.echo.trusted, rp.echo.doubt, st.stream.handler.phase, st.vt.screens.active.cursor.semantic_content });
}
}
};
const applied = if (rp.screens[shell]) |sc| sc.rep.seq else 0;
for (typed.items) |*k| if (k.echoed and k.seen == null and applied > k.version_before) {
k.seen = m.at;
};
const busy = next_step < corpus.steps.items.len or queue.count() + 1 > periodic;
left[1] = if (busy) 60 else left[1] -| 1;
if (left[1] > 0) try push(&queue, &n, .{ .at = m.at + frame_us, .n = 0, .kind = .client_frame, .who = 1, .bytes = &.{} }) else periodic -= 1;
},
.to_replica => {
rp.confirm(m.bytes);
rp.settle();
},
.screen_down => {
if (rp.screen(m.bytes)) |reply| {
const pane = m.bytes[0];
if (reply[5] == 1) {
const sc = rp.screens[pane].?;
if (versions.get(@as(u64, sc.serial) << 32 | sc.rep.seq)) |want| {
if (want != screenHash(&rp.core().panes[pane].?.terminal.?.vt)) t.screens_wrong += 1;
} else t.screens_wrong += 1;
}
try push(&queue, &n, .{ .at = up[0].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .screen_up, .who = 1, .bytes = try gpa.dupe(u8, reply) });
}
if (rp.screenRequest()) |req| try push(&queue, &n, .{ .at = up[0].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .screen_up, .who = 1, .bytes = try gpa.dupe(u8, req) });
},
.screen_up => {
if (try screens.take(a.core, peer, m.bytes)) |answer| try push(&queue, &n, .{ .at = down[0].arrive(random, m.at, one_way, jitter), .n = 0, .kind = .screen_down, .who = 1, .bytes = answer });
},
.output => {},
}
}
const s = rp.echo.stats;
et.predicted += s.predicted;
et.unpredicted += s.unpredicted;
et.confirmed += s.confirmed;
et.mispredicted += s.mispredicted;
et.cursor_missed += s.cursor_missed;
et.screens_wrong += t.screens_wrong;
for (typed.items) |k| {
et.keys += 1;
const sec = &et.sec[k.section];
sec.keys += 1;
if (k.pcur != null) sec.predicted += 1;
if (k.visible_frames > 0 and k.truth != true) sec.wrong_frames += k.visible_frames;
if (k.truth) |ok| {
if (ok) et.right += 1 else et.wrong += 1;
}
const md = k.mode orelse .prompt;
if (md == .password or md == .sink) et.secret_frames += k.visible_frames;
if ((md == .vim or md == .vim_insert) and k.pcur != null) et.alt_predicted += 1;
if (k.visible_frames > 0 and k.truth != true) {
if (trace) std.debug.print("seed {d}: key {d} at {d} ms in {t} drawn {d} frames, truth {?}, predicted {any} cursor {any}, the authority after it {any}\n", .{ seed, k.seq, k.at / 1000, md, k.visible_frames, k.truth, k.pred, k.pcur, if (k.after) |af| .{ af.x, af.y } else null });
et.wrong_shown += 1;
et.wrong_frames += k.visible_frames;
et.wrong_max = @max(et.wrong_max, k.visible_frames);
}
if (k.printable and k.echoed and (md == .prompt or md == .cat)) {
const seen = k.seen orelse std.math.maxInt(u64);
const drawn = if (k.first_visible != null and k.truth == true) @min(k.first_visible.?, seen) else seen;
if (drawn == std.math.maxInt(u64)) {
std.debug.print("seed {d} rtt {d} ms: a key's echo was never drawn\n", .{ seed, rtt_ms });
et.unequal += 1;
continue;
}
et.echo_keys += 1;
try et.latency.append(gpa, drawn - k.at);
try sec.latency.append(gpa, drawn - k.at);
if (trace and drawn - k.at > 2 * frame_us) std.debug.print("key {d} at {d} ms in {t}: drawn after {d} ms, {s}\n", .{ k.seq, k.at / 1000, md, (drawn - k.at) / 1000, if (k.pcur == null) "not predicted" else if (k.first_visible == null) "predicted, not drawn" else if (k.truth != true) "predicted wrong" else "predicted, drawn late" });
}
}
if (rp.diverged) |why| {
et.divergences += 1;
std.debug.print("seed {d} rtt {d} ms: the replica diverged: {s}\n", .{ seed, rtt_ms, why });
} else if (!rp.quiet() or rp.digest() != replica.digestOf(a)) {
et.unequal += 1;
std.debug.print("seed {d} rtt {d} ms: the replica at quiescence: {s}\n", .{ seed, rtt_ms, if (rp.quiet()) "another digest" else "inputs still pending" });
}
if (screenHash(&a.core.panes[shell].?.terminal.?.vt) != screenHash(&rp.core().panes[shell].?.terminal.?.vt)) {
et.unequal += 1;
std.debug.print("seed {d} rtt {d} ms: the shell at quiescence is not the authority's\n", .{ seed, rtt_ms });
}
}
fn echoMain(seeds: u64, first: u64, rtts: []const u64) !bool {
var failed = false;
var draw_lines: std.ArrayList(u8) = .empty;
defer draw_lines.deinit(gpa);
std.debug.print("predicted echo: one client typing the corpus into a shell, {d} seeds a cell; from a key to its character on the client's screen (ms, frames at 60 Hz)\n", .{seeds});
std.debug.print(" rtt echo keys echoed p50 p90 max predicted unpredicted right/wrong confirmed mispredicted cursor-missed drawn-wrong keys/frames/max secret-frames vim-predicted diverged unequal screens-wrong\n", .{});
var by_section: std.ArrayList(u8) = .empty;
defer by_section.deinit(gpa);
var off: [sections.len][2]u64 = undefined;
for (rtts) |rtt| for ([_]bool{ false, true }) |on| {
var et: EchoTotals = .{};
defer et.deinit();
for (first..first + seeds) |seed| try runEcho(seed, rtt, on, &et);
const ms = struct {
fn f(us: u64) f64 {
return @as(f64, @floatFromInt(us)) / 1000;
}
}.f;
std.debug.print("{d:>4} {s:>4} {d:>4} {d:>6} {d:>5.1} {d:>5.1} {d:>5.1} {d:>9} {d:>11} {d:>5}/{d:<5} {d:>9} {d:>12} {d:>13} {d:>5}/{d:>5}/{d:<5} {d:>13} {d:>13} {d:>8} {d:>7} {d:>13}\n", .{
rtt, if (on) "on" else "off", et.keys, et.echo_keys,
ms(pct(et.latency.items, 50)), ms(pct(et.latency.items, 90)), ms(pct(et.latency.items, 100)), et.predicted,
et.unpredicted, et.right, et.wrong, et.confirmed,
et.mispredicted, et.cursor_missed, et.wrong_shown, et.wrong_frames,
et.wrong_max, et.secret_frames, et.alt_predicted, et.divergences,
et.unequal, et.screens_wrong,
});
for (&et.sec, &off, sections) |*x, *o, name| {
const p50 = pct(x.latency.items, 50) / 1000;
const p90 = pct(x.latency.items, 90) / 1000;
if (!on) {
o.* = .{ p50, p90 };
continue;
}
try by_section.print(gpa, "{d:>4} {s:<14} {d:>5} {d:>9} {d:>7} {d:>4} -> {d:<4} {d:>4} -> {d:<4} {d:>12}\n", .{ rtt, name, x.keys, x.predicted, x.latency.items.len, o[0], p50, o[1], p90, x.wrong_frames });
}
if (on) try draw_lines.print(gpa, "{d:>4} {d:>12} {d:>6}/{d:<6} {d:>5}/{d:<5} {d:>10}\n", .{ rtt, et.drawn_frames, pct(et.render_ns.items, 50) / 1000, pct(et.render_ns.items, 99) / 1000, pct(et.paint_ns.items, 50), pct(et.paint_ns.items, 99), et.paint_wrong });
if (et.divergences > 0 or et.unequal > 0 or et.screens_wrong > 0 or et.secret_frames > 0 or et.alt_predicted > 0 or et.paint_wrong > 0) failed = true;
};
std.debug.print("by part of the corpus (all seeds): keys typed, predicted, echoed (the latency sample); key to its character on the screen, ms, without -> with; frames a wrong prediction was drawn\n rtt part keys predicted echoed p50 off/on p90 off/on drawn-wrong\n{s}", .{by_section.items});
std.debug.print("the client's frames drawn with a prediction shown: its render (us) and the overlay's part (ns); cells drawn otherwise than predicted\n rtt frames-drawn render p50/p99 paint p50/p99 drawn-wrong\n{s}", .{draw_lines.items});
return !failed;
}
fn pct(samples: []u64, p: usize) u64 {
if (samples.len == 0) return 0;
std.mem.sort(u64, samples, {}, std.sort.asc(u64));
return samples[@min(samples.len - 1, samples.len * p / 100)];
}
/// `--screen-state `: every recorded pty stream in `dir` (a
/// `-x` in its name sizes it, 80x24 otherwise) through
/// screen_state's soak at three seeds, then the encode's cost and the bytes
/// floods put on the wire (numbers from a ReleaseFast build).
fn screenState(io: std.Io, dir_path: []const u8, w: *std.Io.Writer) !void {
var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true });
defer dir.close(io);
var it = dir.iterate();
var files: usize = 0;
var failed: usize = 0;
while (try it.next(io)) |entry| {
if (entry.kind != .file) continue;
const bytes = try dir.readFileAlloc(io, entry.name, gpa, .unlimited);
defer gpa.free(bytes);
var cols: u16 = 80;
var rows: u16 = 24;
if (std.mem.lastIndexOfScalar(u8, entry.name, '-')) |dash| {
const size = std.mem.sliceTo(entry.name[dash + 1 ..], '.');
if (std.mem.indexOfScalar(u8, size, 'x')) |x| {
cols = std.fmt.parseInt(u16, size[0..x], 10) catch cols;
rows = std.fmt.parseInt(u16, size[x + 1 ..], 10) catch rows;
}
}
files += 1;
for (0..3) |seed| {
const st = screen_state.soak(gpa, io, bytes, cols, rows, 128 * 1024, seed, seed != 0) catch |err| {
try w.print("screen-state {s} seed {d}: {s}\n", .{ entry.name, seed, @errorName(err) });
failed += 1;
continue;
};
if (seed == 0) try w.print("screen-state {s} {d}x{d}: {d} bytes, {d} deltas ({d} lost, {d} refused, {d} in flight at most), {d} wire bytes, {d} history rows checked, {d} fetched\n", .{ entry.name, cols, rows, bytes.len, st.deltas, st.lost, st.refused, st.most_in_flight, st.bytes, st.history_checked, st.fetched });
}
}
try w.print("screen-state corpus: {d} streams x 3 seeds, {d} failed\n", .{ files, failed });
try w.flush();
try encodeBench(io, w);
}
fn encodeBench(io: std.Io, w: *std.Io.Writer) !void {
const t = try gpa.create(screen_state.Terminal);
defer gpa.destroy(t);
t.* = try screen_state.Terminal.init(io, gpa, .{ .cols = 200, .rows = 60, .max_scrollback_bytes = 16 << 20 });
defer t.deinit(gpa);
var stream = t.vtStream();
defer stream.deinit();
var enc: screen_state.Encoder = .init(gpa);
defer enc.deinit();
var out: std.Io.Writer.Allocating = .init(gpa);
defer out.deinit();
var line: [1024]u8 = undefined;
// A full 200x60 screen, every row new to the peer: each a blob.
var full: [200]u64 = undefined;
var same: [200]u64 = undefined;
var full_bytes: usize = 0;
var same_bytes: usize = 0;
var changes: u32 = 0;
for (&full, &same, 0..) |*f, *u, rep| {
for (0..60) |y| {
var lw: std.Io.Writer = .fixed(&line);
try lw.print("\x1b[{d};1H\x1b[0;3{d};4{d}m{d} {d} ", .{ y + 1, rep % 8, y % 8, rep, y });
while (lw.end < 190) try lw.writeAll("\x1b[1mbold\x1b[22m text ");
stream.nextSlice(lw.buffered());
}
changes += 1;
var tr: screen_state.Track = .init(gpa);
defer tr.deinit(gpa, t);
var peer: screen_state.Peer = .{};
defer peer.deinit(gpa);
out.clearRetainingCapacity();
var t0 = std.Io.Clock.awake.now(io).nanoseconds;
_ = try screen_state.encode(&enc, &tr, &peer, t, changes, "", &out.writer);
f.* = @intCast(std.Io.Clock.awake.now(io).nanoseconds - t0);
full_bytes = out.written().len;
// The same screen again as a new version: every row held.
out.clearRetainingCapacity();
t0 = std.Io.Clock.awake.now(io).nanoseconds;
_ = try screen_state.encode(&enc, &tr, &peer, t, changes + 1, "", &out.writer);
u.* = @intCast(std.Io.Clock.awake.now(io).nanoseconds - t0);
same_bytes = out.written().len;
}
std.mem.sort(u64, &full, {}, std.sort.asc(u64));
std.mem.sort(u64, &same, {}, std.sort.asc(u64));
try w.print("screen-state encode 200x60, every row new: p50 {d} us p99 {d} us, {d} bytes\n", .{ full[100] / 1000, full[198] / 1000, full_bytes });
try w.print("screen-state encode 200x60, nothing new: p50 {d} us p99 {d} us, {d} bytes\n", .{ same[100] / 1000, same[198] / 1000, same_bytes });
try w.flush();
// Floods: a second of output as fast as ghostty takes it, a frame every
// 16 ms: one version, a delta for each peer (the same bytes for peers
// at the same base), each replica applying its own at once. Under the C
// allocator, as a replica's.
try floods(io, w);
}
fn floods(io: std.Io, w: *std.Io.Writer) !void {
var enc: screen_state.Encoder = .init(gpa);
defer enc.deinit();
var out: std.Io.Writer.Allocating = .init(gpa);
defer out.deinit();
const yes = try screen_state.yesStream(gpa, 1 << 16);
defer gpa.free(yes);
const shell = try screen_state.shellStream(gpa);
defer gpa.free(shell);
for ([_]struct { name: []const u8, bytes: []const u8 }{ .{ .name = "yes", .bytes = yes }, .{ .name = "varied text", .bytes = shell } }) |flood| for ([_]usize{ 1, 3 }) |npeers| {
const a = try gpa.create(screen_state.Terminal);
defer gpa.destroy(a);
a.* = try screen_state.Terminal.init(io, gpa, .{ .cols = 200, .rows = 60, .max_scrollback_bytes = 16 << 20 });
defer a.deinit(gpa);
var as = a.vtStream();
defer as.deinit();
var tr: screen_state.Track = .init(gpa);
defer tr.deinit(gpa, a);
var peers: [3]screen_state.Peer = @splat(.{});
var reps: [3]screen_state.Replica = @splat(.{});
var bs: [3]screen_state.Terminal = undefined;
for (bs[0..npeers]) |*b| b.* = try screen_state.Terminal.init(io, gpa, .{ .cols = 2, .rows = 2, .max_scrollback_bytes = 32 << 20 });
defer for (bs[0..npeers], reps[0..npeers], peers[0..npeers]) |*b, *r, *peer| {
b.deinit(gpa);
r.deinit(gpa);
peer.deinit(gpa);
};
var applies: std.ArrayList(u64) = .empty;
defer applies.deinit(gpa);
var frames: std.ArrayList(u64) = .empty;
defer frames.deinit(gpa);
const start = std.Io.Clock.awake.now(io).nanoseconds;
var next = start;
var fed: u64 = 0;
var deltas: u64 = 0;
var wire: u64 = 0;
var version: u32 = 0;
while (true) {
const now = std.Io.Clock.awake.now(io).nanoseconds;
if (now - start >= std.time.ns_per_s) break;
as.nextSlice(flood.bytes);
version += 1;
fed += flood.bytes.len;
if (now < next) continue;
next += 16 * std.time.ns_per_ms;
var frame_ns: u64 = 0;
for (peers[0..npeers], reps[0..npeers], bs[0..npeers]) |*peer, *rep, *b| {
out.clearRetainingCapacity();
const e0 = std.Io.Clock.awake.now(io).nanoseconds;
if (!try screen_state.encode(&enc, &tr, peer, a, version, "", &out.writer)) continue;
frame_ns += @intCast(std.Io.Clock.awake.now(io).nanoseconds - e0);
const a0 = std.Io.Clock.awake.now(io).nanoseconds;
peer.ack((try rep.apply(gpa, io, b, out.written())).seq);
try applies.append(gpa, @intCast(std.Io.Clock.awake.now(io).nanoseconds - a0));
deltas += 1;
wire += out.written().len;
}
try frames.append(gpa, frame_ns);
}
std.mem.sort(u64, applies.items, {}, std.sort.asc(u64));
std.mem.sort(u64, frames.items, {}, std.sort.asc(u64));
const p = struct {
fn at(s: []const u64, q: usize) u64 {
return if (s.len == 0) 0 else s[@min(s.len - 1, s.len * q / 100)] / 1000;
}
}.at;
try w.print("screen-state flood of {s}, {d} peer(s): {d} MB/s of output, {d} deltas/s a peer, {d} wire bytes/s a peer ({d} a delta); encode a frame, all peers: p50 {d} us p99 {d} us; apply a delta p50 {d} us p99 {d} us\n", .{
flood.name, npeers, fed / 1_000_000, deltas / npeers, wire / npeers,
wire / @max(1, deltas), p(frames.items, 50), p(frames.items, 99), p(applies.items, 50),
p(applies.items, 99),
});
};
try w.flush();
}
pub fn main(init: std.process.Init) !void {
const args = try init.minimal.args.toSlice(init.arena.allocator());
var seeds: u64 = 4;
var first: u64 = 0;
var steps: usize = 120;
var clients: u16 = 3;
var rtts: []const u64 = &.{ 30, 80, 207 };
var shared = false;
var echo_mode = false;
var flood_mb: u64 = 8;
var screen_dir: ?[]const u8 = null;
var rtt_buf: [8]u64 = undefined;
var i: usize = 1;
while (i < args.len) : (i += 1) {
const name = args[i];
if (std.mem.eql(u8, name, "--shared")) {
shared = true;
continue;
}
if (std.mem.eql(u8, name, "--echo")) {
echo_mode = true;
continue;
}
if (i + 1 >= args.len) return error.MissingValue;
i += 1;
const value = args[i];
if (std.mem.eql(u8, name, "--seeds")) seeds = try std.fmt.parseInt(u64, value, 10) else if (std.mem.eql(u8, name, "--first")) first = try std.fmt.parseInt(u64, value, 10) else if (std.mem.eql(u8, name, "--steps")) steps = try std.fmt.parseInt(usize, value, 10) else if (std.mem.eql(u8, name, "--clients")) clients = @min(8, try std.fmt.parseInt(u16, value, 10)) else if (std.mem.eql(u8, name, "--flood")) flood_mb = try std.fmt.parseInt(u64, value, 10) else if (std.mem.eql(u8, name, "--screen-state")) screen_dir = value else if (std.mem.eql(u8, name, "--rtt")) {
var it = std.mem.tokenizeScalar(u8, value, ',');
var k: usize = 0;
while (it.next()) |v| : (k += 1) rtt_buf[k] = try std.fmt.parseInt(u64, v, 10);
rtts = rtt_buf[0..k];
} else return error.UnknownOption;
}
if (screen_dir) |d| {
var buf: [4096]u8 = undefined;
var out = std.Io.File.stderr().writer(init.io, &buf);
return screenState(init.io, d, &out.interface);
}
trace = libc.getenv("PARDES_REPLICA_TRACE") != null;
if (libc.getenv("PARDES_REPLICA_FILL")) |b| pardes.rollback.Epoch.fill = std.fmt.parseInt(u8, std.mem.span(b), 0) catch 0xff;
var dir_buf: [256]u8 = undefined;
const tmp = std.mem.span(libc.getenv("TMPDIR") orelse "/tmp");
const dir = try std.fmt.bufPrint(&dir_buf, "{s}/replicas-{d}", .{ tmp, @as(u32, @intCast(libc.getpid())) });
try std.Io.Dir.createDirAbsolute(init.io, dir, .default_dir);
defer std.Io.Dir.cwd().deleteTree(init.io, dir) catch {};
if (echo_mode) {
if (!try echoMain(seeds, first, rtts)) std.process.exit(1);
return;
}
var failed = false;
std.debug.print("{d} seeds x {d} inputs from each of {d} clients, {s}, a TUI then {d} MB of yes in the shell; costs in us (frame {d})\n", .{ seeds, steps, clients, if (shared) "sharing one view" else "a view each", flood_mb, frame_us });
std.debug.print(" rtt inputs at-once stopped view units fast rollback(mis+foreign) rate rebase spec p50/p99 copy p50/p99 (KB) rollback p50/p99 diverged unequal\n", .{});
var term_lines: std.ArrayList(u8) = .empty;
defer term_lines.deinit(gpa);
for (rtts) |rtt| {
var t: Totals = .{};
for (first..first + seeds) |seed| try run(.{ .seed = seed, .rtt_ms = rtt, .steps = steps, .clients = clients, .shared = shared, .flood_mb = flood_mb }, dir, &t);
const rollbacks = t.mispredicted + t.foreign;
std.debug.print("{d:>4} {d:>6} {d:>7} {d:>7} {d:>4} {d:>6} {d:>6} {d:>6} ({d}+{d}) {d:>5.1}% {d:>6} {d:>5}/{d:<6} {d:>5}/{d:<5} ({d}) {d:>7}/{d:<7} {d:>8} {d:>7}\n", .{
rtt, t.inputs, t.speculated,
t.stopped, t.unspeculated, t.units,
t.fast, rollbacks, t.mispredicted,
t.foreign, @as(f64, @floatFromInt(rollbacks)) * 100 / @as(f64, @floatFromInt(@max(t.inputs, 1))), t.rebases,
pct(t.spec_ns.items, 50) / 1000, pct(t.spec_ns.items, 99) / 1000, pct(t.copy_ns.items, 50) / 1000,
pct(t.copy_ns.items, 99) / 1000, t.copy_bytes / @max(t.copies, 1) / 1024, pct(t.rollback_ns.items, 50) / 1000,
pct(t.rollback_ns.items, 99) / 1000, t.divergences, t.unequal,
});
const flood_s = @as(f64, @floatFromInt(@max(1, t.flood_us))) / 1e6;
try term_lines.print(gpa, "{d:>4} {d:>7.1} {d:>9.0} {d:>9.0} {d:>5}/{d:<5} {d:>5}/{d:<5} {d:>7} {d:>9} {d:>6} {d:>6}/{d:<6}\n", .{
rtt,
@as(f64, @floatFromInt(t.flood_deltas)) / flood_s / @as(f64, @floatFromInt(clients)),
@as(f64, @floatFromInt(t.flood_delta_bytes)) / flood_s / @as(f64, @floatFromInt(clients)),
@as(f64, @floatFromInt(t.flood_unit_bytes)) / flood_s / @as(f64, @floatFromInt(clients)),
pct(t.frame_ns.items, 50) / 1000,
pct(t.frame_ns.items, 99) / 1000,
pct(t.apply_ns.items, 50) / 1000,
pct(t.apply_ns.items, 99) / 1000,
t.fetched,
t.filled_bytes / 1024,
t.resets,
t.screens_checked,
t.screens_wrong,
});
if (t.divergences > 0 or t.unequal > 0 or t.screens_wrong > 0) failed = true;
}
std.debug.print("terminals: deltas a replica applied a second under the flood; a peer's bytes a second under the flood (deltas, units); encode a frame of the flood for every peer, apply a delta (us); rows fetched, KB of rows sent before the step that read them; resets; deltas checked/wrong\n", .{});
std.debug.print(" rtt deltas/s delta B/s unit B/s encode p50/p99 apply p50/p99 fetched filled KB resets checked/wrong\n{s}", .{term_lines.items});
if (failed) std.process.exit(1);
}