From 174e031985db910b4907ba324d6fd12c67898a5a Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Sun, 8 Nov 2026 23:49:04 -0300 Subject: tests: monkey and replay are modes of the T1 runner and snapshot.zig goes: behaviour-test runs test/monkey9p.zig bare (its smoke), `-- --monkey ` drives an in-process session with test/monkey.zig's input (keys, leader bursts, the mouse, tiny resizes, over /kbdin, /mousein and Session.resize) hunting panics and hangs, `-- --replay` records each seed's sessions as a tape in memory and replays it (tape.replayBytes, replayMain's loop: three replicas and the dump), and `process-test -- --replay` records and replays the tty cases' real sessions as `zig build replay` did (with the addons it lacked, so images passes); the snap, snap-driver-test, monkey, monkey-test (it no longer compiled), replay and monkey-session steps, pardes-snap and build/snap.zig (now build/process.zig) go; checked: a counter outside the core bumped at the 40th update diverges every T1 seed and 89 of 103 old tapes, one bumped on terminal output diverges no T1 seed but 69 tty cases and 70 old tapes (why the tty replay stays), a panic on a window under 5x5 is found by the T1 monkey (seeds 1 8 17 20) and the old one (seed 13) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge --- test/monkey.zig | 254 ------- test/monkey9p.zig | 162 ++++- test/process_test.zig | 10 +- test/session.zig | 31 + test/snapshot.zig | 1753 ------------------------------------------------- test/tty.zig | 45 ++ 6 files changed, 229 insertions(+), 2026 deletions(-) delete mode 100644 test/monkey.zig delete mode 100644 test/snapshot.zig (limited to 'test') diff --git a/test/monkey.zig b/test/monkey.zig deleted file mode 100644 index 243f2fbe..00000000 --- a/test/monkey.zig +++ /dev/null @@ -1,254 +0,0 @@ -// pardes-monkey: random snapshot scripts against the tty, hunting panics. -// -// zig build monkey -Dplatform=tui -- [steps] [--from=N] [--out=DIR] [--keep] -// -// Each seed makes one script (a fixed seed makes the same script, so a hit -// replays): keys and leader bursts, typing, mouse presses, drags and wheel, -// tiny resizes, prompts and long prompt text, over files of bad UTF-8 and -// wide glyphs. pardes-snap runs it with --update into DIR (default -// /tmp/pardes-monkey), and a panic is a record in the app's crash file -// (src/crash.zig, in the script's fake home: found even when the crash cut -// the script short) or a `.zig:N:N: 0x` stack frame in the report or the -// grid it captured (a trace renders INTO the screen). A hit keeps DIR/seed-N.snap, its captures and its report; the rest are deleted. -// A panic's trace can scroll off a small grid: replay the kept script, or -// run the app in a larger pty. Every hit gets a fix and a pinned regression -// script in test/snapshots/. -// -// Not a test gate: it is slow, random over seeds and aimed at finding, not -// holding. Shell text typed into a pane only ever runs in the throwaway -// script directory, and its alphabet has no path characters. -const std = @import("std"); -const libc = std.c; - -const Script = struct { - out: std.ArrayListUnmanaged(u8) = .empty, - gpa: std.mem.Allocator, - rows: u16 = 24, - cols: u16 = 80, - - fn line(s: *Script, comptime fmt: []const u8, args: anytype) !void { - try s.out.print(s.gpa, fmt ++ "\n", args); - } -}; - -// Workspace-tag words: clicked with the middle button they run, and the -// harness finds their column on row 1 while the window is wide enough. -const commands = [_][]const u8{ "Newcol", "Find", "Grep", "Joincol", "Help", "Dump", "NextColor", "Debug" }; -const leader_keys = "abcdefghijklmnopqrstuvwxyz0123456789$^"; -const typed = "abcdefghijklmnopqrstuvwxyz0123456789 -_=+,:;()"; - -fn generate(gpa: std.mem.Allocator, seed: u64, steps: usize) ![]u8 { - var prng = std.Random.DefaultPrng.init(seed); - const random = prng.random(); - var s: Script = .{ .gpa = gpa }; - try s.line("# pardes-monkey seed {d}, {d} steps", .{ seed, steps }); - try s.line("file bad.txt hello \\xff\\xfe world\\nlatin-1 caf\\xe9 tail\\ntruncated \\xe6\\x97 here\\nend\\n", .{}); - try s.line("file wide.txt {s}tail\\nsecond line\\n", .{std.mem.asBytes(&@as([8][37]u8, @splat("\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e ".*)))}); - try s.line("lines long.txt 400 some words after the number, long enough to wrap in a narrow pane", .{}); - try s.line("start 24 80 bad.txt", .{}); - try s.line("wait 8000 Newcol Joincol", .{}); - try s.line("stable 300 5000", .{}); - for (0..steps) |step| { - const wide = s.cols >= 70 and s.rows >= 6; - switch (random.uintLessThan(u8, 12)) { - // A workspace word, run with a middle click: new columns make - // narrow panes, Find and Grep open prompts. - 0, 1 => if (wide) { - const word = commands[random.uintLessThan(usize, commands.len)]; - try s.line("press middle @{s} 1", .{word}); - try s.line("release middle @{s} 1", .{word}); - }, - // Typing: into a prompt when one is up, far past its band. - 2, 3 => { - var text: [200]u8 = undefined; - const n = random.intRangeAtMost(usize, 1, if (random.uintLessThan(u8, 4) == 0) text.len else 12); - for (text[0..n]) |*c| c.* = typed[random.uintLessThan(usize, typed.len)]; - try s.line("text {s}", .{text[0..n]}); - }, - 4 => try s.line("key {s}", .{switch (random.uintLessThan(u8, 10)) { - 0 => "enter", - 1 => "tab", - 2 => "bs", - 3 => "up", - 4 => "down", - 5 => "left", - 6 => "right", - 7 => "pgdn", - 8 => "home", - else => "end", - }}), - 5 => { - try s.line("key esc", .{}); - try s.line("settle 120", .{}); - }, - // A leader burst: space and a chord's keys. - 6 => { - var chord: [3][2]u8 = undefined; - const n = random.intRangeAtMost(usize, 1, 3); - for (chord[0..n]) |*c| c.* = .{ leader_keys[random.uintLessThan(usize, leader_keys.len)], 0 }; - try s.out.appendSlice(gpa, "key space"); - for (chord[0..n]) |c| try s.out.print(gpa, " {c}", .{c[0]}); - try s.out.append(gpa, '\n'); - }, - 7, 8 => { - const buttons = [_][]const u8{ "left", "middle", "right" }; - const button = buttons[random.uintLessThan(usize, buttons.len)]; - const col = random.intRangeAtMost(u16, 1, s.cols); - const row = random.intRangeAtMost(u16, 1, s.rows); - try s.line("press {s} {d} {d}", .{ button, col, row }); - if (random.boolean()) try s.line("drag {s} {d} {d}", .{ button, random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }); - try s.line("release {s} {d} {d}", .{ button, random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }); - }, - 9 => try s.line("wheel {s} {d} {d}", .{ if (random.boolean()) "up" else "down", random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }), - 10 => try s.line("motion {d} {d}", .{ random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }), - // Tiny windows as often as ordinary ones. - else => { - s.rows = if (random.boolean()) random.intRangeAtMost(u16, 1, 6) else random.intRangeAtMost(u16, 7, 40); - s.cols = if (random.boolean()) random.intRangeAtMost(u16, 1, 20) else random.intRangeAtMost(u16, 21, 140); - try s.line("resize {d} {d}", .{ s.rows, s.cols }); - }, - } - try s.line("settle 30", .{}); - if (step % 10 == 9) try s.line("snap s{d}", .{step}); - } - try s.line("stable 300 5000", .{}); - try s.line("snap end", .{}); - return s.out.toOwnedSlice(gpa); -} - -/// A stack frame of a Zig panic trace, `.zig:12:34: 0x`, of the app's: the -/// harness's own error trace (a script it could not finish) is not a hit. -fn hasTrace(bytes: []const u8) bool { - var lines = std.mem.splitScalar(u8, bytes, '\n'); - while (lines.next()) |line| { - if (std.mem.indexOf(u8, line, "test/snapshot.zig") != null or std.mem.indexOf(u8, line, "test/e2e_harness.zig") != null) continue; - if (frameIn(line)) return true; - } - return false; -} - -fn frameIn(bytes: []const u8) bool { - var from: usize = 0; - while (std.mem.indexOfPos(u8, bytes, from, ".zig:")) |at| : (from = at + 1) { - var i = at + ".zig:".len; - var digits: usize = 0; - while (i < bytes.len and std.ascii.isDigit(bytes[i])) : (i += 1) digits += 1; - if (digits == 0 or i >= bytes.len or bytes[i] != ':') continue; - i += 1; - digits = 0; - while (i < bytes.len and std.ascii.isDigit(bytes[i])) : (i += 1) digits += 1; - if (digits > 0 and std.mem.startsWith(u8, bytes[i..], ": 0x")) return true; - } - return false; -} - -test "a panic trace is found, and ordinary colons are not" { - try std.testing.expect(hasTrace(" /src/draw.zig:814:31: 0x4212821 in joinNotice")); - try std.testing.expect(!hasTrace("draw.zig:814 is where it was; 12:30: 0x")); - try std.testing.expect(!hasTrace("/x/test/snapshot.zig:197:28: 0x1279761 in main (snapshot.zig)")); -} - -test "a seed makes the same script every time" { - const a = try generate(std.testing.allocator, 42, 50, 3); - defer std.testing.allocator.free(a); - const b = try generate(std.testing.allocator, 42, 50, 3); - defer std.testing.allocator.free(b); - try std.testing.expectEqualStrings(a, b); -} - -fn readAll(gpa: std.mem.Allocator, io: std.Io, path: []const u8) []u8 { - return std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(64 << 20)) catch &.{}; -} - -pub fn main(init: std.process.Init) !void { - const gpa = std.heap.page_allocator; - const io = init.io; - const args = try init.minimal.args.toSlice(init.arena.allocator()); - if (args.len < 4) { - std.debug.print("usage: pardes-monkey [steps] [--from=N] [--out=DIR] [--keep] [--tape=DIR] [--dump]\n", .{}); - std.process.exit(2); - } - const snap = args[1]; - const app = args[2]; - var seeds: u64 = 20; - var steps: usize = 70; - var from: u64 = 1; - var out: []const u8 = "/tmp/pardes-monkey"; - // Keep every script and capture, not only a hit's: to see what it did. - var keep = false; - // --tape=DIR: each seed's sessions record their tapes there and the - // seed is a hit if one does not replay (pardes-snap --tape). - var tape: ?[]const u8 = null; - // --dump (with --tape): each replay loads the session's dump at its end - // too (pardes-snap --dump). - var dump = false; - var positional: usize = 0; - for (args[3..]) |a| { - if (std.mem.startsWith(u8, a, "--from=")) { - from = try std.fmt.parseInt(u64, a["--from=".len..], 10); - } else if (std.mem.eql(u8, a, "--keep")) { - keep = true; - } else if (std.mem.startsWith(u8, a, "--tape=")) { - tape = a; - } else if (std.mem.eql(u8, a, "--dump")) { - dump = true; - } else if (std.mem.startsWith(u8, a, "--out=")) { - out = a["--out=".len..]; - } else { - switch (positional) { - 0 => seeds = try std.fmt.parseInt(u64, a, 10), - 1 => steps = try std.fmt.parseInt(usize, a, 10), - else => {}, - } - positional += 1; - } - } - std.Io.Dir.cwd().createDirPath(io, out) catch {}; - var hits: usize = 0; - var seed = from; - while (seed < from + seeds) : (seed += 1) { - const script = try generate(gpa, seed, steps); - defer gpa.free(script); - var path_buf: [4096]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/seed-{d}.snap", .{ out, seed }); - try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = script }); - const run = std.process.run(gpa, io, .{ - .argv = if (tape) |t| (if (dump) &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, "--dump", path } else &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, path }) else &.{ snap, app, "--update", "--no-retry", "--jobs=1", path }, - .stdout_limit = .limited(64 << 20), - .stderr_limit = .limited(64 << 20), - }) catch |err| { - std.debug.print("seed {d}: pardes-snap did not run ({t})\n", .{ seed, err }); - continue; - }; - defer gpa.free(run.stdout); - defer gpa.free(run.stderr); - var golden_buf: [4096]u8 = undefined; - const golden_path = try std.fmt.bufPrint(&golden_buf, "{s}/seed-{d}.golden", .{ out, seed }); - const golden = readAll(gpa, io, golden_path); - defer gpa.free(golden); - // The app's own crash file (src/crash.zig) in the script's fake home: - // found even when the crash cut the script short of a capture. - var crash_buf: [4096]u8 = undefined; - const crash_path = try std.fmt.bufPrint(&crash_buf, "/tmp/pardes-snap/seed-{d}/home/.config/pardes/crashes", .{seed}); - const crashes = readAll(gpa, io, crash_path); - defer gpa.free(crashes); - const diverged = tape != null and (std.mem.indexOf(u8, run.stdout, "DIVERGED") != null or std.mem.indexOf(u8, run.stderr, "DIVERGED") != null); - if (crashes.len != 0 or diverged or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) { - hits += 1; - var log_buf: [4096]u8 = undefined; - const log_path = try std.fmt.bufPrint(&log_buf, "{s}/seed-{d}.log", .{ out, seed }); - const both = try std.mem.concat(gpa, u8, &.{ crashes, run.stdout, run.stderr }); - defer gpa.free(both); - try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = log_path, .data = both }); - std.debug.print("{s} seed {d}: {s}\n", .{ if (diverged) "DIVERGED" else "PANIC", seed, path }); - } else if (!keep) { - std.Io.Dir.cwd().deleteFile(io, path) catch {}; - std.Io.Dir.cwd().deleteFile(io, golden_path) catch {}; - std.debug.print("ok seed {d}\n", .{seed}); - } else { - std.debug.print("ok seed {d} (kept)\n", .{seed}); - } - } - std.debug.print("{d} of {d} seeds panicked; kept in {s}\n", .{ hits, seeds, out }); - if (hits != 0) std.process.exit(1); -} diff --git a/test/monkey9p.zig b/test/monkey9p.zig index 39a1a187..c9dd5083 100644 --- a/test/monkey9p.zig +++ b/test/monkey9p.zig @@ -4,8 +4,19 @@ //! /log said meanwhile. It was test/monkey9p.py, which drove a real //! `pardes --detach` over its socket. //! -//! zig build monkey-session -Dplatform=tui -- [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [-v] -//! zig build monkey-session-smoke -Dplatform=tui (part of behaviour-test) +//! zig build behaviour-test -Dplatform=tui (a short fixed-seed run) +//! zig build behaviour-test -Dplatform=tui -- [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [-v] +//! zig build behaviour-test -Dplatform=tui -- --monkey [--steps M] +//! zig build behaviour-test -Dplatform=tui -- --replay [--monkey ] +//! +//! The T1 runner's modes: this, the rules over random 9P requests; with +//! `--monkey`, random input (keys, the mouse, resizes) hunting panics and +//! hangs, as test/monkey.zig's random snapshot scripts did through a real +//! pty; with `--replay`, those seeds (20 unless named) each recording the +//! tape of what its sessions consumed and replaying it, as `PARDES_REPLAY` +//! replays a real session's: into two replicas at different addresses and +//! one that never draws, every digest checked, and each session's dump +//! loaded to its digest and dumped to its bytes again (src/tape.zig). //! //! A step's op is drawn from the seed alone: the panes, columns and held //! files it names are tokens (`{p3}`: /index's 4th serial, mod its length, @@ -45,6 +56,9 @@ const write_max = msize - 23; const room = msize - 24; /// A step that has not finished in this long is a hang. const hang_ms = 30_000; +/// What the runs are (see the top): the rules, the input, or the input +/// with every session's tape replayed. +var run_mode: enum { rules, input, replay } = .rules; const fixtures = [_][2][]const u8{ .{ "alpha.txt", "alpha beta gamma\nsecond line\n\nthird after a blank\n" }, @@ -71,12 +85,11 @@ const words = [_][]const u8{ }; const shell_lines = [_][]const u8{ "true", "false", "echo hi", "echo m9p", "sleep 1", "ls", "cat nonexist", "printf abc", "exit 3", "seq 1 2000", "yes | head -n 3000", "date", "pwd", "env | wc" }; const builtins = [_][]const u8{ - "Look", "Exec", "Kill", "Dump", "Msg", "NextColor", "Themes", "DumpThemes", "TreeContext", "PdfFit", "PdfTint", "PdfSections", "Petscii", - "Palette", "Ascii", "Save", "New", "Newcol", "Del", "Filter", "Mode", "Togglettymode", "Edit", "Undo", "Redo", "Collapse", - "Delcol", "DelAbove", "DelBelow", "Term", "Repl", "Joincol", "Config", "LocationsConfig", "Messages", "Mini", "Find", "Grep", "Left", - "Down", "Up", "Right", "Back", "Forward", "Last", "Recent", "Jumplist", "Wrap", "Debug", - "Verbose", "InactiveDim", "DumpDir", "Theme", "Get", "Put", "Undo", "Redo", "Zerox", "Snarf", "Paste", "Cut", - "Sort", "Font", "Local", + "Look", "Exec", "Kill", "Dump", "Msg", "NextColor", "Themes", "DumpThemes", "TreeContext", "PdfFit", "PdfTint", "PdfSections", "Petscii", + "Palette", "Ascii", "Save", "New", "Newcol", "Del", "Filter", "Mode", "Togglettymode", "Edit", "Undo", "Redo", "Collapse", + "Delcol", "DelAbove", "DelBelow", "Term", "Repl", "Joincol", "Config", "LocationsConfig", "Messages", "Mini", "Find", "Grep", "Left", + "Down", "Up", "Right", "Back", "Forward", "Last", "Recent", "Jumplist", "Wrap", "Debug", "Verbose", "InactiveDim", "DumpDir", + "Theme", "Get", "Put", "Undo", "Redo", "Zerox", "Snarf", "Paste", "Cut", "Sort", "Font", "Local", }; const pane_files = [_][]const u8{ "name", "body", "tag", "ctl", "addr", "dot", "limit", "data", "xdata", "sel", "dirty", "mark", "scroll", "errors", "event", "look", "exec", "tagexec", "pty", "pty/run", "pty/data", "pty/ctl", "pty/status" }; const root_files = [_][]const u8{ "README", "index", "status", "focus", "ctl", "commands", "recent", "dumps", "look", "exec", "log", "screen", "listeners", "layout", "tag", "tagexec", "pane", "pane/new", "col", "" }; @@ -364,8 +377,75 @@ const Op = union(enum) { /// Down to the empty window (every pane closed but `keep`, the columns /// too), sometimes rebuilt with a new pane. clear: struct { keep: ?usize, cols: bool, rebuild: bool }, + /// The input modes' (genInput): runes at /kbdin, lines at /mousein, a + /// word on the screen clicked with the middle button (a workspace + /// word run), the window resized to `[cols, rows]`. + keys: []const u8, + mouse: []const u8, + word: []const u8, + resize: [2]u16, }; +// --------------------------------------------------------------------------- +// The tty monkey's input (--monkey): it was test/monkey.zig, random +// snapshot scripts through a real pty hunting panics. Keys and leader +// bursts, typing (sometimes far past a prompt's band), the mouse's presses, +// drags, wheel and hover, and windows as tiny as ordinary ones, at the +// fixtures' panes (bad UTF-8, wide glyphs). A hit is a panic or a hang, +// which the parent reports and shrinks as any failure; the rules are not +// checked between its steps. + +/// Workspace words, run with a middle click: new columns make narrow +/// panes, Find and Grep open prompts. +const monkey_words = [_][]const u8{ "Newcol", "Find", "Grep", "Joincol", "Help", "Dump", "NextColor", "Debug" }; +const leader_keys = "abcdefghijklmnopqrstuvwxyz0123456789$^"; +const typed = "abcdefghijklmnopqrstuvwxyz0123456789 -_=+,:;()"; +/// Enter, Tab, Backspace, the arrows, Page Down, Home and End as kbdin's +/// runes (docs/typ/reference.typ, Other files). +const named_keys = [_][]const u8{ "\n", "\t", "\x08", "\u{f00e}", "\u{f800}", "\u{f011}", "\u{f012}", "\u{f013}", "\u{f00d}", "\u{f018}" }; + +fn genInput(r: Random, a: Allocator, size: *[2]u16) Op { + const cols = size[0]; + const rows = size[1]; + switch (below(r, 12)) { + 0, 1 => if (cols >= 70 and rows >= 6) return .{ .word = pick(r, &monkey_words) }, + 2, 3 => { + const n = range(r, 1, if (below(r, 4) == 0) 201 else 13); + const text = a.alloc(u8, n) catch oom(); + for (text) |*c| c.* = typed[below(r, typed.len)]; + return .{ .keys = text }; + }, + 4 => return .{ .keys = pick(r, &named_keys) }, + 5 => return .{ .keys = "\x1b" }, + 6 => { + var chord: std.ArrayList(u8) = .empty; + chord.append(a, ' ') catch oom(); + for (0..range(r, 1, 4)) |_| chord.append(a, leader_keys[below(r, leader_keys.len)]) catch oom(); + return .{ .keys = chord.items }; + }, + 7, 8 => { + const button: u8 = pick(r, &[_]u8{ 1, 2, 4 }); + var lines: std.ArrayList(u8) = .empty; + lines.print(a, "m {d} {d} {d}\n", .{ below(r, cols), below(r, rows), button }) catch oom(); + if (r.boolean()) lines.print(a, "m {d} {d} {d}\n", .{ below(r, cols), below(r, rows), button }) catch oom(); + lines.print(a, "m {d} {d} 0\n", .{ below(r, cols), below(r, rows) }) catch oom(); + return .{ .mouse = lines.items }; + }, + 9 => { + const x = below(r, cols); + const y = below(r, rows); + return .{ .mouse = print(a, "m {d} {d} {d}\nm {d} {d} 0\n", .{ x, y, @as(u8, if (r.boolean()) 8 else 16), x, y }) }; + }, + 10 => return .{ .mouse = print(a, "m {d} {d} 0\n", .{ below(r, cols), below(r, rows) }) }, + else => { + size[1] = @intCast(if (r.boolean()) range(r, 1, 7) else range(r, 7, 41)); + size[0] = @intCast(if (r.boolean()) range(r, 1, 21) else range(r, 21, 141)); + return .{ .resize = size.* }; + }, + } + return .{ .keys = "" }; +} + fn seq(a: Allocator, ops: []const Op) Op { return .{ .seq = a.dupe(Op, ops) catch oom() }; } @@ -522,11 +602,22 @@ const Ctx = struct { step: usize = 0, fail_kind: []const u8 = "", fail_why: []const u8 = "", + /// Sessions whose tape did not replay (--replay). + diverged: usize = 0, fn s(c: *Ctx) *Session { return &c.sess.?; } + /// The tape of the session just ended replayed (--replay): a + /// divergence is a failure at the step it ended. + fn replayed(c: *Ctx, at_step: usize) void { + if (run_mode != .replay or Session.replayTape(c.gpa)) return; + c.diverged += 1; + std.debug.print("[seed {d} step {d}] FAIL replay: a replica diverged\n", .{ c.seed, at_step }); + std.debug.print("mky fail {d} replay:diverged\n", .{at_step -| 1}); + } + fn fail(c: *Ctx, a: Allocator, kind: []const u8, comptime f: []const u8, args: anytype) error{Fail} { c.fail_kind = kind; c.fail_why = print(a, f, args); @@ -543,7 +634,10 @@ const Ctx = struct { /// close does not end it), the session's own first pane removed, and /// half the time a follower of the log. fn fresh(c: *Ctx, at_step: usize) !void { - if (c.sess) |*old| old.deinit(); + if (c.sess) |*old| { + old.deinit(); + c.replayed(at_step); + } c.sess = null; _ = c.life.reset(.free_all); c.serials = .empty; @@ -556,7 +650,10 @@ const Ctx = struct { const la = c.life.allocator(); for (fixtures) |fx| try putFile(la, c.root, fx[0], fx[1]); removeFile(la, c.root, "fresh.txt"); - c.sess = try Session.initFile(c.gpa, fixtures[0][1]); + // The input modes start as `pardes` does, everything after on the + // tape: a file set in place before the first input is not. + if (run_mode == .replay) try Session.recordTape(c.gpa); + c.sess = try if (run_mode == .rules) Session.initFile(c.gpa, fixtures[0][1]) else Session.initTerm(c.gpa); const s_ = c.s(); const first = try serialsOf(la, (try readAll(c, la, "/index")) orelse ""); for (fixtures[0..3]) |fx| try s_.write("/look", print(la, "{s}/{s}\n", .{ c.root, fx[0] })); @@ -939,6 +1036,11 @@ fn exec(c: *Ctx, a: Allocator, res: *Res, op: Op) anyerror!void { .host => |o| if (o.data) |data| try putFile(a, c.root, o.name, data) else removeFile(a, c.root, o.name), .seq => |ops| for (ops) |sub| try exec(c, a, res, sub), .clear => unreachable, // resolved into a seq + // A refusal is an answer; a panic or a hang is what is hunted. + .keys => |typed_keys| _ = c.s().refused("/kbdin", typed_keys) catch {}, + .mouse => |lines| _ = c.s().refused("/mousein", lines) catch {}, + .word => |w| c.s().click(w, 0, 2) catch {}, + .resize => |size| c.s().resize(size[0], size[1]), } } @@ -1323,6 +1425,7 @@ fn runStep(c: *Ctx, a: Allocator, raw: Op) Fail!void { }; // Closing the session's last pane quits it (docs/typ/reference.typ). if (c.s().ended()) return error.Ended; + if (run_mode != .rules) return; checks(c, a, &res) catch |err| switch (err) { error.Fail => return error.Fail, else => return c.fail(a, "checker", "after {s}: {t}", .{ what, err }), @@ -1392,6 +1495,8 @@ fn fuzz(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: var step_arena: std.heap.ArenaAllocator = .init(gpa); var found: std.StringArrayHashMapUnmanaged(struct { n: usize, first: usize }) = .empty; try c.fresh(0); + // The window's size as the input's resizes leave it. + var size: [2]u16 = .{ 80, 24 }; const t0 = nowMs(); var ended: usize = 0; var ran: usize = 0; @@ -1399,7 +1504,7 @@ fn fuzz(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: _ = step_arena.reset(.retain_capacity); const a = step_arena.allocator(); c.step = step; - const op = gen(r, a); + const op = if (run_mode == .rules) gen(r, a) else genInput(r, a, &size); if (only) |list| if (std.mem.indexOfScalar(usize, list, step) == null) continue; std.debug.print("mky step {d}\n", .{step}); ran += 1; @@ -1425,8 +1530,11 @@ fn fuzz(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: const dt = @max(1, nowMs() - t0); std.debug.print("[seed {d}] {d} steps in {d} ms ({d}/s), {d} clean quit(s)\n", .{ seed, ran, dt, @divTrunc(@as(i64, @intCast(ran)) * 1000, dt), ended }); for (found.keys(), found.values()) |sig, v| std.debug.print(" {d}x {s} (first at step {d})\n", .{ v.n, sig[0..@min(sig.len, 200)], v.first }); - if (c.sess) |*s| s.deinit(); - return if (found.count() > 0) 1 else 0; + if (c.sess) |*s| { + s.deinit(); + c.replayed(steps); + } + return if (found.count() > 0 or c.diverged > 0) 1 else 0; } fn nowMs() i64 { @@ -1455,6 +1563,7 @@ fn spawn(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: var argv: std.ArrayList(?[*:0]const u8) = .empty; for ([_][]const u8{ "pardes-monkey-session", "--seed", print(a, "{d}", .{seed}), "--steps", print(a, "{d}", .{steps}) }) |arg| try argv.append(a, try a.dupeSentinel(u8, arg, 0)); if (verbose) try argv.append(a, "-v"); + if (run_mode != .rules) try argv.append(a, if (run_mode == .input) "--input-mode" else "--replay"); if (only) |list| { var text: std.ArrayList(u8) = .empty; for (list, 0..) |n, i| try text.print(a, "{s}{d}", .{ if (i > 0) "," else "", n }); @@ -1607,11 +1716,15 @@ fn shrink(gpa: Allocator, seed: u64, f: Found, budget: usize) !void { const small = try ddmin(gpa, body, &trial, Trial.fails); var text: std.ArrayList(u8) = .empty; for (small, 0..) |n, i| try text.print(gpa, "{s}{d}", .{ if (i > 0) "," else "", n }); - std.debug.print(" shrunk {d} -> {d} ops: zig build monkey-session -Dplatform=tui -- --seed {d} --steps {d} --only {s} -v\n", .{ body.len, small.len, seed, f.step + 1, text.items }); + std.debug.print(" shrunk {d} -> {d} ops: zig build behaviour-test -Dplatform=tui -- {s}--seed {d} --steps {d} --only {s} -v\n", .{ body.len, small.len, switch (run_mode) { + .rules => "", + .input => "--monkey 1 ", + .replay => "--replay ", + }, seed, f.step + 1, text.items }); } fn usage() noreturn { - std.debug.print("usage: pardes-monkey-session [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [--shrink-budget N] [--smoke] [-v]\n", .{}); + std.debug.print("usage: pardes-monkey-session [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [--shrink-budget N] [--monkey ] [--replay] [-v]\n", .{}); std.process.exit(2); } @@ -1623,8 +1736,11 @@ pub fn main(init: std.process.Init) !void { var only: ?[]const usize = null; var no_shrink = false; var budget: usize = 300; - var smoke = false; + // Bare, as behaviour-test runs it: a short fixed-seed run. + var smoke = args.len == 1; var verbose = false; + // --monkey's count of seeds, 1 to N, when none is named. + var monkey: usize = 0; var i: usize = 1; while (i < args.len) : (i += 1) { const arg = args[i]; @@ -1648,6 +1764,14 @@ pub fn main(init: std.process.Init) !void { no_shrink = true; } else if (std.mem.eql(u8, arg, "--smoke")) { smoke = true; + } else if (std.mem.eql(u8, arg, "--monkey")) { + if (run_mode == .rules) run_mode = .input; + monkey = std.fmt.parseInt(usize, value, 10) catch usage(); + i += 1; + } else if (std.mem.eql(u8, arg, "--input-mode")) { + run_mode = .input; + } else if (std.mem.eql(u8, arg, "--replay")) { + run_mode = .replay; } else if (std.mem.eql(u8, arg, "-v") or std.mem.eql(u8, arg, "--verbose")) { verbose = true; } else usage(); @@ -1671,6 +1795,11 @@ pub fn main(init: std.process.Init) !void { steps = steps orelse 800; no_shrink = true; } + // The input's seeds are 70 steps each, as the tty monkey's scripts were. + if (run_mode != .rules) { + if (seeds.items.len == 0) for (1..if (monkey > 0) monkey + 1 else 21) |n| try seeds.append(gpa, n); + steps = steps orelse 70; + } if (seeds.items.len == 0) try seeds.append(gpa, 1); var total: usize = 0; for (seeds.items) |seed| { @@ -1680,5 +1809,6 @@ pub fn main(init: std.process.Init) !void { var seen: std.StringHashMapUnmanaged(void) = .empty; for (found) |f| if (!(try seen.getOrPut(gpa, f.sig)).found_existing) try shrink(gpa, seed, f, budget); } + if (run_mode != .rules) std.debug.print("{s}: {d} seeds of {d} steps, {d} failure(s)\n", .{ @tagName(run_mode), seeds.items.len, steps.?, total }); if (total > 0) std.process.exit(1); } diff --git a/test/process_test.zig b/test/process_test.zig index bab80183..5876e005 100644 --- a/test/process_test.zig +++ b/test/process_test.zig @@ -5,7 +5,7 @@ //! its own instead (test/tty.zig), on the tty alone. //! //! pardes-process-test [--backend=tui|gui] [--addon=]... -//! [--lspmock=] [--update] [filter] +//! [--lspmock=] [--update] [--replay [--replay-with=]] [filter] const std = @import("std"); const process = @import("process.zig"); const tty = @import("tty.zig"); @@ -53,6 +53,10 @@ pub fn main(init: std.process.Init) !void { tty.lspmock = try std.Io.Dir.cwd().realPathFileAlloc(init.io, a["--lspmock=".len..], arena); } else if (std.mem.eql(u8, a, "--update")) { tty.update = true; + } else if (std.mem.eql(u8, a, "--replay")) { + tty.replay = true; + } else if (std.mem.startsWith(u8, a, "--replay-with=")) { + tty.replay_with = try std.Io.Dir.cwd().realPathFileAlloc(init.io, a["--replay-with=".len..], arena); } else if (n < args.len) { args[n] = a; n += 1; @@ -62,13 +66,13 @@ pub fn main(init: std.process.Init) !void { // Absolute: a tty case runs in a directory of its own. const exe = try std.Io.Dir.cwd().realPathFileAlloc(init.io, args[0], arena); const filter: []const u8 = if (n > 1) args[1] else ""; - tty.goldens = try std.Io.Dir.cwd().realPathFileAlloc(init.io, "test/snapshots", arena); + tty.goldens = std.Io.Dir.cwd().realPathFileAlloc(init.io, "test/snapshots", arena) catch tty.goldens; const gpa = std.heap.page_allocator; const on = @tagName(process.backend); var failed: usize = 0; var skipped: usize = 0; var ran: usize = 0; - if (!tty.update) inline for (files) |file| inline for (file.cases) |case| { + if (!tty.update and !tty.replay) inline for (files) |file| inline for (file.cases) |case| { if (std.mem.indexOf(u8, case[0], filter) != null) { ran += 1; const t0 = process.nowMs(); diff --git a/test/session.zig b/test/session.zig index b35d08ad..136d34d1 100644 --- a/test/session.zig +++ b/test/session.zig @@ -190,6 +190,37 @@ pub const Session = struct { return .{ try std.fmt.parseInt(u32, it.next() orelse return error.NoRange, 10), try std.fmt.parseInt(u32, it.next() orelse return error.NoRange, 10) }; } + /// The window resized to `cols` by `rows`, as a terminal's SIGWINCH or + /// a window's resize tells its host. + pub fn resize(s: *Session, cols: u16, rows: u16) void { + pardesOf(s).update(.{ .resize = .{ .cols = cols, .rows = rows } }); + s.drain(); + } + + /// Every session started from here on records what it consumes as a + /// tape in memory, as PARDES_RECORD has a real one record it + /// (src/tape.zig); `replayTape` replays it once the session is gone. + pub fn recordTape(gpa: std.mem.Allocator) !void { + const t = try gpa.create(pardes.tape.Tape); + t.* = pardes.tape.memory(gpa); + pardes.tape.on = t; + } + + /// The tape `recordTape` started, replayed as `PARDES_REPLAY` replays a + /// real session's: into two replicas at different addresses and one + /// that never draws, every digest checked, and the session's dump at + /// its end loaded to its digest and dumped to its bytes again. False + /// when a replica diverged (said on stderr). After `deinit`. + pub fn replayTape(gpa: std.mem.Allocator) bool { + const t = pardes.tape.on orelse return true; + pardes.tape.on = null; + defer { + pardes.tape.release(t); + gpa.destroy(t); + } + return pardes.tape.replayBytes(gpa, t.buf.items, "session", true); + } + /// No frontend attached, as a detached session's daemon says. pub fn detach(s: *Session) void { pardesOf(s).unattended = true; diff --git a/test/snapshot.zig b/test/snapshot.zig deleted file mode 100644 index 51c81dd9..00000000 --- a/test/snapshot.zig +++ /dev/null @@ -1,1753 +0,0 @@ -// pardes-snap: drive a pardes binary with a scripted input trace in a pty and -// capture the rendered grid at named sync points. The same script run against -// the prototype and the rewrite must produce byte-identical captures — this is -// the old-vs-new parity oracle for the rewrite. -// -// zig build snap run all snapshots/*.snap, diff goldens -// zig build snap -- --update regenerate goldens serially -// zig build snap -- snapshots/boot.snap one script -// zig build snap -- --jobs=1 serial (default: 1.25 per cpu) -// zig build snap -- --trace-stable show what every `stable` waited on -// -// Scripts run one per forked child, in parallel; each child's report is held -// in a temp file and printed in script order at the end. The runner takes its -// own lockfile, so `zig build snap` never needs an `flock` wrapper: a second -// run waits for the first instead of trampling the fixed /tmp paths that the -// goldens record verbatim. A script that fails in the parallel pass is re-run -// alone before it counts as a failure. -// -// `stable` no longer burns its wall-clock window: it returns as soon as the -// app is provably done — screen unchanged, pty empty, and every process it -// forked asleep (see waitStable). That, the fan-out, and building the vendored -// C optimized (build.zig's c_optimize: 90% of a Debug startup used to be -// tree-sitter's query analyser at -O0) took the suite from 7m54s to ~4s. The -// app now reaches first paint in ~25ms, so what is left is mostly real work: -// bash spawns, their output, and the 40 `settle`s after `key esc`. -// -// Script commands (line-oriented, # comments): -// file create file in the script's cwd (before start) -// lines [tail] create file with n numbered lines, tail on each -// dirmk create a subdirectory -// run run a command in the script's cwd (NOT in a pane) -// start [arg] fork the app in a pty (optional extra CLI arg) -// wait pump until needle appears on the grid (fails hard) -// settle pump for a fixed duration -// stable pump until the grid stops changing -// text send literal bytes -// send send with \e \r \n \t \\ \xNN escapes -// key enter esc tab bs space up down left right home end -// pgup pgdn c- a- (after esc: settle, or -// vaxis reads the next byte in the same batch and -// parses it as alt-. The scripts said 700ms and -// it cost the suite 28s; there is no timer in vaxis -// to outlast — 20ms passes, they use 100ms) -// press|release <[ctrl-]left|middle|right> SGR mouse, 1-based -// drag motion with button held -// motion button-less motion (hover) -// wheel -// resize -// snap