From 174e031985db910b4907ba324d6fd12c67898a5a Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Sun, 8 Nov 2026 23:49:04 -0300 Subject: tests: monkey and replay are modes of the T1 runner and snapshot.zig goes: behaviour-test runs test/monkey9p.zig bare (its smoke), `-- --monkey ` drives an in-process session with test/monkey.zig's input (keys, leader bursts, the mouse, tiny resizes, over /kbdin, /mousein and Session.resize) hunting panics and hangs, `-- --replay` records each seed's sessions as a tape in memory and replays it (tape.replayBytes, replayMain's loop: three replicas and the dump), and `process-test -- --replay` records and replays the tty cases' real sessions as `zig build replay` did (with the addons it lacked, so images passes); the snap, snap-driver-test, monkey, monkey-test (it no longer compiled), replay and monkey-session steps, pardes-snap and build/snap.zig (now build/process.zig) go; checked: a counter outside the core bumped at the 40th update diverges every T1 seed and 89 of 103 old tapes, one bumped on terminal output diverges no T1 seed but 69 tty cases and 70 old tapes (why the tty replay stays), a panic on a window under 5x5 is found by the T1 monkey (seeds 1 8 17 20) and the old one (seed 13) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge --- .gitignore | 6 +- build.zig | 46 +- build/process.zig | 87 +++ build/snap.zig | 168 ----- src/tape.zig | 39 +- test/monkey.zig | 254 ------- test/monkey9p.zig | 162 ++++- test/process_test.zig | 10 +- test/session.zig | 31 + test/snapshot.zig | 1753 ------------------------------------------------- test/tty.zig | 45 ++ 11 files changed, 363 insertions(+), 2238 deletions(-) create mode 100644 build/process.zig delete mode 100644 build/snap.zig delete mode 100644 test/monkey.zig delete mode 100644 test/snapshot.zig diff --git a/.gitignore b/.gitignore index fa5a6ee0..92bf54d9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,9 @@ zig-out* .zig-cache/ .zig-cache/* test/__pycache__/ -# a failing snapshot writes .actual beside its golden; it is the failure -# report, not a source of truth, and eleven had been committed by accident -test/snapshots/*.actual +# a failing macOS snapshot writes .actual beside its golden; it is the +# failure report, not a source of truth, and eleven had been committed by +# accident (a tty case writes what it drew under /tmp/pardes-snap instead) test/macos-snapshots/*.actual # a REAL rust-analyzer pointed at the client fixture (which its Cargo.toml # invites) runs cargo there, and cargo writes a target/ and a lock beside the diff --git a/build.zig b/build.zig index 1491ed35..e731f682 100644 --- a/build.zig +++ b/build.zig @@ -1,7 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); const mupdf_build = @import("mupdf.zig"); -const snap_build = @import("build/snap.zig"); +const process_build = @import("build/process.zig"); const grammar_manifest = @import("src/grammar_manifest.zig"); /// The package manifest, imported UNTYPED on purpose — see the `version` /// option in `shellOptions` for why that word is load-bearing. @@ -760,13 +760,13 @@ pub fn build(b: *std.Build) void { vaxis_mod.root_source_file = files.getDirectory().path(b, "main.zig"); } - var ghostty_vt_for_snap: ?*std.Build.Module = null; + var ghostty_vt_for_tests: ?*std.Build.Module = null; const ghostty_simd = !target.result.os.tag.isDarwin() or b.graph.host.result.os.tag.isDarwin(); const ghostty_optimize: std.lang.Optimize = if (optimize == .debug) .safe else optimize; const ghostty_dep = b.lazyDependency("ghostty", .{ .target = target, .optimize = ghostty_optimize, .simd = ghostty_simd, .@"app-runtime" = .none, .@"emit-xcframework" = false }); if (ghostty_dep) |dep| { const ghostty_vt = dep.module("ghostty-vt"); - ghostty_vt_for_snap = ghostty_vt; + ghostty_vt_for_tests = ghostty_vt; ghostty_vt.addImport("uucode", uucode_mod); patchGhosttyKitty(b, io, dep, ghostty_vt); ghostty_core_vt = ghostty_vt; @@ -1221,12 +1221,12 @@ pub fn build(b: *std.Build) void { } const addons: []const *std.Build.Step.Compile = if (pdf_addon) |l| b.allocator.dupe(*std.Build.Step.Compile, &.{ image_addon, l }) catch @panic("OOM") else &.{image_addon}; - snap_build.addTty(b, .{ + process_build.add(b, .{ .exe = exe, .addons = addons, .target = target, .optimize = optimize, - .ghostty_vt = ghostty_vt_for_snap, + .ghostty_vt = ghostty_vt_for_tests, .cloud9 = cloud9, .gui = platform == .gui, }); @@ -1240,7 +1240,7 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - if (ghostty_vt_for_snap) |vt| image_harness.root_module.addImport("ghostty-vt", vt); + if (ghostty_vt_for_tests) |vt| image_harness.root_module.addImport("ghostty-vt", vt); switch (target.result.os.tag) { .freebsd, .netbsd, .openbsd => image_harness.root_module.linkSystemLibrary("util", .{}), else => {}, @@ -1274,7 +1274,7 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - if (ghostty_vt_for_snap) |vt| kitty_harness.root_module.addImport("ghostty-vt", vt); + if (ghostty_vt_for_tests) |vt| kitty_harness.root_module.addImport("ghostty-vt", vt); switch (target.result.os.tag) { .freebsd, .netbsd, .openbsd => kitty_harness.root_module.linkSystemLibrary("util", .{}), else => {}, @@ -1428,7 +1428,7 @@ pub fn build(b: *std.Build) void { }); perf.root_module.addImport("pardes", core_module); // --live drives the real binary through a pty (test/e2e_harness.zig). - if (ghostty_vt_for_snap) |vt| perf.root_module.addImport("ghostty-vt", vt); + if (ghostty_vt_for_tests) |vt| perf.root_module.addImport("ghostty-vt", vt); perf.root_module.addOptions("perf_options", perf_options); const run_perf = b.addRunArtifact(perf); run_perf.has_side_effects = true; @@ -1588,7 +1588,7 @@ pub fn build(b: *std.Build) void { }); session_module.addImport("pardes", core_module); // The scripted language server its language words ask - // (Session.initLanguage), as the snapshot scripts' do. + // (Session.initLanguage), as the tty cases do. const session_options = b.addOptions(); session_options.addOptionPath("lspmock", b.addExecutable(.{ .name = "pardes-lspmock", @@ -1632,7 +1632,7 @@ pub fn build(b: *std.Build) void { // window's word knows which build it runs in. behaviour_module.addImport("pardes_config", core_module.import_table.get("pardes_config").?); const behaviour_tests = b.addTest(.{ .root_module = behaviour_module, .filters = test_filters }); - const behaviour_test_step = b.step("behaviour-test", "run the behavioural tests, through the 9P tree alone (test/behaviour/)"); + const behaviour_test_step = b.step("behaviour-test", "run the behavioural tests, through the 9P tree alone (test/behaviour/), and the T1 runner (-- [--seed N]... | --monkey | --replay)"); _ = units.add(b, behaviour_tests, behaviour_test_step); // An addon's own test: `--addon-test` over the fake (the template) // and the snake (a shipped example, built but not installed), each @@ -1659,10 +1659,12 @@ pub fn build(b: *std.Build) void { addon_test_step.dependOn(&compare.step); if (lib == pdf_addon) mupdf_check.dependOn(&compare.step); }; - // The 9P monkey at T1 (test/monkey9p.zig): random requests against - // an in-process session, the documented rules checked after each. - // Its fixed-seed smoke run is part of behaviour-test. - const monkey_session = b.addExecutable(.{ + // The T1 runner (test/monkey9p.zig): its modes over an in-process + // session, random 9P requests with the documented rules checked + // after each, random input hunting panics (`-- --monkey `), + // and its seeds' tapes replayed (`-- --replay`). Bare, a short + // fixed-seed run of the first, part of behaviour-test. + const t1_runner = b.addExecutable(.{ .name = "pardes-monkey-session", .root_module = b.createModule(.{ .target = target, @@ -1671,17 +1673,11 @@ pub fn build(b: *std.Build) void { .link_libc = true, }), }); - monkey_session.root_module.addImport("session", session_module); - const run_monkey_session = b.addRunArtifact(monkey_session); - run_monkey_session.addPassthruArgs(); - run_monkey_session.has_side_effects = true; - b.step("monkey-session", "random 9P requests against an in-process session, checking the documented rules (-- [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [-v])").dependOn(&run_monkey_session.step); - const run_monkey_session_smoke = b.addRunArtifact(monkey_session); - run_monkey_session_smoke.addArg("--smoke"); - run_monkey_session_smoke.has_side_effects = true; - const monkey_session_smoke_step = b.step("monkey-session-smoke", "a short fixed-seed run of monkey-session (part of behaviour-test)"); - monkey_session_smoke_step.dependOn(&run_monkey_session_smoke.step); - if (test_filters.len == 0) behaviour_test_step.dependOn(&run_monkey_session_smoke.step); + t1_runner.root_module.addImport("session", session_module); + const run_t1_runner = b.addRunArtifact(t1_runner); + run_t1_runner.addPassthruArgs(); + run_t1_runner.has_side_effects = true; + if (test_filters.len == 0) behaviour_test_step.dependOn(&run_t1_runner.step); // Kitty graphics conformance at the core: what a terminal pane's // program sends, what it reads back, what ghostty stores and what a diff --git a/build/process.zig b/build/process.zig new file mode 100644 index 00000000..bec37d65 --- /dev/null +++ b/build/process.zig @@ -0,0 +1,87 @@ +const std = @import("std"); + +/// The process tests (test/process_test.zig): real sessions over their +/// sockets, and the tty's own cases each in a pty of its own +/// (test/tty.zig), on the runner they share (test/process.zig); and the +/// window's own, run in full when asked. +pub fn add(b: *std.Build, opts: struct { + exe: *std.Build.Step.Compile, + /// The addons the build made (image, pdf), which every session loads: + /// each `--addon=`, linked into its data directory's addons. + addons: []const *std.Build.Step.Compile, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + ghostty_vt: ?*std.Build.Module, + cloud9: *std.Build.Module, + /// The build is the window's (-Dplatform=gui): the process tests drive it. + gui: bool = false, +}) void { + const ninep_client = b.createModule(.{ + .target = opts.target, + .optimize = opts.optimize, + .root_source_file = b.path("src/9p_io.zig"), + .link_libc = true, + }); + ninep_client.addImport("cloud9", opts.cloud9); + // The language server a tty case's `.rs` files speak to. + const lspmock = b.addExecutable(.{ + .name = "pardes-lspmock", + .root_module = b.createModule(.{ + .target = opts.target, + .optimize = opts.optimize, + .root_source_file = b.path("test/lspmock.zig"), + .link_libc = true, + }), + }); + const process_test = b.addExecutable(.{ + .name = "pardes-process-test", + .root_module = b.createModule(.{ + .target = opts.target, + .optimize = opts.optimize, + .root_source_file = b.path("test/process_test.zig"), + .link_libc = true, + }), + }); + if (opts.ghostty_vt) |vt| process_test.root_module.addImport("ghostty-vt", vt); + process_test.root_module.addImport("9p_io", ninep_client); + process_test.root_module.addImport("cloud9", opts.cloud9); + switch (opts.target.result.os.tag) { + .freebsd, .netbsd, .openbsd => process_test.root_module.linkSystemLibrary("util", .{}), + else => {}, + } + const run_process_test = b.addRunArtifact(process_test); + run_process_test.addArtifactArg(opts.exe); + for (opts.addons) |a| run_process_test.addPrefixedFileArg("--addon=", a.getEmittedBin()); + run_process_test.addPrefixedFileArg("--lspmock=", lspmock.getEmittedBin()); + run_process_test.addArg(if (opts.gui) "--backend=gui" else "--backend=tui"); + run_process_test.addPassthruArgs(); + // The tty cases' goldens are test/snapshots/'s, from the build root. + run_process_test.setCwd(b.path(".")); + run_process_test.has_side_effects = true; + b.step("process-test", "run the process tests: real sessions over their sockets, the tty's in a pty each (-- [--update] [filter])").dependOn(&run_process_test.step); + // The window's own, run in full when asked (test/process/window.zig): + // the monkey over every Motion, and the frames against + // this machine's goldens (PARDES_GUI_GOLDEN=update rewrites them). + const monkey_gui_step = b.step("monkey-gui", "random input to the hidden window across the effect settings (-Dplatform=gui; PARDES_GUI_MONKEY_SEED, _STEPS)"); + const gui_golden_step = b.step("gui-golden", "the hidden window's scenes against test/gui-goldens.txt, this machine's (-Dplatform=gui; PARDES_GUI_GOLDEN=update rewrites)"); + if (opts.gui) { + const run_monkey_gui = b.addRunArtifact(process_test); + run_monkey_gui.addArtifactArg(opts.exe); + for (opts.addons) |a| run_monkey_gui.addPrefixedFileArg("--addon=", a.getEmittedBin()); + run_monkey_gui.addArgs(&.{ "--backend=gui", "window: a monkey" }); + run_monkey_gui.setEnvironmentVariable("PARDES_GUI_MONKEY", "all"); + run_monkey_gui.has_side_effects = true; + monkey_gui_step.dependOn(&run_monkey_gui.step); + const run_gui_golden = b.addRunArtifact(process_test); + run_gui_golden.addArtifactArg(opts.exe); + for (opts.addons) |a| run_gui_golden.addPrefixedFileArg("--addon=", a.getEmittedBin()); + run_gui_golden.addArgs(&.{ "--backend=gui", "window: its scenes" }); + run_gui_golden.setEnvironmentVariable("PARDES_GUI_GOLDEN", b.graph.environ_map.get("PARDES_GUI_GOLDEN") orelse "check"); + run_gui_golden.setCwd(b.path(".")); + run_gui_golden.has_side_effects = true; + gui_golden_step.dependOn(&run_gui_golden.step); + } else { + monkey_gui_step.dependOn(&b.addFail("monkey-gui needs -Dplatform=gui").step); + gui_golden_step.dependOn(&b.addFail("gui-golden needs -Dplatform=gui").step); + } +} diff --git a/build/snap.zig b/build/snap.zig deleted file mode 100644 index e7f4069c..00000000 --- a/build/snap.zig +++ /dev/null @@ -1,168 +0,0 @@ -const std = @import("std"); - -pub fn addTty(b: *std.Build, opts: struct { - exe: *std.Build.Step.Compile, - /// The addons the build made (image, pdf), which every session loads: - /// each `--addon=`, linked into its data directory's addons. - addons: []const *std.Build.Step.Compile, - target: std.Build.ResolvedTarget, - optimize: std.builtin.OptimizeMode, - ghostty_vt: ?*std.Build.Module, - cloud9: *std.Build.Module, - /// The build is the window's (-Dplatform=gui): the process tests drive it. - gui: bool = false, -}) void { - const snap = b.addExecutable(.{ - .name = "pardes-snap", - .root_module = b.createModule(.{ - .target = opts.target, - .optimize = opts.optimize, - .root_source_file = b.path("test/snapshot.zig"), - .link_libc = true, - }), - }); - if (opts.ghostty_vt) |vt| snap.root_module.addImport("ghostty-vt", vt); - const ninep_client = b.createModule(.{ - .target = opts.target, - .optimize = opts.optimize, - .root_source_file = b.path("src/9p_io.zig"), - .link_libc = true, - }); - ninep_client.addImport("cloud9", opts.cloud9); - snap.root_module.addImport("9p_io", ninep_client); - snap.root_module.addImport("cloud9", opts.cloud9); - switch (opts.target.result.os.tag) { - .freebsd, .netbsd, .openbsd => snap.root_module.linkSystemLibrary("util", .{}), - else => {}, - } - const lspmock = b.addExecutable(.{ - .name = "pardes-lspmock", - .root_module = b.createModule(.{ - .target = opts.target, - .optimize = opts.optimize, - .root_source_file = b.path("test/lspmock.zig"), - .link_libc = true, - }), - }); - // The process tests (test/process_test.zig): real sessions over their - // sockets, on the runner the snap scripts share (test/process.zig). - const process_test = b.addExecutable(.{ - .name = "pardes-process-test", - .root_module = b.createModule(.{ - .target = opts.target, - .optimize = opts.optimize, - .root_source_file = b.path("test/process_test.zig"), - .link_libc = true, - }), - }); - if (opts.ghostty_vt) |vt| process_test.root_module.addImport("ghostty-vt", vt); - process_test.root_module.addImport("9p_io", ninep_client); - process_test.root_module.addImport("cloud9", opts.cloud9); - switch (opts.target.result.os.tag) { - .freebsd, .netbsd, .openbsd => process_test.root_module.linkSystemLibrary("util", .{}), - else => {}, - } - const run_process_test = b.addRunArtifact(process_test); - run_process_test.addArtifactArg(opts.exe); - for (opts.addons) |a| run_process_test.addPrefixedFileArg("--addon=", a.getEmittedBin()); - run_process_test.addPrefixedFileArg("--lspmock=", lspmock.getEmittedBin()); - run_process_test.addArg(if (opts.gui) "--backend=gui" else "--backend=tui"); - run_process_test.addPassthruArgs(); - // The tty cases' goldens are test/snapshots/'s, from the build root. - run_process_test.setCwd(b.path(".")); - run_process_test.has_side_effects = true; - b.step("process-test", "run the process tests: real sessions over their sockets, the tty's in a pty each (-- [--update] [filter])").dependOn(&run_process_test.step); - // The window's own, run in full when asked (test/process/window.zig): - // the monkey over every Motion, and the frames against - // this machine's goldens (PARDES_GUI_GOLDEN=update rewrites them). - const monkey_gui_step = b.step("monkey-gui", "random input to the hidden window across the effect settings (-Dplatform=gui; PARDES_GUI_MONKEY_SEED, _STEPS)"); - const gui_golden_step = b.step("gui-golden", "the hidden window's scenes against test/gui-goldens.txt, this machine's (-Dplatform=gui; PARDES_GUI_GOLDEN=update rewrites)"); - if (opts.gui) { - const run_monkey_gui = b.addRunArtifact(process_test); - run_monkey_gui.addArtifactArg(opts.exe); - for (opts.addons) |a| run_monkey_gui.addPrefixedFileArg("--addon=", a.getEmittedBin()); - run_monkey_gui.addArgs(&.{ "--backend=gui", "window: a monkey" }); - run_monkey_gui.setEnvironmentVariable("PARDES_GUI_MONKEY", "all"); - run_monkey_gui.has_side_effects = true; - monkey_gui_step.dependOn(&run_monkey_gui.step); - const run_gui_golden = b.addRunArtifact(process_test); - run_gui_golden.addArtifactArg(opts.exe); - for (opts.addons) |a| run_gui_golden.addPrefixedFileArg("--addon=", a.getEmittedBin()); - run_gui_golden.addArgs(&.{ "--backend=gui", "window: its scenes" }); - run_gui_golden.setEnvironmentVariable("PARDES_GUI_GOLDEN", b.graph.environ_map.get("PARDES_GUI_GOLDEN") orelse "check"); - run_gui_golden.setCwd(b.path(".")); - run_gui_golden.has_side_effects = true; - gui_golden_step.dependOn(&run_gui_golden.step); - } else { - monkey_gui_step.dependOn(&b.addFail("monkey-gui needs -Dplatform=gui").step); - gui_golden_step.dependOn(&b.addFail("gui-golden needs -Dplatform=gui").step); - } - const run_snap = b.addRunArtifact(snap); - run_snap.addArtifactArg(opts.exe); - run_snap.addPrefixedFileArg("--lspmock=", lspmock.getEmittedBin()); - for (opts.addons) |a| run_snap.addPrefixedFileArg("--addon=", a.getEmittedBin()); - run_snap.addPassthruArgs(); - // scripts fork children and drive real ptys: never cached, never "up to date" - run_snap.has_side_effects = true; - const snap_step = b.step("snap", "run the snapshot parity suite (-- --update regenerates goldens)"); - snap_step.dependOn(&run_snap.step); - // Random scripts through pardes-snap, hunting panics (test/monkey.zig): - // not a gate, it is slow and random over seeds; a seed replays. - const monkey = b.addExecutable(.{ - .name = "pardes-monkey", - .root_module = b.createModule(.{ - .target = opts.target, - .optimize = opts.optimize, - .root_source_file = b.path("test/monkey.zig"), - .link_libc = true, - }), - }); - const run_monkey = b.addRunArtifact(monkey); - run_monkey.addArtifactArg(snap); - run_monkey.addArtifactArg(opts.exe); - run_monkey.addPassthruArgs(); - run_monkey.has_side_effects = true; - b.step("monkey", "random snapshot scripts hunting panics (-- [steps] [--from=N] [--out=DIR])").dependOn(&run_monkey.step); - // The replay-divergence gate (src/tape.zig, docs/next-daemon.md): every - // snapshot script and `replay-seeds` monkey seeds with each session's - // tape recorded, each replayed into two replicas at different - // addresses and one that never draws, every digest checked; and at each - // session's end its dump loaded, which must be that session, digest for - // digest, and dump again to the same bytes (phase 2). - const tapes = ".zig-cache/tapes"; - const run_replay = b.addRunArtifact(snap); - run_replay.addArtifactArg(opts.exe); - run_replay.addPrefixedFileArg("--lspmock=", lspmock.getEmittedBin()); - run_replay.addArgs(&.{ "--tape=" ++ tapes, "--dump" }); - // Each script's tape replayed by another build too: a window's build - // replaying a terminal's sessions, as its replica of one does. - if (b.option([]const u8, "replay-with", "another build of pardes `zig build replay` replays every script's tape with too (a pardes-gui, say)")) |other| - run_replay.addArg(b.fmt("--replay-with={s}", .{other})); - run_replay.setCwd(b.path(".")); - run_replay.has_side_effects = true; - const seeds = b.option(u32, "replay-seeds", "monkey seeds `zig build replay` records and replays (default 20)") orelse 20; - const run_monkey_replay = b.addRunArtifact(monkey); - run_monkey_replay.addArtifactArg(snap); - run_monkey_replay.addArtifactArg(opts.exe); - run_monkey_replay.addArgs(&.{ b.fmt("{d}", .{seeds}), "70", "--tape=" ++ tapes ++ "/monkey", "--dump", "--out=" ++ tapes ++ "/monkey" }); - run_monkey_replay.setCwd(b.path(".")); - run_monkey_replay.has_side_effects = true; - // One pardes-snap run at a time (its lock): the scripts, then the seeds. - run_monkey_replay.step.dependOn(&run_replay.step); - // The same seeds with three terminals on the session, each a view of - // it, typed into in turn (views.zig). - const run_views_replay = b.addRunArtifact(monkey); - run_views_replay.addArtifactArg(snap); - run_views_replay.addArtifactArg(opts.exe); - run_views_replay.addArgs(&.{ b.fmt("{d}", .{seeds}), "70", "--views=3", "--tape=" ++ tapes ++ "/monkey-views", "--dump", "--out=" ++ tapes ++ "/monkey-views" }); - run_views_replay.setCwd(b.path(".")); - run_views_replay.has_side_effects = true; - run_views_replay.step.dependOn(&run_monkey_replay.step); - b.step("replay", "record every snapshot script's and -Dreplay-seeds monkey seeds' tapes (one terminal and three) and require each to replay without diverging, and each session's dump to load to its digest").dependOn(&run_views_replay.step); - const monkey_tests = b.addTest(.{ .root_module = monkey.root_module }); - b.step("monkey-test", "test the monkey's generator and panic detector").dependOn(&b.addRunArtifact(monkey_tests).step); - const run_driver_test = b.addRunArtifact(snap); - run_driver_test.addArg("--self-test"); - run_driver_test.has_side_effects = true; - b.step("snap-driver-test", "test snapshot retry evidence and failure reporting").dependOn(&run_driver_test.step); -} diff --git a/src/tape.zig b/src/tape.zig index 2354fb06..b1bcc31d 100644 --- a/src/tape.zig +++ b/src/tape.zig @@ -129,21 +129,20 @@ pub const Site = enum(u16) { /// listed is a compile error (`call`). An error union is listed as /// `anyerror!` its payload: an error goes by its name (walk.encode). pub const results = .{ - void, bool, - u32, i64, - [8]u8, ?u64, - []const u8, ?[]const u8, - ?std.Io.File.Kind, ?pardes.filesystem.Resolved, - pardes.filesystem.OsReadWrite, anyerror!pardes.filesystem.OsStat, - anyerror![]const u8, - anyerror![]u8, anyerror!void, - terminal.Told, ?pardes.ctlfs.pty.Printed, - @typeInfo(@TypeOf(terminal.greeting)).@"fn".return_type.?, + void, bool, + u32, i64, + [8]u8, ?u64, + []const u8, ?[]const u8, + ?std.Io.File.Kind, ?pardes.filesystem.Resolved, + pardes.filesystem.OsReadWrite, anyerror!pardes.filesystem.OsStat, + anyerror![]const u8, anyerror![]u8, + anyerror!void, terminal.Told, + ?pardes.ctlfs.pty.Printed, @typeInfo(@TypeOf(terminal.greeting)).@"fn".return_type.?, []pardes.Stream.Batch, - // What only the authority holds (`authority`): a closed pane's record, - // Recent's entry for a closed jump, the closed panes' names. - ?@import("Store.zig").Record, ?@import("recent.zig").Entry, - []const @import("Store.zig").Named, + // What only the authority holds (`authority`): a closed pane's record, + // Recent's entry for a closed jump, the closed panes' names. + ?@import("Store.zig").Record, + ?@import("recent.zig").Entry, []const @import("Store.zig").Named, @import("Kind.zig").Outcome, }; @@ -1253,6 +1252,14 @@ pub fn replayMain(io: std.Io, path: []const u8) u8 { std.debug.print("replay: cannot read {s}: {t}\n", .{ path, err }); return 2; }; + return if (replayBytes(gpa, bytes, path, libc.getenv("PARDES_REPLAY_DUMP") != null)) 0 else 1; +} + +/// `bytes`, a tape named `path`, replayed into three replicas, each digest +/// checked, and with `dump` the session's dump at its end loaded to its +/// digest and dumped to its bytes again: true when none diverged. An +/// in-process session's tape too (test/session.zig's replayTape). +pub fn replayBytes(gpa: std.mem.Allocator, bytes: []const u8, path: []const u8, dump: bool) bool { const runs = [_]struct { name: []const u8, headless: bool }{ .{ .name = "first", .headless = false }, // The first replica's memory is still held: this one's lies elsewhere. @@ -1262,7 +1269,7 @@ pub fn replayMain(io: std.Io, path: []const u8) u8 { var failed = false; for (runs) |run| { var t: Tape = .{ .gpa = gpa, .replay = true, .headless = run.headless, .scratch = .init(gpa), .arena = .init(gpa) }; - t.dump_check = std.mem.eql(u8, run.name, "first") and libc.getenv("PARDES_REPLAY_DUMP") != null; + t.dump_check = std.mem.eql(u8, run.name, "first") and dump; on = &t; defer on = null; _ = replayUnit(&t, bytes); @@ -1279,7 +1286,7 @@ pub fn replayMain(io: std.Io, path: []const u8) u8 { } else std.debug.print("replay {s}: {s} replica: {d} digests agree\n", .{ path, run.name, t.digests }); if (t.dump_check and t.diverged == null) std.debug.print("replay {s}: {d} dumps loaded to their digests and dumped to their bytes again ({d} bytes)\n", .{ path, t.dumps, t.dump_bytes }); } - return if (failed) 1 else 0; + return !failed; } test "a syscall that is no site, made replaying, diverges the replay and poisons a speculation" { diff --git a/test/monkey.zig b/test/monkey.zig deleted file mode 100644 index 243f2fbe..00000000 --- a/test/monkey.zig +++ /dev/null @@ -1,254 +0,0 @@ -// pardes-monkey: random snapshot scripts against the tty, hunting panics. -// -// zig build monkey -Dplatform=tui -- [steps] [--from=N] [--out=DIR] [--keep] -// -// Each seed makes one script (a fixed seed makes the same script, so a hit -// replays): keys and leader bursts, typing, mouse presses, drags and wheel, -// tiny resizes, prompts and long prompt text, over files of bad UTF-8 and -// wide glyphs. pardes-snap runs it with --update into DIR (default -// /tmp/pardes-monkey), and a panic is a record in the app's crash file -// (src/crash.zig, in the script's fake home: found even when the crash cut -// the script short) or a `.zig:N:N: 0x` stack frame in the report or the -// grid it captured (a trace renders INTO the screen). A hit keeps DIR/seed-N.snap, its captures and its report; the rest are deleted. -// A panic's trace can scroll off a small grid: replay the kept script, or -// run the app in a larger pty. Every hit gets a fix and a pinned regression -// script in test/snapshots/. -// -// Not a test gate: it is slow, random over seeds and aimed at finding, not -// holding. Shell text typed into a pane only ever runs in the throwaway -// script directory, and its alphabet has no path characters. -const std = @import("std"); -const libc = std.c; - -const Script = struct { - out: std.ArrayListUnmanaged(u8) = .empty, - gpa: std.mem.Allocator, - rows: u16 = 24, - cols: u16 = 80, - - fn line(s: *Script, comptime fmt: []const u8, args: anytype) !void { - try s.out.print(s.gpa, fmt ++ "\n", args); - } -}; - -// Workspace-tag words: clicked with the middle button they run, and the -// harness finds their column on row 1 while the window is wide enough. -const commands = [_][]const u8{ "Newcol", "Find", "Grep", "Joincol", "Help", "Dump", "NextColor", "Debug" }; -const leader_keys = "abcdefghijklmnopqrstuvwxyz0123456789$^"; -const typed = "abcdefghijklmnopqrstuvwxyz0123456789 -_=+,:;()"; - -fn generate(gpa: std.mem.Allocator, seed: u64, steps: usize) ![]u8 { - var prng = std.Random.DefaultPrng.init(seed); - const random = prng.random(); - var s: Script = .{ .gpa = gpa }; - try s.line("# pardes-monkey seed {d}, {d} steps", .{ seed, steps }); - try s.line("file bad.txt hello \\xff\\xfe world\\nlatin-1 caf\\xe9 tail\\ntruncated \\xe6\\x97 here\\nend\\n", .{}); - try s.line("file wide.txt {s}tail\\nsecond line\\n", .{std.mem.asBytes(&@as([8][37]u8, @splat("\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e ".*)))}); - try s.line("lines long.txt 400 some words after the number, long enough to wrap in a narrow pane", .{}); - try s.line("start 24 80 bad.txt", .{}); - try s.line("wait 8000 Newcol Joincol", .{}); - try s.line("stable 300 5000", .{}); - for (0..steps) |step| { - const wide = s.cols >= 70 and s.rows >= 6; - switch (random.uintLessThan(u8, 12)) { - // A workspace word, run with a middle click: new columns make - // narrow panes, Find and Grep open prompts. - 0, 1 => if (wide) { - const word = commands[random.uintLessThan(usize, commands.len)]; - try s.line("press middle @{s} 1", .{word}); - try s.line("release middle @{s} 1", .{word}); - }, - // Typing: into a prompt when one is up, far past its band. - 2, 3 => { - var text: [200]u8 = undefined; - const n = random.intRangeAtMost(usize, 1, if (random.uintLessThan(u8, 4) == 0) text.len else 12); - for (text[0..n]) |*c| c.* = typed[random.uintLessThan(usize, typed.len)]; - try s.line("text {s}", .{text[0..n]}); - }, - 4 => try s.line("key {s}", .{switch (random.uintLessThan(u8, 10)) { - 0 => "enter", - 1 => "tab", - 2 => "bs", - 3 => "up", - 4 => "down", - 5 => "left", - 6 => "right", - 7 => "pgdn", - 8 => "home", - else => "end", - }}), - 5 => { - try s.line("key esc", .{}); - try s.line("settle 120", .{}); - }, - // A leader burst: space and a chord's keys. - 6 => { - var chord: [3][2]u8 = undefined; - const n = random.intRangeAtMost(usize, 1, 3); - for (chord[0..n]) |*c| c.* = .{ leader_keys[random.uintLessThan(usize, leader_keys.len)], 0 }; - try s.out.appendSlice(gpa, "key space"); - for (chord[0..n]) |c| try s.out.print(gpa, " {c}", .{c[0]}); - try s.out.append(gpa, '\n'); - }, - 7, 8 => { - const buttons = [_][]const u8{ "left", "middle", "right" }; - const button = buttons[random.uintLessThan(usize, buttons.len)]; - const col = random.intRangeAtMost(u16, 1, s.cols); - const row = random.intRangeAtMost(u16, 1, s.rows); - try s.line("press {s} {d} {d}", .{ button, col, row }); - if (random.boolean()) try s.line("drag {s} {d} {d}", .{ button, random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }); - try s.line("release {s} {d} {d}", .{ button, random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }); - }, - 9 => try s.line("wheel {s} {d} {d}", .{ if (random.boolean()) "up" else "down", random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }), - 10 => try s.line("motion {d} {d}", .{ random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }), - // Tiny windows as often as ordinary ones. - else => { - s.rows = if (random.boolean()) random.intRangeAtMost(u16, 1, 6) else random.intRangeAtMost(u16, 7, 40); - s.cols = if (random.boolean()) random.intRangeAtMost(u16, 1, 20) else random.intRangeAtMost(u16, 21, 140); - try s.line("resize {d} {d}", .{ s.rows, s.cols }); - }, - } - try s.line("settle 30", .{}); - if (step % 10 == 9) try s.line("snap s{d}", .{step}); - } - try s.line("stable 300 5000", .{}); - try s.line("snap end", .{}); - return s.out.toOwnedSlice(gpa); -} - -/// A stack frame of a Zig panic trace, `.zig:12:34: 0x`, of the app's: the -/// harness's own error trace (a script it could not finish) is not a hit. -fn hasTrace(bytes: []const u8) bool { - var lines = std.mem.splitScalar(u8, bytes, '\n'); - while (lines.next()) |line| { - if (std.mem.indexOf(u8, line, "test/snapshot.zig") != null or std.mem.indexOf(u8, line, "test/e2e_harness.zig") != null) continue; - if (frameIn(line)) return true; - } - return false; -} - -fn frameIn(bytes: []const u8) bool { - var from: usize = 0; - while (std.mem.indexOfPos(u8, bytes, from, ".zig:")) |at| : (from = at + 1) { - var i = at + ".zig:".len; - var digits: usize = 0; - while (i < bytes.len and std.ascii.isDigit(bytes[i])) : (i += 1) digits += 1; - if (digits == 0 or i >= bytes.len or bytes[i] != ':') continue; - i += 1; - digits = 0; - while (i < bytes.len and std.ascii.isDigit(bytes[i])) : (i += 1) digits += 1; - if (digits > 0 and std.mem.startsWith(u8, bytes[i..], ": 0x")) return true; - } - return false; -} - -test "a panic trace is found, and ordinary colons are not" { - try std.testing.expect(hasTrace(" /src/draw.zig:814:31: 0x4212821 in joinNotice")); - try std.testing.expect(!hasTrace("draw.zig:814 is where it was; 12:30: 0x")); - try std.testing.expect(!hasTrace("/x/test/snapshot.zig:197:28: 0x1279761 in main (snapshot.zig)")); -} - -test "a seed makes the same script every time" { - const a = try generate(std.testing.allocator, 42, 50, 3); - defer std.testing.allocator.free(a); - const b = try generate(std.testing.allocator, 42, 50, 3); - defer std.testing.allocator.free(b); - try std.testing.expectEqualStrings(a, b); -} - -fn readAll(gpa: std.mem.Allocator, io: std.Io, path: []const u8) []u8 { - return std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(64 << 20)) catch &.{}; -} - -pub fn main(init: std.process.Init) !void { - const gpa = std.heap.page_allocator; - const io = init.io; - const args = try init.minimal.args.toSlice(init.arena.allocator()); - if (args.len < 4) { - std.debug.print("usage: pardes-monkey [steps] [--from=N] [--out=DIR] [--keep] [--tape=DIR] [--dump]\n", .{}); - std.process.exit(2); - } - const snap = args[1]; - const app = args[2]; - var seeds: u64 = 20; - var steps: usize = 70; - var from: u64 = 1; - var out: []const u8 = "/tmp/pardes-monkey"; - // Keep every script and capture, not only a hit's: to see what it did. - var keep = false; - // --tape=DIR: each seed's sessions record their tapes there and the - // seed is a hit if one does not replay (pardes-snap --tape). - var tape: ?[]const u8 = null; - // --dump (with --tape): each replay loads the session's dump at its end - // too (pardes-snap --dump). - var dump = false; - var positional: usize = 0; - for (args[3..]) |a| { - if (std.mem.startsWith(u8, a, "--from=")) { - from = try std.fmt.parseInt(u64, a["--from=".len..], 10); - } else if (std.mem.eql(u8, a, "--keep")) { - keep = true; - } else if (std.mem.startsWith(u8, a, "--tape=")) { - tape = a; - } else if (std.mem.eql(u8, a, "--dump")) { - dump = true; - } else if (std.mem.startsWith(u8, a, "--out=")) { - out = a["--out=".len..]; - } else { - switch (positional) { - 0 => seeds = try std.fmt.parseInt(u64, a, 10), - 1 => steps = try std.fmt.parseInt(usize, a, 10), - else => {}, - } - positional += 1; - } - } - std.Io.Dir.cwd().createDirPath(io, out) catch {}; - var hits: usize = 0; - var seed = from; - while (seed < from + seeds) : (seed += 1) { - const script = try generate(gpa, seed, steps); - defer gpa.free(script); - var path_buf: [4096]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/seed-{d}.snap", .{ out, seed }); - try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = script }); - const run = std.process.run(gpa, io, .{ - .argv = if (tape) |t| (if (dump) &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, "--dump", path } else &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, path }) else &.{ snap, app, "--update", "--no-retry", "--jobs=1", path }, - .stdout_limit = .limited(64 << 20), - .stderr_limit = .limited(64 << 20), - }) catch |err| { - std.debug.print("seed {d}: pardes-snap did not run ({t})\n", .{ seed, err }); - continue; - }; - defer gpa.free(run.stdout); - defer gpa.free(run.stderr); - var golden_buf: [4096]u8 = undefined; - const golden_path = try std.fmt.bufPrint(&golden_buf, "{s}/seed-{d}.golden", .{ out, seed }); - const golden = readAll(gpa, io, golden_path); - defer gpa.free(golden); - // The app's own crash file (src/crash.zig) in the script's fake home: - // found even when the crash cut the script short of a capture. - var crash_buf: [4096]u8 = undefined; - const crash_path = try std.fmt.bufPrint(&crash_buf, "/tmp/pardes-snap/seed-{d}/home/.config/pardes/crashes", .{seed}); - const crashes = readAll(gpa, io, crash_path); - defer gpa.free(crashes); - const diverged = tape != null and (std.mem.indexOf(u8, run.stdout, "DIVERGED") != null or std.mem.indexOf(u8, run.stderr, "DIVERGED") != null); - if (crashes.len != 0 or diverged or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) { - hits += 1; - var log_buf: [4096]u8 = undefined; - const log_path = try std.fmt.bufPrint(&log_buf, "{s}/seed-{d}.log", .{ out, seed }); - const both = try std.mem.concat(gpa, u8, &.{ crashes, run.stdout, run.stderr }); - defer gpa.free(both); - try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = log_path, .data = both }); - std.debug.print("{s} seed {d}: {s}\n", .{ if (diverged) "DIVERGED" else "PANIC", seed, path }); - } else if (!keep) { - std.Io.Dir.cwd().deleteFile(io, path) catch {}; - std.Io.Dir.cwd().deleteFile(io, golden_path) catch {}; - std.debug.print("ok seed {d}\n", .{seed}); - } else { - std.debug.print("ok seed {d} (kept)\n", .{seed}); - } - } - std.debug.print("{d} of {d} seeds panicked; kept in {s}\n", .{ hits, seeds, out }); - if (hits != 0) std.process.exit(1); -} diff --git a/test/monkey9p.zig b/test/monkey9p.zig index 39a1a187..c9dd5083 100644 --- a/test/monkey9p.zig +++ b/test/monkey9p.zig @@ -4,8 +4,19 @@ //! /log said meanwhile. It was test/monkey9p.py, which drove a real //! `pardes --detach` over its socket. //! -//! zig build monkey-session -Dplatform=tui -- [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [-v] -//! zig build monkey-session-smoke -Dplatform=tui (part of behaviour-test) +//! zig build behaviour-test -Dplatform=tui (a short fixed-seed run) +//! zig build behaviour-test -Dplatform=tui -- [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [-v] +//! zig build behaviour-test -Dplatform=tui -- --monkey [--steps M] +//! zig build behaviour-test -Dplatform=tui -- --replay [--monkey ] +//! +//! The T1 runner's modes: this, the rules over random 9P requests; with +//! `--monkey`, random input (keys, the mouse, resizes) hunting panics and +//! hangs, as test/monkey.zig's random snapshot scripts did through a real +//! pty; with `--replay`, those seeds (20 unless named) each recording the +//! tape of what its sessions consumed and replaying it, as `PARDES_REPLAY` +//! replays a real session's: into two replicas at different addresses and +//! one that never draws, every digest checked, and each session's dump +//! loaded to its digest and dumped to its bytes again (src/tape.zig). //! //! A step's op is drawn from the seed alone: the panes, columns and held //! files it names are tokens (`{p3}`: /index's 4th serial, mod its length, @@ -45,6 +56,9 @@ const write_max = msize - 23; const room = msize - 24; /// A step that has not finished in this long is a hang. const hang_ms = 30_000; +/// What the runs are (see the top): the rules, the input, or the input +/// with every session's tape replayed. +var run_mode: enum { rules, input, replay } = .rules; const fixtures = [_][2][]const u8{ .{ "alpha.txt", "alpha beta gamma\nsecond line\n\nthird after a blank\n" }, @@ -71,12 +85,11 @@ const words = [_][]const u8{ }; const shell_lines = [_][]const u8{ "true", "false", "echo hi", "echo m9p", "sleep 1", "ls", "cat nonexist", "printf abc", "exit 3", "seq 1 2000", "yes | head -n 3000", "date", "pwd", "env | wc" }; const builtins = [_][]const u8{ - "Look", "Exec", "Kill", "Dump", "Msg", "NextColor", "Themes", "DumpThemes", "TreeContext", "PdfFit", "PdfTint", "PdfSections", "Petscii", - "Palette", "Ascii", "Save", "New", "Newcol", "Del", "Filter", "Mode", "Togglettymode", "Edit", "Undo", "Redo", "Collapse", - "Delcol", "DelAbove", "DelBelow", "Term", "Repl", "Joincol", "Config", "LocationsConfig", "Messages", "Mini", "Find", "Grep", "Left", - "Down", "Up", "Right", "Back", "Forward", "Last", "Recent", "Jumplist", "Wrap", "Debug", - "Verbose", "InactiveDim", "DumpDir", "Theme", "Get", "Put", "Undo", "Redo", "Zerox", "Snarf", "Paste", "Cut", - "Sort", "Font", "Local", + "Look", "Exec", "Kill", "Dump", "Msg", "NextColor", "Themes", "DumpThemes", "TreeContext", "PdfFit", "PdfTint", "PdfSections", "Petscii", + "Palette", "Ascii", "Save", "New", "Newcol", "Del", "Filter", "Mode", "Togglettymode", "Edit", "Undo", "Redo", "Collapse", + "Delcol", "DelAbove", "DelBelow", "Term", "Repl", "Joincol", "Config", "LocationsConfig", "Messages", "Mini", "Find", "Grep", "Left", + "Down", "Up", "Right", "Back", "Forward", "Last", "Recent", "Jumplist", "Wrap", "Debug", "Verbose", "InactiveDim", "DumpDir", + "Theme", "Get", "Put", "Undo", "Redo", "Zerox", "Snarf", "Paste", "Cut", "Sort", "Font", "Local", }; const pane_files = [_][]const u8{ "name", "body", "tag", "ctl", "addr", "dot", "limit", "data", "xdata", "sel", "dirty", "mark", "scroll", "errors", "event", "look", "exec", "tagexec", "pty", "pty/run", "pty/data", "pty/ctl", "pty/status" }; const root_files = [_][]const u8{ "README", "index", "status", "focus", "ctl", "commands", "recent", "dumps", "look", "exec", "log", "screen", "listeners", "layout", "tag", "tagexec", "pane", "pane/new", "col", "" }; @@ -364,8 +377,75 @@ const Op = union(enum) { /// Down to the empty window (every pane closed but `keep`, the columns /// too), sometimes rebuilt with a new pane. clear: struct { keep: ?usize, cols: bool, rebuild: bool }, + /// The input modes' (genInput): runes at /kbdin, lines at /mousein, a + /// word on the screen clicked with the middle button (a workspace + /// word run), the window resized to `[cols, rows]`. + keys: []const u8, + mouse: []const u8, + word: []const u8, + resize: [2]u16, }; +// --------------------------------------------------------------------------- +// The tty monkey's input (--monkey): it was test/monkey.zig, random +// snapshot scripts through a real pty hunting panics. Keys and leader +// bursts, typing (sometimes far past a prompt's band), the mouse's presses, +// drags, wheel and hover, and windows as tiny as ordinary ones, at the +// fixtures' panes (bad UTF-8, wide glyphs). A hit is a panic or a hang, +// which the parent reports and shrinks as any failure; the rules are not +// checked between its steps. + +/// Workspace words, run with a middle click: new columns make narrow +/// panes, Find and Grep open prompts. +const monkey_words = [_][]const u8{ "Newcol", "Find", "Grep", "Joincol", "Help", "Dump", "NextColor", "Debug" }; +const leader_keys = "abcdefghijklmnopqrstuvwxyz0123456789$^"; +const typed = "abcdefghijklmnopqrstuvwxyz0123456789 -_=+,:;()"; +/// Enter, Tab, Backspace, the arrows, Page Down, Home and End as kbdin's +/// runes (docs/typ/reference.typ, Other files). +const named_keys = [_][]const u8{ "\n", "\t", "\x08", "\u{f00e}", "\u{f800}", "\u{f011}", "\u{f012}", "\u{f013}", "\u{f00d}", "\u{f018}" }; + +fn genInput(r: Random, a: Allocator, size: *[2]u16) Op { + const cols = size[0]; + const rows = size[1]; + switch (below(r, 12)) { + 0, 1 => if (cols >= 70 and rows >= 6) return .{ .word = pick(r, &monkey_words) }, + 2, 3 => { + const n = range(r, 1, if (below(r, 4) == 0) 201 else 13); + const text = a.alloc(u8, n) catch oom(); + for (text) |*c| c.* = typed[below(r, typed.len)]; + return .{ .keys = text }; + }, + 4 => return .{ .keys = pick(r, &named_keys) }, + 5 => return .{ .keys = "\x1b" }, + 6 => { + var chord: std.ArrayList(u8) = .empty; + chord.append(a, ' ') catch oom(); + for (0..range(r, 1, 4)) |_| chord.append(a, leader_keys[below(r, leader_keys.len)]) catch oom(); + return .{ .keys = chord.items }; + }, + 7, 8 => { + const button: u8 = pick(r, &[_]u8{ 1, 2, 4 }); + var lines: std.ArrayList(u8) = .empty; + lines.print(a, "m {d} {d} {d}\n", .{ below(r, cols), below(r, rows), button }) catch oom(); + if (r.boolean()) lines.print(a, "m {d} {d} {d}\n", .{ below(r, cols), below(r, rows), button }) catch oom(); + lines.print(a, "m {d} {d} 0\n", .{ below(r, cols), below(r, rows) }) catch oom(); + return .{ .mouse = lines.items }; + }, + 9 => { + const x = below(r, cols); + const y = below(r, rows); + return .{ .mouse = print(a, "m {d} {d} {d}\nm {d} {d} 0\n", .{ x, y, @as(u8, if (r.boolean()) 8 else 16), x, y }) }; + }, + 10 => return .{ .mouse = print(a, "m {d} {d} 0\n", .{ below(r, cols), below(r, rows) }) }, + else => { + size[1] = @intCast(if (r.boolean()) range(r, 1, 7) else range(r, 7, 41)); + size[0] = @intCast(if (r.boolean()) range(r, 1, 21) else range(r, 21, 141)); + return .{ .resize = size.* }; + }, + } + return .{ .keys = "" }; +} + fn seq(a: Allocator, ops: []const Op) Op { return .{ .seq = a.dupe(Op, ops) catch oom() }; } @@ -522,11 +602,22 @@ const Ctx = struct { step: usize = 0, fail_kind: []const u8 = "", fail_why: []const u8 = "", + /// Sessions whose tape did not replay (--replay). + diverged: usize = 0, fn s(c: *Ctx) *Session { return &c.sess.?; } + /// The tape of the session just ended replayed (--replay): a + /// divergence is a failure at the step it ended. + fn replayed(c: *Ctx, at_step: usize) void { + if (run_mode != .replay or Session.replayTape(c.gpa)) return; + c.diverged += 1; + std.debug.print("[seed {d} step {d}] FAIL replay: a replica diverged\n", .{ c.seed, at_step }); + std.debug.print("mky fail {d} replay:diverged\n", .{at_step -| 1}); + } + fn fail(c: *Ctx, a: Allocator, kind: []const u8, comptime f: []const u8, args: anytype) error{Fail} { c.fail_kind = kind; c.fail_why = print(a, f, args); @@ -543,7 +634,10 @@ const Ctx = struct { /// close does not end it), the session's own first pane removed, and /// half the time a follower of the log. fn fresh(c: *Ctx, at_step: usize) !void { - if (c.sess) |*old| old.deinit(); + if (c.sess) |*old| { + old.deinit(); + c.replayed(at_step); + } c.sess = null; _ = c.life.reset(.free_all); c.serials = .empty; @@ -556,7 +650,10 @@ const Ctx = struct { const la = c.life.allocator(); for (fixtures) |fx| try putFile(la, c.root, fx[0], fx[1]); removeFile(la, c.root, "fresh.txt"); - c.sess = try Session.initFile(c.gpa, fixtures[0][1]); + // The input modes start as `pardes` does, everything after on the + // tape: a file set in place before the first input is not. + if (run_mode == .replay) try Session.recordTape(c.gpa); + c.sess = try if (run_mode == .rules) Session.initFile(c.gpa, fixtures[0][1]) else Session.initTerm(c.gpa); const s_ = c.s(); const first = try serialsOf(la, (try readAll(c, la, "/index")) orelse ""); for (fixtures[0..3]) |fx| try s_.write("/look", print(la, "{s}/{s}\n", .{ c.root, fx[0] })); @@ -939,6 +1036,11 @@ fn exec(c: *Ctx, a: Allocator, res: *Res, op: Op) anyerror!void { .host => |o| if (o.data) |data| try putFile(a, c.root, o.name, data) else removeFile(a, c.root, o.name), .seq => |ops| for (ops) |sub| try exec(c, a, res, sub), .clear => unreachable, // resolved into a seq + // A refusal is an answer; a panic or a hang is what is hunted. + .keys => |typed_keys| _ = c.s().refused("/kbdin", typed_keys) catch {}, + .mouse => |lines| _ = c.s().refused("/mousein", lines) catch {}, + .word => |w| c.s().click(w, 0, 2) catch {}, + .resize => |size| c.s().resize(size[0], size[1]), } } @@ -1323,6 +1425,7 @@ fn runStep(c: *Ctx, a: Allocator, raw: Op) Fail!void { }; // Closing the session's last pane quits it (docs/typ/reference.typ). if (c.s().ended()) return error.Ended; + if (run_mode != .rules) return; checks(c, a, &res) catch |err| switch (err) { error.Fail => return error.Fail, else => return c.fail(a, "checker", "after {s}: {t}", .{ what, err }), @@ -1392,6 +1495,8 @@ fn fuzz(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: var step_arena: std.heap.ArenaAllocator = .init(gpa); var found: std.StringArrayHashMapUnmanaged(struct { n: usize, first: usize }) = .empty; try c.fresh(0); + // The window's size as the input's resizes leave it. + var size: [2]u16 = .{ 80, 24 }; const t0 = nowMs(); var ended: usize = 0; var ran: usize = 0; @@ -1399,7 +1504,7 @@ fn fuzz(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: _ = step_arena.reset(.retain_capacity); const a = step_arena.allocator(); c.step = step; - const op = gen(r, a); + const op = if (run_mode == .rules) gen(r, a) else genInput(r, a, &size); if (only) |list| if (std.mem.indexOfScalar(usize, list, step) == null) continue; std.debug.print("mky step {d}\n", .{step}); ran += 1; @@ -1425,8 +1530,11 @@ fn fuzz(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: const dt = @max(1, nowMs() - t0); std.debug.print("[seed {d}] {d} steps in {d} ms ({d}/s), {d} clean quit(s)\n", .{ seed, ran, dt, @divTrunc(@as(i64, @intCast(ran)) * 1000, dt), ended }); for (found.keys(), found.values()) |sig, v| std.debug.print(" {d}x {s} (first at step {d})\n", .{ v.n, sig[0..@min(sig.len, 200)], v.first }); - if (c.sess) |*s| s.deinit(); - return if (found.count() > 0) 1 else 0; + if (c.sess) |*s| { + s.deinit(); + c.replayed(steps); + } + return if (found.count() > 0 or c.diverged > 0) 1 else 0; } fn nowMs() i64 { @@ -1455,6 +1563,7 @@ fn spawn(gpa: Allocator, seed: u64, steps: usize, only: ?[]const usize, verbose: var argv: std.ArrayList(?[*:0]const u8) = .empty; for ([_][]const u8{ "pardes-monkey-session", "--seed", print(a, "{d}", .{seed}), "--steps", print(a, "{d}", .{steps}) }) |arg| try argv.append(a, try a.dupeSentinel(u8, arg, 0)); if (verbose) try argv.append(a, "-v"); + if (run_mode != .rules) try argv.append(a, if (run_mode == .input) "--input-mode" else "--replay"); if (only) |list| { var text: std.ArrayList(u8) = .empty; for (list, 0..) |n, i| try text.print(a, "{s}{d}", .{ if (i > 0) "," else "", n }); @@ -1607,11 +1716,15 @@ fn shrink(gpa: Allocator, seed: u64, f: Found, budget: usize) !void { const small = try ddmin(gpa, body, &trial, Trial.fails); var text: std.ArrayList(u8) = .empty; for (small, 0..) |n, i| try text.print(gpa, "{s}{d}", .{ if (i > 0) "," else "", n }); - std.debug.print(" shrunk {d} -> {d} ops: zig build monkey-session -Dplatform=tui -- --seed {d} --steps {d} --only {s} -v\n", .{ body.len, small.len, seed, f.step + 1, text.items }); + std.debug.print(" shrunk {d} -> {d} ops: zig build behaviour-test -Dplatform=tui -- {s}--seed {d} --steps {d} --only {s} -v\n", .{ body.len, small.len, switch (run_mode) { + .rules => "", + .input => "--monkey 1 ", + .replay => "--replay ", + }, seed, f.step + 1, text.items }); } fn usage() noreturn { - std.debug.print("usage: pardes-monkey-session [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [--shrink-budget N] [--smoke] [-v]\n", .{}); + std.debug.print("usage: pardes-monkey-session [--seed N]... [--steps M] [--only i,j,k] [--no-shrink] [--shrink-budget N] [--monkey ] [--replay] [-v]\n", .{}); std.process.exit(2); } @@ -1623,8 +1736,11 @@ pub fn main(init: std.process.Init) !void { var only: ?[]const usize = null; var no_shrink = false; var budget: usize = 300; - var smoke = false; + // Bare, as behaviour-test runs it: a short fixed-seed run. + var smoke = args.len == 1; var verbose = false; + // --monkey's count of seeds, 1 to N, when none is named. + var monkey: usize = 0; var i: usize = 1; while (i < args.len) : (i += 1) { const arg = args[i]; @@ -1648,6 +1764,14 @@ pub fn main(init: std.process.Init) !void { no_shrink = true; } else if (std.mem.eql(u8, arg, "--smoke")) { smoke = true; + } else if (std.mem.eql(u8, arg, "--monkey")) { + if (run_mode == .rules) run_mode = .input; + monkey = std.fmt.parseInt(usize, value, 10) catch usage(); + i += 1; + } else if (std.mem.eql(u8, arg, "--input-mode")) { + run_mode = .input; + } else if (std.mem.eql(u8, arg, "--replay")) { + run_mode = .replay; } else if (std.mem.eql(u8, arg, "-v") or std.mem.eql(u8, arg, "--verbose")) { verbose = true; } else usage(); @@ -1671,6 +1795,11 @@ pub fn main(init: std.process.Init) !void { steps = steps orelse 800; no_shrink = true; } + // The input's seeds are 70 steps each, as the tty monkey's scripts were. + if (run_mode != .rules) { + if (seeds.items.len == 0) for (1..if (monkey > 0) monkey + 1 else 21) |n| try seeds.append(gpa, n); + steps = steps orelse 70; + } if (seeds.items.len == 0) try seeds.append(gpa, 1); var total: usize = 0; for (seeds.items) |seed| { @@ -1680,5 +1809,6 @@ pub fn main(init: std.process.Init) !void { var seen: std.StringHashMapUnmanaged(void) = .empty; for (found) |f| if (!(try seen.getOrPut(gpa, f.sig)).found_existing) try shrink(gpa, seed, f, budget); } + if (run_mode != .rules) std.debug.print("{s}: {d} seeds of {d} steps, {d} failure(s)\n", .{ @tagName(run_mode), seeds.items.len, steps.?, total }); if (total > 0) std.process.exit(1); } diff --git a/test/process_test.zig b/test/process_test.zig index bab80183..5876e005 100644 --- a/test/process_test.zig +++ b/test/process_test.zig @@ -5,7 +5,7 @@ //! its own instead (test/tty.zig), on the tty alone. //! //! pardes-process-test [--backend=tui|gui] [--addon=]... -//! [--lspmock=] [--update] [filter] +//! [--lspmock=] [--update] [--replay [--replay-with=]] [filter] const std = @import("std"); const process = @import("process.zig"); const tty = @import("tty.zig"); @@ -53,6 +53,10 @@ pub fn main(init: std.process.Init) !void { tty.lspmock = try std.Io.Dir.cwd().realPathFileAlloc(init.io, a["--lspmock=".len..], arena); } else if (std.mem.eql(u8, a, "--update")) { tty.update = true; + } else if (std.mem.eql(u8, a, "--replay")) { + tty.replay = true; + } else if (std.mem.startsWith(u8, a, "--replay-with=")) { + tty.replay_with = try std.Io.Dir.cwd().realPathFileAlloc(init.io, a["--replay-with=".len..], arena); } else if (n < args.len) { args[n] = a; n += 1; @@ -62,13 +66,13 @@ pub fn main(init: std.process.Init) !void { // Absolute: a tty case runs in a directory of its own. const exe = try std.Io.Dir.cwd().realPathFileAlloc(init.io, args[0], arena); const filter: []const u8 = if (n > 1) args[1] else ""; - tty.goldens = try std.Io.Dir.cwd().realPathFileAlloc(init.io, "test/snapshots", arena); + tty.goldens = std.Io.Dir.cwd().realPathFileAlloc(init.io, "test/snapshots", arena) catch tty.goldens; const gpa = std.heap.page_allocator; const on = @tagName(process.backend); var failed: usize = 0; var skipped: usize = 0; var ran: usize = 0; - if (!tty.update) inline for (files) |file| inline for (file.cases) |case| { + if (!tty.update and !tty.replay) inline for (files) |file| inline for (file.cases) |case| { if (std.mem.indexOf(u8, case[0], filter) != null) { ran += 1; const t0 = process.nowMs(); diff --git a/test/session.zig b/test/session.zig index b35d08ad..136d34d1 100644 --- a/test/session.zig +++ b/test/session.zig @@ -190,6 +190,37 @@ pub const Session = struct { return .{ try std.fmt.parseInt(u32, it.next() orelse return error.NoRange, 10), try std.fmt.parseInt(u32, it.next() orelse return error.NoRange, 10) }; } + /// The window resized to `cols` by `rows`, as a terminal's SIGWINCH or + /// a window's resize tells its host. + pub fn resize(s: *Session, cols: u16, rows: u16) void { + pardesOf(s).update(.{ .resize = .{ .cols = cols, .rows = rows } }); + s.drain(); + } + + /// Every session started from here on records what it consumes as a + /// tape in memory, as PARDES_RECORD has a real one record it + /// (src/tape.zig); `replayTape` replays it once the session is gone. + pub fn recordTape(gpa: std.mem.Allocator) !void { + const t = try gpa.create(pardes.tape.Tape); + t.* = pardes.tape.memory(gpa); + pardes.tape.on = t; + } + + /// The tape `recordTape` started, replayed as `PARDES_REPLAY` replays a + /// real session's: into two replicas at different addresses and one + /// that never draws, every digest checked, and the session's dump at + /// its end loaded to its digest and dumped to its bytes again. False + /// when a replica diverged (said on stderr). After `deinit`. + pub fn replayTape(gpa: std.mem.Allocator) bool { + const t = pardes.tape.on orelse return true; + pardes.tape.on = null; + defer { + pardes.tape.release(t); + gpa.destroy(t); + } + return pardes.tape.replayBytes(gpa, t.buf.items, "session", true); + } + /// No frontend attached, as a detached session's daemon says. pub fn detach(s: *Session) void { pardesOf(s).unattended = true; diff --git a/test/snapshot.zig b/test/snapshot.zig deleted file mode 100644 index 51c81dd9..00000000 --- a/test/snapshot.zig +++ /dev/null @@ -1,1753 +0,0 @@ -// pardes-snap: drive a pardes binary with a scripted input trace in a pty and -// capture the rendered grid at named sync points. The same script run against -// the prototype and the rewrite must produce byte-identical captures — this is -// the old-vs-new parity oracle for the rewrite. -// -// zig build snap run all snapshots/*.snap, diff goldens -// zig build snap -- --update regenerate goldens serially -// zig build snap -- snapshots/boot.snap one script -// zig build snap -- --jobs=1 serial (default: 1.25 per cpu) -// zig build snap -- --trace-stable show what every `stable` waited on -// -// Scripts run one per forked child, in parallel; each child's report is held -// in a temp file and printed in script order at the end. The runner takes its -// own lockfile, so `zig build snap` never needs an `flock` wrapper: a second -// run waits for the first instead of trampling the fixed /tmp paths that the -// goldens record verbatim. A script that fails in the parallel pass is re-run -// alone before it counts as a failure. -// -// `stable` no longer burns its wall-clock window: it returns as soon as the -// app is provably done — screen unchanged, pty empty, and every process it -// forked asleep (see waitStable). That, the fan-out, and building the vendored -// C optimized (build.zig's c_optimize: 90% of a Debug startup used to be -// tree-sitter's query analyser at -O0) took the suite from 7m54s to ~4s. The -// app now reaches first paint in ~25ms, so what is left is mostly real work: -// bash spawns, their output, and the 40 `settle`s after `key esc`. -// -// Script commands (line-oriented, # comments): -// file create file in the script's cwd (before start) -// lines [tail] create file with n numbered lines, tail on each -// dirmk create a subdirectory -// run run a command in the script's cwd (NOT in a pane) -// start [arg] fork the app in a pty (optional extra CLI arg) -// wait pump until needle appears on the grid (fails hard) -// settle pump for a fixed duration -// stable pump until the grid stops changing -// text send literal bytes -// send send with \e \r \n \t \\ \xNN escapes -// key enter esc tab bs space up down left right home end -// pgup pgdn c- a- (after esc: settle, or -// vaxis reads the next byte in the same batch and -// parses it as alt-. The scripts said 700ms and -// it cost the suite 28s; there is no timer in vaxis -// to outlast — 20ms passes, they use 100ms) -// press|release <[ctrl-]left|middle|right> SGR mouse, 1-based -// drag motion with button held -// motion button-less motion (hover) -// wheel -// resize -// snap