From a4b85cb7689c111c9f06891164e765a1b49f63c8 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Thu, 29 Oct 2026 11:14:38 -0300 Subject: replay gate: pardes-snap --tape and pardes-monkey --tape record every session's tape and fail a script whose tapes do not replay; `zig build replay` pardes-snap --tape=DIR sets PARDES_RECORD for each script (each session, a nested pardes's too, writes DIR/-.tape) and, once the script has run and its goldens compared, replays every tape with the same build (PARDES_REPLAY): a divergence, or no tape at all, fails the script as DIVERGED with the replay's words. pardes-monkey --tape=DIR passes it on and counts a divergence as a hit. `zig build replay -Dplatform=tty [-Dreplay-seeds=N]` runs every snapshot script, then N monkey seeds of 70 steps (default 20), tapes under .zig-cache/tapes. Co-Authored-By: Claude Opus 5.5 --- build/snap.zig | 21 +++++++++++++++++++++ test/monkey.zig | 14 ++++++++++---- test/snapshot.zig | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/build/snap.zig b/build/snap.zig index 35a246ad..6982379c 100644 --- a/build/snap.zig +++ b/build/snap.zig @@ -66,6 +66,27 @@ pub fn addTty(b: *std.Build, opts: struct { run_monkey.addPassthruArgs(); run_monkey.has_side_effects = true; b.step("monkey", "random snapshot scripts hunting panics (-- [steps] [--from=N] [--out=DIR])").dependOn(&run_monkey.step); + // The replay-divergence gate (src/tape.zig, docs/next-daemon.md): every + // snapshot script and `replay-seeds` monkey seeds with each session's + // tape recorded, each replayed into two replicas at different + // addresses and one that never draws, every digest checked. + const tapes = ".zig-cache/tapes"; + const run_replay = b.addRunArtifact(snap); + run_replay.addArtifactArg(opts.exe); + run_replay.addPrefixedFileArg("--lspmock=", lspmock.getEmittedBin()); + run_replay.addArg("--tape=" ++ tapes); + run_replay.setCwd(b.path(".")); + run_replay.has_side_effects = true; + const seeds = b.option(u32, "replay-seeds", "monkey seeds `zig build replay` records and replays (default 20)") orelse 20; + const run_monkey_replay = b.addRunArtifact(monkey); + run_monkey_replay.addArtifactArg(snap); + run_monkey_replay.addArtifactArg(opts.exe); + run_monkey_replay.addArgs(&.{ b.fmt("{d}", .{seeds}), "70", "--tape=" ++ tapes ++ "/monkey", "--out=" ++ tapes ++ "/monkey" }); + run_monkey_replay.setCwd(b.path(".")); + run_monkey_replay.has_side_effects = true; + // One pardes-snap run at a time (its lock): the scripts, then the seeds. + run_monkey_replay.step.dependOn(&run_replay.step); + b.step("replay", "record every snapshot script's and -Dreplay-seeds monkey seeds' tapes and require each to replay without diverging").dependOn(&run_monkey_replay.step); const monkey_tests = b.addTest(.{ .root_module = monkey.root_module }); b.step("monkey-test", "test the monkey's generator and panic detector").dependOn(&b.addRunArtifact(monkey_tests).step); const run_driver_test = b.addRunArtifact(snap); diff --git a/test/monkey.zig b/test/monkey.zig index dcea0b43..7c9761ff 100644 --- a/test/monkey.zig +++ b/test/monkey.zig @@ -166,7 +166,7 @@ pub fn main(init: std.process.Init) !void { const io = init.io; const args = try init.minimal.args.toSlice(init.arena.allocator()); if (args.len < 4) { - std.debug.print("usage: pardes-monkey [steps] [--from=N] [--out=DIR]\n", .{}); + std.debug.print("usage: pardes-monkey [steps] [--from=N] [--out=DIR] [--keep] [--tape=DIR]\n", .{}); std.process.exit(2); } const snap = args[1]; @@ -177,12 +177,17 @@ pub fn main(init: std.process.Init) !void { var out: []const u8 = "/tmp/pardes-monkey"; // Keep every script and capture, not only a hit's: to see what it did. var keep = false; + // --tape=DIR: each seed's sessions record their tapes there and the + // seed is a hit if one does not replay (pardes-snap --tape). + var tape: ?[]const u8 = null; var positional: usize = 0; for (args[3..]) |a| { if (std.mem.startsWith(u8, a, "--from=")) { from = try std.fmt.parseInt(u64, a["--from=".len..], 10); } else if (std.mem.eql(u8, a, "--keep")) { keep = true; + } else if (std.mem.startsWith(u8, a, "--tape=")) { + tape = a; } else if (std.mem.startsWith(u8, a, "--out=")) { out = a["--out=".len..]; } else { @@ -204,7 +209,7 @@ pub fn main(init: std.process.Init) !void { const path = try std.fmt.bufPrint(&path_buf, "{s}/seed-{d}.snap", .{ out, seed }); try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = script }); const run = std.process.run(gpa, io, .{ - .argv = &.{ snap, app, "--update", "--no-retry", "--jobs=1", path }, + .argv = if (tape) |t| &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, path } else &.{ snap, app, "--update", "--no-retry", "--jobs=1", path }, .stdout_limit = .limited(64 << 20), .stderr_limit = .limited(64 << 20), }) catch |err| { @@ -223,14 +228,15 @@ pub fn main(init: std.process.Init) !void { const crash_path = try std.fmt.bufPrint(&crash_buf, "/tmp/pardes-snap/seed-{d}/home/.config/pardes/crashes", .{seed}); const crashes = readAll(gpa, io, crash_path); defer gpa.free(crashes); - if (crashes.len != 0 or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) { + const diverged = tape != null and (std.mem.indexOf(u8, run.stdout, "DIVERGED") != null or std.mem.indexOf(u8, run.stderr, "DIVERGED") != null); + if (crashes.len != 0 or diverged or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) { hits += 1; var log_buf: [4096]u8 = undefined; const log_path = try std.fmt.bufPrint(&log_buf, "{s}/seed-{d}.log", .{ out, seed }); const both = try std.mem.concat(gpa, u8, &.{ crashes, run.stdout, run.stderr }); defer gpa.free(both); try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = log_path, .data = both }); - std.debug.print("PANIC seed {d}: {s}\n", .{ seed, path }); + std.debug.print("{s} seed {d}: {s}\n", .{ if (diverged) "DIVERGED" else "PANIC", seed, path }); } else if (!keep) { std.Io.Dir.cwd().deleteFile(io, path) catch {}; std.Io.Dir.cwd().deleteFile(io, golden_path) catch {}; diff --git a/test/snapshot.zig b/test/snapshot.zig index 2d8a8664..255f55bf 100644 --- a/test/snapshot.zig +++ b/test/snapshot.zig @@ -86,6 +86,9 @@ var careful = false; /// (--lspmock=); null leaves the protocol client fully disabled for the run var lspmock_bin: ?[:0]const u8 = null; var record_dir: ?[]const u8 = null; +/// --tape=DIR: each session records its tape there (PARDES_RECORD, src/tape.zig) +/// and the script fails unless every tape replays without diverging. +var tape_dir: ?[]const u8 = null; const gpa = std.heap.page_allocator; pub fn main(init: std.process.Init) !void { @@ -123,6 +126,8 @@ pub fn main(init: std.process.Init) !void { lspmock = try absPath(arena, orig_cwd, a["--lspmock=".len..]); } else if (std.mem.startsWith(u8, a, "--record=")) { record_dir = try absPath(arena, orig_cwd, a["--record=".len..]); + } else if (std.mem.startsWith(u8, a, "--tape=")) { + tape_dir = try absPath(arena, orig_cwd, a["--tape=".len..]); } else if (std.mem.startsWith(u8, a, "--jobs=")) { jobs = @max(1, try std.fmt.parseInt(usize, a["--jobs=".len..], 10)); } else if (exe == null) { @@ -148,6 +153,7 @@ pub fn main(init: std.process.Init) !void { std.mem.eql(u8, a, "--trace-stable") or std.mem.startsWith(u8, a, "--lspmock=") or std.mem.startsWith(u8, a, "--record=") or + std.mem.startsWith(u8, a, "--tape=") or std.mem.startsWith(u8, a, "--jobs=")) continue; if (!saw_exe) { saw_exe = true; @@ -179,6 +185,7 @@ pub fn main(init: std.process.Init) !void { if (update) fatal("--record and --update are separate modes", .{}); try mkdir(arena, dir, true); } + if (tape_dir) |dir| try mkdir(arena, dir, true); try mkdir(arena, SNAP_BASE, true); // one run at a time (this is the flock that used to be typed by hand) @@ -463,9 +470,15 @@ fn runOne(arena: std.mem.Allocator, exe_z: [:0]const u8, script_path: []const u8 const started = eh.nowMs(); const out = runScript(arena, exe_z, script_path, stem) catch |e| { std.debug.print("FAIL {s}: script error {s} ({d}ms)\n", .{ stem, @errorName(e), eh.nowMs() - started }); + // What the session did before the script gave up replays all the same. + if (tape_dir != null) if (replayTapes(arena, exe_z, stem)) |why| std.debug.print("DIVERGED {s}: {s}\n", .{ stem, why }); return 1; }; const took = eh.nowMs() - started; + if (tape_dir != null) if (replayTapes(arena, exe_z, stem)) |why| { + std.debug.print("DIVERGED {s}: {s}\n", .{ stem, why }); + return 1; + }; if (record_dir) |dir| { const path = std.fmt.allocPrintSentinel(arena, "{s}/{s}.snapshot", .{ dir, stem }, 0) catch return 1; eh.writeFile(path, out) catch return 1; @@ -495,6 +508,30 @@ fn runOne(arena: std.mem.Allocator, exe_z: [:0]const u8, script_path: []const u8 return 1; } +/// Every tape the script's sessions wrote, replayed by the same build +/// (PARDES_REPLAY): why the first that diverged did, or null. +fn replayTapes(arena: std.mem.Allocator, exe_z: [:0]const u8, stem: []const u8) ?[]const u8 { + const dir = tape_dir.?; + const io = std.Io.Threaded.global_single_threaded.io(); + var d = std.Io.Dir.cwd().openDir(io, dir, .{ .iterate = true }) catch return "the tape directory cannot be read"; + defer d.close(io); + var it = d.iterate(); + var tapes: usize = 0; + while (it.next(io) catch return "the tape directory cannot be listed") |entry| { + if (!std.mem.startsWith(u8, entry.name, stem) or !std.mem.endsWith(u8, entry.name, ".tape")) continue; + const pid = entry.name[stem.len..][0 .. entry.name.len - stem.len - ".tape".len]; + if (pid.len < 2 or pid[0] != '-' or std.mem.indexOfNone(u8, pid[1..], "0123456789") != null) continue; + tapes += 1; + const cmd = std.fmt.allocPrintSentinel(arena, "PARDES_RECORD= PARDES_REPLAY='{s}/{s}' timeout 300 '{s}' > '{s}/{s}.replay' 2>&1", .{ dir, entry.name, exe_z, dir, entry.name }, 0) catch return "out of memory"; + if (system(cmd) != 0) { + const why_path = std.fmt.allocPrint(arena, "{s}/{s}.replay", .{ dir, entry.name }) catch return "out of memory"; + return readFileAlloc(arena, why_path) catch "the replay failed and said nothing"; + } + } + if (tapes == 0) return "no session wrote a tape"; + return null; +} + fn reportPath(arena: std.mem.Allocator, script_path: []const u8) ![]const u8 { return std.fmt.allocPrint(arena, "{s}/{s}.report", .{ SNAP_BASE, std.fs.path.stem(script_path) }); } @@ -603,6 +640,12 @@ fn runScript(arena: std.mem.Allocator, exe_z: [:0]const u8, script_path: []const // together. Scripts name it `$DUMP`. const dump_path = try std.fmt.allocPrint(arena, "{s}/dump.zon", .{base}); _ = setenv("PARDES_DUMP", try arena.dupeSentinel(u8, dump_path, 0), 1); + // Each session the script starts (and each pardes run inside one) + // writes its own tape, -.tape. + if (tape_dir) |dir| { + _ = system(try std.fmt.allocPrintSentinel(arena, "rm -f '{s}/{s}-'[0-9]*.tape '{s}/{s}-'[0-9]*.tape.replay", .{ dir, stem, dir, stem }, 0)); + _ = setenv("PARDES_RECORD", try std.fmt.allocPrintSentinel(arena, "{s}/{s}-", .{ dir, stem }, 0), 1); + } if (chdirRc(try arena.dupeSentinel(u8, work, 0)) < 0) return error.ChdirFailed; const chunks = try arena.alloc([]const u8, countCaptures(src)); -- cgit v1.3