summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--build/snap.zig21
-rw-r--r--test/monkey.zig14
-rw-r--r--test/snapshot.zig43
3 files changed, 74 insertions, 4 deletions
diff --git a/build/snap.zig b/build/snap.zig
index 35a246ad..6982379c 100644
--- a/build/snap.zig
+++ b/build/snap.zig
@@ -66,6 +66,27 @@ pub fn addTty(b: *std.Build, opts: struct {
run_monkey.addPassthruArgs();
run_monkey.has_side_effects = true;
b.step("monkey", "random snapshot scripts hunting panics (-- <seeds> [steps] [--from=N] [--out=DIR])").dependOn(&run_monkey.step);
+ // The replay-divergence gate (src/tape.zig, docs/next-daemon.md): every
+ // snapshot script and `replay-seeds` monkey seeds with each session's
+ // tape recorded, each replayed into two replicas at different
+ // addresses and one that never draws, every digest checked.
+ const tapes = ".zig-cache/tapes";
+ const run_replay = b.addRunArtifact(snap);
+ run_replay.addArtifactArg(opts.exe);
+ run_replay.addPrefixedFileArg("--lspmock=", lspmock.getEmittedBin());
+ run_replay.addArg("--tape=" ++ tapes);
+ run_replay.setCwd(b.path("."));
+ run_replay.has_side_effects = true;
+ const seeds = b.option(u32, "replay-seeds", "monkey seeds `zig build replay` records and replays (default 20)") orelse 20;
+ const run_monkey_replay = b.addRunArtifact(monkey);
+ run_monkey_replay.addArtifactArg(snap);
+ run_monkey_replay.addArtifactArg(opts.exe);
+ run_monkey_replay.addArgs(&.{ b.fmt("{d}", .{seeds}), "70", "--tape=" ++ tapes ++ "/monkey", "--out=" ++ tapes ++ "/monkey" });
+ run_monkey_replay.setCwd(b.path("."));
+ run_monkey_replay.has_side_effects = true;
+ // One pardes-snap run at a time (its lock): the scripts, then the seeds.
+ run_monkey_replay.step.dependOn(&run_replay.step);
+ b.step("replay", "record every snapshot script's and -Dreplay-seeds monkey seeds' tapes and require each to replay without diverging").dependOn(&run_monkey_replay.step);
const monkey_tests = b.addTest(.{ .root_module = monkey.root_module });
b.step("monkey-test", "test the monkey's generator and panic detector").dependOn(&b.addRunArtifact(monkey_tests).step);
const run_driver_test = b.addRunArtifact(snap);
diff --git a/test/monkey.zig b/test/monkey.zig
index dcea0b43..7c9761ff 100644
--- a/test/monkey.zig
+++ b/test/monkey.zig
@@ -166,7 +166,7 @@ pub fn main(init: std.process.Init) !void {
const io = init.io;
const args = try init.minimal.args.toSlice(init.arena.allocator());
if (args.len < 4) {
- std.debug.print("usage: pardes-monkey <pardes-snap> <pardes> <seeds> [steps] [--from=N] [--out=DIR]\n", .{});
+ std.debug.print("usage: pardes-monkey <pardes-snap> <pardes> <seeds> [steps] [--from=N] [--out=DIR] [--keep] [--tape=DIR]\n", .{});
std.process.exit(2);
}
const snap = args[1];
@@ -177,12 +177,17 @@ pub fn main(init: std.process.Init) !void {
var out: []const u8 = "/tmp/pardes-monkey";
// Keep every script and capture, not only a hit's: to see what it did.
var keep = false;
+ // --tape=DIR: each seed's sessions record their tapes there and the
+ // seed is a hit if one does not replay (pardes-snap --tape).
+ var tape: ?[]const u8 = null;
var positional: usize = 0;
for (args[3..]) |a| {
if (std.mem.startsWith(u8, a, "--from=")) {
from = try std.fmt.parseInt(u64, a["--from=".len..], 10);
} else if (std.mem.eql(u8, a, "--keep")) {
keep = true;
+ } else if (std.mem.startsWith(u8, a, "--tape=")) {
+ tape = a;
} else if (std.mem.startsWith(u8, a, "--out=")) {
out = a["--out=".len..];
} else {
@@ -204,7 +209,7 @@ pub fn main(init: std.process.Init) !void {
const path = try std.fmt.bufPrint(&path_buf, "{s}/seed-{d}.snap", .{ out, seed });
try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = script });
const run = std.process.run(gpa, io, .{
- .argv = &.{ snap, app, "--update", "--no-retry", "--jobs=1", path },
+ .argv = if (tape) |t| &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, path } else &.{ snap, app, "--update", "--no-retry", "--jobs=1", path },
.stdout_limit = .limited(64 << 20),
.stderr_limit = .limited(64 << 20),
}) catch |err| {
@@ -223,14 +228,15 @@ pub fn main(init: std.process.Init) !void {
const crash_path = try std.fmt.bufPrint(&crash_buf, "/tmp/pardes-snap/seed-{d}/home/.config/pardes/crashes", .{seed});
const crashes = readAll(gpa, io, crash_path);
defer gpa.free(crashes);
- if (crashes.len != 0 or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) {
+ const diverged = tape != null and (std.mem.indexOf(u8, run.stdout, "DIVERGED") != null or std.mem.indexOf(u8, run.stderr, "DIVERGED") != null);
+ if (crashes.len != 0 or diverged or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) {
hits += 1;
var log_buf: [4096]u8 = undefined;
const log_path = try std.fmt.bufPrint(&log_buf, "{s}/seed-{d}.log", .{ out, seed });
const both = try std.mem.concat(gpa, u8, &.{ crashes, run.stdout, run.stderr });
defer gpa.free(both);
try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = log_path, .data = both });
- std.debug.print("PANIC seed {d}: {s}\n", .{ seed, path });
+ std.debug.print("{s} seed {d}: {s}\n", .{ if (diverged) "DIVERGED" else "PANIC", seed, path });
} else if (!keep) {
std.Io.Dir.cwd().deleteFile(io, path) catch {};
std.Io.Dir.cwd().deleteFile(io, golden_path) catch {};
diff --git a/test/snapshot.zig b/test/snapshot.zig
index 2d8a8664..255f55bf 100644
--- a/test/snapshot.zig
+++ b/test/snapshot.zig
@@ -86,6 +86,9 @@ var careful = false;
/// (--lspmock=); null leaves the protocol client fully disabled for the run
var lspmock_bin: ?[:0]const u8 = null;
var record_dir: ?[]const u8 = null;
+/// --tape=DIR: each session records its tape there (PARDES_RECORD, src/tape.zig)
+/// and the script fails unless every tape replays without diverging.
+var tape_dir: ?[]const u8 = null;
const gpa = std.heap.page_allocator;
pub fn main(init: std.process.Init) !void {
@@ -123,6 +126,8 @@ pub fn main(init: std.process.Init) !void {
lspmock = try absPath(arena, orig_cwd, a["--lspmock=".len..]);
} else if (std.mem.startsWith(u8, a, "--record=")) {
record_dir = try absPath(arena, orig_cwd, a["--record=".len..]);
+ } else if (std.mem.startsWith(u8, a, "--tape=")) {
+ tape_dir = try absPath(arena, orig_cwd, a["--tape=".len..]);
} else if (std.mem.startsWith(u8, a, "--jobs=")) {
jobs = @max(1, try std.fmt.parseInt(usize, a["--jobs=".len..], 10));
} else if (exe == null) {
@@ -148,6 +153,7 @@ pub fn main(init: std.process.Init) !void {
std.mem.eql(u8, a, "--trace-stable") or
std.mem.startsWith(u8, a, "--lspmock=") or
std.mem.startsWith(u8, a, "--record=") or
+ std.mem.startsWith(u8, a, "--tape=") or
std.mem.startsWith(u8, a, "--jobs=")) continue;
if (!saw_exe) {
saw_exe = true;
@@ -179,6 +185,7 @@ pub fn main(init: std.process.Init) !void {
if (update) fatal("--record and --update are separate modes", .{});
try mkdir(arena, dir, true);
}
+ if (tape_dir) |dir| try mkdir(arena, dir, true);
try mkdir(arena, SNAP_BASE, true);
// one run at a time (this is the flock that used to be typed by hand)
@@ -463,9 +470,15 @@ fn runOne(arena: std.mem.Allocator, exe_z: [:0]const u8, script_path: []const u8
const started = eh.nowMs();
const out = runScript(arena, exe_z, script_path, stem) catch |e| {
std.debug.print("FAIL {s}: script error {s} ({d}ms)\n", .{ stem, @errorName(e), eh.nowMs() - started });
+ // What the session did before the script gave up replays all the same.
+ if (tape_dir != null) if (replayTapes(arena, exe_z, stem)) |why| std.debug.print("DIVERGED {s}: {s}\n", .{ stem, why });
return 1;
};
const took = eh.nowMs() - started;
+ if (tape_dir != null) if (replayTapes(arena, exe_z, stem)) |why| {
+ std.debug.print("DIVERGED {s}: {s}\n", .{ stem, why });
+ return 1;
+ };
if (record_dir) |dir| {
const path = std.fmt.allocPrintSentinel(arena, "{s}/{s}.snapshot", .{ dir, stem }, 0) catch return 1;
eh.writeFile(path, out) catch return 1;
@@ -495,6 +508,30 @@ fn runOne(arena: std.mem.Allocator, exe_z: [:0]const u8, script_path: []const u8
return 1;
}
+/// Every tape the script's sessions wrote, replayed by the same build
+/// (PARDES_REPLAY): why the first that diverged did, or null.
+fn replayTapes(arena: std.mem.Allocator, exe_z: [:0]const u8, stem: []const u8) ?[]const u8 {
+ const dir = tape_dir.?;
+ const io = std.Io.Threaded.global_single_threaded.io();
+ var d = std.Io.Dir.cwd().openDir(io, dir, .{ .iterate = true }) catch return "the tape directory cannot be read";
+ defer d.close(io);
+ var it = d.iterate();
+ var tapes: usize = 0;
+ while (it.next(io) catch return "the tape directory cannot be listed") |entry| {
+ if (!std.mem.startsWith(u8, entry.name, stem) or !std.mem.endsWith(u8, entry.name, ".tape")) continue;
+ const pid = entry.name[stem.len..][0 .. entry.name.len - stem.len - ".tape".len];
+ if (pid.len < 2 or pid[0] != '-' or std.mem.indexOfNone(u8, pid[1..], "0123456789") != null) continue;
+ tapes += 1;
+ const cmd = std.fmt.allocPrintSentinel(arena, "PARDES_RECORD= PARDES_REPLAY='{s}/{s}' timeout 300 '{s}' > '{s}/{s}.replay' 2>&1", .{ dir, entry.name, exe_z, dir, entry.name }, 0) catch return "out of memory";
+ if (system(cmd) != 0) {
+ const why_path = std.fmt.allocPrint(arena, "{s}/{s}.replay", .{ dir, entry.name }) catch return "out of memory";
+ return readFileAlloc(arena, why_path) catch "the replay failed and said nothing";
+ }
+ }
+ if (tapes == 0) return "no session wrote a tape";
+ return null;
+}
+
fn reportPath(arena: std.mem.Allocator, script_path: []const u8) ![]const u8 {
return std.fmt.allocPrint(arena, "{s}/{s}.report", .{ SNAP_BASE, std.fs.path.stem(script_path) });
}
@@ -603,6 +640,12 @@ fn runScript(arena: std.mem.Allocator, exe_z: [:0]const u8, script_path: []const
// together. Scripts name it `$DUMP`.
const dump_path = try std.fmt.allocPrint(arena, "{s}/dump.zon", .{base});
_ = setenv("PARDES_DUMP", try arena.dupeSentinel(u8, dump_path, 0), 1);
+ // Each session the script starts (and each pardes run inside one)
+ // writes its own tape, <stem>-<pid>.tape.
+ if (tape_dir) |dir| {
+ _ = system(try std.fmt.allocPrintSentinel(arena, "rm -f '{s}/{s}-'[0-9]*.tape '{s}/{s}-'[0-9]*.tape.replay", .{ dir, stem, dir, stem }, 0));
+ _ = setenv("PARDES_RECORD", try std.fmt.allocPrintSentinel(arena, "{s}/{s}-", .{ dir, stem }, 0), 1);
+ }
if (chdirRc(try arena.dupeSentinel(u8, work, 0)) < 0) return error.ChdirFailed;
const chunks = try arena.alloc([]const u8, countCaptures(src));