diff options
Diffstat (limited to 'test/monkey.zig')
| -rw-r--r-- | test/monkey.zig | 254 |
1 files changed, 0 insertions, 254 deletions
diff --git a/test/monkey.zig b/test/monkey.zig deleted file mode 100644 index 243f2fbe..00000000 --- a/test/monkey.zig +++ /dev/null @@ -1,254 +0,0 @@ -// pardes-monkey: random snapshot scripts against the tty, hunting panics. -// -// zig build monkey -Dplatform=tui -- <seeds> [steps] [--from=N] [--out=DIR] [--keep] -// -// Each seed makes one script (a fixed seed makes the same script, so a hit -// replays): keys and leader bursts, typing, mouse presses, drags and wheel, -// tiny resizes, prompts and long prompt text, over files of bad UTF-8 and -// wide glyphs. pardes-snap runs it with --update into DIR (default -// /tmp/pardes-monkey), and a panic is a record in the app's crash file -// (src/crash.zig, in the script's fake home: found even when the crash cut -// the script short) or a `.zig:N:N: 0x` stack frame in the report or the -// grid it captured (a trace renders INTO the screen). A hit keeps DIR/seed-N.snap, its captures and its report; the rest are deleted. -// A panic's trace can scroll off a small grid: replay the kept script, or -// run the app in a larger pty. Every hit gets a fix and a pinned regression -// script in test/snapshots/. -// -// Not a test gate: it is slow, random over seeds and aimed at finding, not -// holding. Shell text typed into a pane only ever runs in the throwaway -// script directory, and its alphabet has no path characters. -const std = @import("std"); -const libc = std.c; - -const Script = struct { - out: std.ArrayListUnmanaged(u8) = .empty, - gpa: std.mem.Allocator, - rows: u16 = 24, - cols: u16 = 80, - - fn line(s: *Script, comptime fmt: []const u8, args: anytype) !void { - try s.out.print(s.gpa, fmt ++ "\n", args); - } -}; - -// Workspace-tag words: clicked with the middle button they run, and the -// harness finds their column on row 1 while the window is wide enough. -const commands = [_][]const u8{ "Newcol", "Find", "Grep", "Joincol", "Help", "Dump", "NextColor", "Debug" }; -const leader_keys = "abcdefghijklmnopqrstuvwxyz0123456789$^"; -const typed = "abcdefghijklmnopqrstuvwxyz0123456789 -_=+,:;()"; - -fn generate(gpa: std.mem.Allocator, seed: u64, steps: usize) ![]u8 { - var prng = std.Random.DefaultPrng.init(seed); - const random = prng.random(); - var s: Script = .{ .gpa = gpa }; - try s.line("# pardes-monkey seed {d}, {d} steps", .{ seed, steps }); - try s.line("file bad.txt hello \\xff\\xfe world\\nlatin-1 caf\\xe9 tail\\ntruncated \\xe6\\x97 here\\nend\\n", .{}); - try s.line("file wide.txt {s}tail\\nsecond line\\n", .{std.mem.asBytes(&@as([8][37]u8, @splat("\\xe6\\x97\\xa5\\xe6\\x9c\\xac\\xe8\\xaa\\x9e ".*)))}); - try s.line("lines long.txt 400 some words after the number, long enough to wrap in a narrow pane", .{}); - try s.line("start 24 80 bad.txt", .{}); - try s.line("wait 8000 Newcol Joincol", .{}); - try s.line("stable 300 5000", .{}); - for (0..steps) |step| { - const wide = s.cols >= 70 and s.rows >= 6; - switch (random.uintLessThan(u8, 12)) { - // A workspace word, run with a middle click: new columns make - // narrow panes, Find and Grep open prompts. - 0, 1 => if (wide) { - const word = commands[random.uintLessThan(usize, commands.len)]; - try s.line("press middle @{s} 1", .{word}); - try s.line("release middle @{s} 1", .{word}); - }, - // Typing: into a prompt when one is up, far past its band. - 2, 3 => { - var text: [200]u8 = undefined; - const n = random.intRangeAtMost(usize, 1, if (random.uintLessThan(u8, 4) == 0) text.len else 12); - for (text[0..n]) |*c| c.* = typed[random.uintLessThan(usize, typed.len)]; - try s.line("text {s}", .{text[0..n]}); - }, - 4 => try s.line("key {s}", .{switch (random.uintLessThan(u8, 10)) { - 0 => "enter", - 1 => "tab", - 2 => "bs", - 3 => "up", - 4 => "down", - 5 => "left", - 6 => "right", - 7 => "pgdn", - 8 => "home", - else => "end", - }}), - 5 => { - try s.line("key esc", .{}); - try s.line("settle 120", .{}); - }, - // A leader burst: space and a chord's keys. - 6 => { - var chord: [3][2]u8 = undefined; - const n = random.intRangeAtMost(usize, 1, 3); - for (chord[0..n]) |*c| c.* = .{ leader_keys[random.uintLessThan(usize, leader_keys.len)], 0 }; - try s.out.appendSlice(gpa, "key space"); - for (chord[0..n]) |c| try s.out.print(gpa, " {c}", .{c[0]}); - try s.out.append(gpa, '\n'); - }, - 7, 8 => { - const buttons = [_][]const u8{ "left", "middle", "right" }; - const button = buttons[random.uintLessThan(usize, buttons.len)]; - const col = random.intRangeAtMost(u16, 1, s.cols); - const row = random.intRangeAtMost(u16, 1, s.rows); - try s.line("press {s} {d} {d}", .{ button, col, row }); - if (random.boolean()) try s.line("drag {s} {d} {d}", .{ button, random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }); - try s.line("release {s} {d} {d}", .{ button, random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }); - }, - 9 => try s.line("wheel {s} {d} {d}", .{ if (random.boolean()) "up" else "down", random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }), - 10 => try s.line("motion {d} {d}", .{ random.intRangeAtMost(u16, 1, s.cols), random.intRangeAtMost(u16, 1, s.rows) }), - // Tiny windows as often as ordinary ones. - else => { - s.rows = if (random.boolean()) random.intRangeAtMost(u16, 1, 6) else random.intRangeAtMost(u16, 7, 40); - s.cols = if (random.boolean()) random.intRangeAtMost(u16, 1, 20) else random.intRangeAtMost(u16, 21, 140); - try s.line("resize {d} {d}", .{ s.rows, s.cols }); - }, - } - try s.line("settle 30", .{}); - if (step % 10 == 9) try s.line("snap s{d}", .{step}); - } - try s.line("stable 300 5000", .{}); - try s.line("snap end", .{}); - return s.out.toOwnedSlice(gpa); -} - -/// A stack frame of a Zig panic trace, `.zig:12:34: 0x`, of the app's: the -/// harness's own error trace (a script it could not finish) is not a hit. -fn hasTrace(bytes: []const u8) bool { - var lines = std.mem.splitScalar(u8, bytes, '\n'); - while (lines.next()) |line| { - if (std.mem.indexOf(u8, line, "test/snapshot.zig") != null or std.mem.indexOf(u8, line, "test/e2e_harness.zig") != null) continue; - if (frameIn(line)) return true; - } - return false; -} - -fn frameIn(bytes: []const u8) bool { - var from: usize = 0; - while (std.mem.indexOfPos(u8, bytes, from, ".zig:")) |at| : (from = at + 1) { - var i = at + ".zig:".len; - var digits: usize = 0; - while (i < bytes.len and std.ascii.isDigit(bytes[i])) : (i += 1) digits += 1; - if (digits == 0 or i >= bytes.len or bytes[i] != ':') continue; - i += 1; - digits = 0; - while (i < bytes.len and std.ascii.isDigit(bytes[i])) : (i += 1) digits += 1; - if (digits > 0 and std.mem.startsWith(u8, bytes[i..], ": 0x")) return true; - } - return false; -} - -test "a panic trace is found, and ordinary colons are not" { - try std.testing.expect(hasTrace(" /src/draw.zig:814:31: 0x4212821 in joinNotice")); - try std.testing.expect(!hasTrace("draw.zig:814 is where it was; 12:30: 0x")); - try std.testing.expect(!hasTrace("/x/test/snapshot.zig:197:28: 0x1279761 in main (snapshot.zig)")); -} - -test "a seed makes the same script every time" { - const a = try generate(std.testing.allocator, 42, 50, 3); - defer std.testing.allocator.free(a); - const b = try generate(std.testing.allocator, 42, 50, 3); - defer std.testing.allocator.free(b); - try std.testing.expectEqualStrings(a, b); -} - -fn readAll(gpa: std.mem.Allocator, io: std.Io, path: []const u8) []u8 { - return std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(64 << 20)) catch &.{}; -} - -pub fn main(init: std.process.Init) !void { - const gpa = std.heap.page_allocator; - const io = init.io; - const args = try init.minimal.args.toSlice(init.arena.allocator()); - if (args.len < 4) { - std.debug.print("usage: pardes-monkey <pardes-snap> <pardes> <seeds> [steps] [--from=N] [--out=DIR] [--keep] [--tape=DIR] [--dump]\n", .{}); - std.process.exit(2); - } - const snap = args[1]; - const app = args[2]; - var seeds: u64 = 20; - var steps: usize = 70; - var from: u64 = 1; - var out: []const u8 = "/tmp/pardes-monkey"; - // Keep every script and capture, not only a hit's: to see what it did. - var keep = false; - // --tape=DIR: each seed's sessions record their tapes there and the - // seed is a hit if one does not replay (pardes-snap --tape). - var tape: ?[]const u8 = null; - // --dump (with --tape): each replay loads the session's dump at its end - // too (pardes-snap --dump). - var dump = false; - var positional: usize = 0; - for (args[3..]) |a| { - if (std.mem.startsWith(u8, a, "--from=")) { - from = try std.fmt.parseInt(u64, a["--from=".len..], 10); - } else if (std.mem.eql(u8, a, "--keep")) { - keep = true; - } else if (std.mem.startsWith(u8, a, "--tape=")) { - tape = a; - } else if (std.mem.eql(u8, a, "--dump")) { - dump = true; - } else if (std.mem.startsWith(u8, a, "--out=")) { - out = a["--out=".len..]; - } else { - switch (positional) { - 0 => seeds = try std.fmt.parseInt(u64, a, 10), - 1 => steps = try std.fmt.parseInt(usize, a, 10), - else => {}, - } - positional += 1; - } - } - std.Io.Dir.cwd().createDirPath(io, out) catch {}; - var hits: usize = 0; - var seed = from; - while (seed < from + seeds) : (seed += 1) { - const script = try generate(gpa, seed, steps); - defer gpa.free(script); - var path_buf: [4096]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/seed-{d}.snap", .{ out, seed }); - try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = script }); - const run = std.process.run(gpa, io, .{ - .argv = if (tape) |t| (if (dump) &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, "--dump", path } else &.{ snap, app, "--update", "--no-retry", "--jobs=1", t, path }) else &.{ snap, app, "--update", "--no-retry", "--jobs=1", path }, - .stdout_limit = .limited(64 << 20), - .stderr_limit = .limited(64 << 20), - }) catch |err| { - std.debug.print("seed {d}: pardes-snap did not run ({t})\n", .{ seed, err }); - continue; - }; - defer gpa.free(run.stdout); - defer gpa.free(run.stderr); - var golden_buf: [4096]u8 = undefined; - const golden_path = try std.fmt.bufPrint(&golden_buf, "{s}/seed-{d}.golden", .{ out, seed }); - const golden = readAll(gpa, io, golden_path); - defer gpa.free(golden); - // The app's own crash file (src/crash.zig) in the script's fake home: - // found even when the crash cut the script short of a capture. - var crash_buf: [4096]u8 = undefined; - const crash_path = try std.fmt.bufPrint(&crash_buf, "/tmp/pardes-snap/seed-{d}/home/.config/pardes/crashes", .{seed}); - const crashes = readAll(gpa, io, crash_path); - defer gpa.free(crashes); - const diverged = tape != null and (std.mem.indexOf(u8, run.stdout, "DIVERGED") != null or std.mem.indexOf(u8, run.stderr, "DIVERGED") != null); - if (crashes.len != 0 or diverged or hasTrace(run.stdout) or hasTrace(run.stderr) or hasTrace(golden)) { - hits += 1; - var log_buf: [4096]u8 = undefined; - const log_path = try std.fmt.bufPrint(&log_buf, "{s}/seed-{d}.log", .{ out, seed }); - const both = try std.mem.concat(gpa, u8, &.{ crashes, run.stdout, run.stderr }); - defer gpa.free(both); - try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = log_path, .data = both }); - std.debug.print("{s} seed {d}: {s}\n", .{ if (diverged) "DIVERGED" else "PANIC", seed, path }); - } else if (!keep) { - std.Io.Dir.cwd().deleteFile(io, path) catch {}; - std.Io.Dir.cwd().deleteFile(io, golden_path) catch {}; - std.debug.print("ok seed {d}\n", .{seed}); - } else { - std.debug.print("ok seed {d} (kept)\n", .{seed}); - } - } - std.debug.print("{d} of {d} seeds panicked; kept in {s}\n", .{ hits, seeds, out }); - if (hits != 0) std.process.exit(1); -} |
