summaryrefslogtreecommitdiff
path: root/test/history.zig
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-09-06 18:11:36 -0300
committerGabriel Schneider <[email protected]>2026-09-07 13:59:12 -0300
commit60367d8fe23f6af98ec28e3cf6c2094dfe332df0 (patch)
tree310fc734173cf771881f4691c71909135fadde97 /test/history.zig
parentfa82cac885cb4738fe36d1e49b4749b5a3e31a4a (diff)
downloadpardes-60367d8fe23f6af98ec28e3cf6c2094dfe332df0.tar.gz
pardes-60367d8fe23f6af98ec28e3cf6c2094dfe332df0.zip
Refactor panes and filesystem; replace FUSE with 9P
Consolidate pane, layout, memory and host code. Serve 9P by default over Unix sockets, with runtime mounts and optional TCP/QUIC transports. Remove FUSE and obsolete proof-of-concept examples. Fix highlighting and terminal-history performance, expand differential and stress-test infrastructure, sort navigation results while preserving the next occurrence, add syntax-colored Braille minimaps, remove SPC-k, and document 9P interaction as a repository skill.
Diffstat (limited to 'test/history.zig')
-rw-r--r--test/history.zig502
1 files changed, 502 insertions, 0 deletions
diff --git a/test/history.zig b/test/history.zig
new file mode 100644
index 00000000..5ce45239
--- /dev/null
+++ b/test/history.zig
@@ -0,0 +1,502 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const Dir = std.Io.Dir;
+const schema_version = 2;
+const harness_id = blk: {
+ @setEvalBranchQuota(100_000);
+ break :blk std.fmt.comptimePrint("{x}", .{std.hash.Wyhash.hash(0, @embedFile("history.zig"))});
+};
+
+const Measurement = struct {
+ schema: u32 = 0,
+ harness: []const u8 = "",
+ zig_version: []const u8 = "",
+ revision: []const u8,
+ cwd: []const u8,
+ command: []const []const u8,
+ first_ns: u64,
+ median_ns: u64,
+ samples_ns: []const u64,
+ stdout: []const u8,
+ stderr: []const u8,
+ stable_stdout: bool,
+ stable_stderr: bool = false,
+ passed: bool,
+};
+
+const Compare = struct {
+ snapshots: bool = false,
+ benchmarks: bool = false,
+ max_ratio: ?f64 = null,
+};
+
+pub fn main(init: std.process.Init) !void {
+ const arena = init.arena.allocator();
+ const io = init.io;
+ const args = try init.minimal.args.toSlice(arena);
+ if (args.len < 2) return usage(io);
+ if (std.mem.eql(u8, args[1], "run")) {
+ const immutable = args.len > 2 and std.mem.eql(u8, args[2], "--allow-immutable");
+ const pos: usize = if (immutable) 3 else 2;
+ if (args.len < pos + 4 or !std.mem.eql(u8, args[pos + 2], "--")) return usage(io);
+ try Dir.cwd().createDirPath(io, args[pos + 1]);
+ const output = try Dir.cwd().realPathFileAlloc(io, args[pos + 1], arena);
+ const exe = try std.process.executablePathAlloc(io, arena);
+ var command: std.ArrayList([]const u8) = .empty;
+ try command.appendSlice(arena, &.{ "jj", "--ignore-working-copy", "--no-integrate-operation" });
+ if (immutable) try command.append(arena, "--ignore-immutable");
+ try command.appendSlice(arena, &.{ "run", "--root", "-j", "1", "-r", args[pos], "--", exe, "record", output, "--" });
+ try command.appendSlice(arena, args[pos + 3 ..]);
+ const result = try std.process.run(arena, io, .{ .argv = command.items });
+ try std.Io.File.stdout().writeStreamingAll(io, result.stdout);
+ try std.Io.File.stderr().writeStreamingAll(io, result.stderr);
+ if (result.term != .exited or result.term.exited != 0) return error.HistoryRunFailed;
+ } else if (std.mem.eql(u8, args[1], "record")) {
+ if (args.len < 5 or !std.mem.eql(u8, args[3], "--")) return usage(io);
+ const command = args[4..];
+ try Dir.cwd().createDirPath(io, args[2]);
+ const output = try Dir.cwd().realPathFileAlloc(io, args[2], arena);
+ const revision = init.environ_map.get("JJ_COMMIT_ID") orelse "working";
+ const encoded_command = try std.json.Stringify.valueAlloc(arena, command, .{});
+ const stem = try std.fmt.allocPrint(arena, "{s}/{s}-{x}", .{
+ output, revision, std.hash.Wyhash.hash(0, encoded_command),
+ });
+ try record(io, arena, stem, revision, command);
+ } else if (std.mem.eql(u8, args[1], "compare")) {
+ if (args.len < 4) return usage(io);
+ var options: Compare = .{};
+ for (args[4..]) |arg| {
+ if (std.mem.eql(u8, arg, "--snapshots")) {
+ options.snapshots = true;
+ } else if (std.mem.eql(u8, arg, "--benchmarks")) {
+ options.benchmarks = true;
+ } else if (options.max_ratio == null) {
+ const ratio = std.fmt.parseFloat(f64, arg) catch return usage(io);
+ if (!std.math.isFinite(ratio) or ratio <= 0) return error.InvalidRatio;
+ options.max_ratio = ratio;
+ } else return usage(io);
+ }
+ if (options.snapshots and options.benchmarks) return usage(io);
+ var values: [2]Measurement = undefined;
+ for (&values, args[2..4]) |*value, path| {
+ const bytes = try Dir.cwd().readFileAlloc(io, path, arena, .limited(1024 * 1024));
+ value.* = (try std.json.parseFromSlice(Measurement, arena, bytes, .{})).value;
+ }
+ var report: std.Io.Writer.Allocating = .init(arena);
+ defer report.deinit();
+ const result = compare(io, arena, values, options, &report.writer);
+ try std.Io.File.stdout().writeStreamingAll(io, report.written());
+ try result;
+ } else return usage(io);
+}
+
+fn writeNew(io: std.Io, path: []const u8, bytes: []const u8) !void {
+ const file = try Dir.cwd().createFile(io, path, .{ .exclusive = true });
+ defer file.close(io);
+ try file.writeStreamingAll(io, bytes);
+}
+
+fn record(io: std.Io, arena: std.mem.Allocator, stem: []const u8, revision: []const u8, command: []const []const u8) !void {
+ const reservation = try std.fmt.allocPrint(arena, "{s}.reserved", .{stem});
+ writeNew(io, reservation, "") catch |err| return if (err == error.PathAlreadyExists) error.MeasurementExists else err;
+ for ([_][]const u8{ ".json", ".0.stdout", ".0.stderr", ".1.stdout", ".1.stderr", ".2.stdout", ".2.stderr", ".3.stdout", ".3.stderr" }) |suffix| {
+ const path = try std.fmt.allocPrint(arena, "{s}{s}", .{ stem, suffix });
+ _ = Dir.cwd().statFile(io, path, .{ .follow_symlinks = false }) catch |err| {
+ if (err == error.FileNotFound) continue;
+ return err;
+ };
+ return error.MeasurementExists;
+ }
+ var times: [4]u64 = undefined;
+ var first_output: []const u8 = "";
+ var first_error: []const u8 = "";
+ var stable_stdout = true;
+ var stable_stderr = true;
+ var passed = true;
+ for (&times, 0..) |*elapsed, index| {
+ const started = std.Io.Clock.awake.now(io);
+ const result = try std.process.run(arena, io, .{
+ .argv = command,
+ .stdout_limit = .limited(64 * 1024 * 1024),
+ .stderr_limit = .limited(64 * 1024 * 1024),
+ });
+ elapsed.* = @intCast(started.durationTo(std.Io.Clock.awake.now(io)).toNanoseconds());
+ passed = passed and result.term == .exited and result.term.exited == 0;
+ if (index == 0) {
+ first_output = result.stdout;
+ first_error = result.stderr;
+ } else {
+ stable_stdout = stable_stdout and std.mem.eql(u8, first_output, result.stdout);
+ stable_stderr = stable_stderr and std.mem.eql(u8, first_error, result.stderr);
+ }
+ try writeNew(io, try std.fmt.allocPrint(arena, "{s}.{d}.stdout", .{ stem, index }), result.stdout);
+ try writeNew(io, try std.fmt.allocPrint(arena, "{s}.{d}.stderr", .{ stem, index }), result.stderr);
+ }
+ var warmed = times[1..].*;
+ std.mem.sort(u64, &warmed, {}, std.sort.asc(u64));
+ const measurement: Measurement = .{
+ .schema = schema_version,
+ .harness = harness_id,
+ .zig_version = builtin.zig_version_string,
+ .revision = revision,
+ .cwd = try std.process.currentPathAlloc(io, arena),
+ .command = command,
+ .first_ns = times[0],
+ .median_ns = warmed[1],
+ .samples_ns = &times,
+ .stdout = try std.fmt.allocPrint(arena, "{s}.0.stdout", .{stem}),
+ .stderr = try std.fmt.allocPrint(arena, "{s}.0.stderr", .{stem}),
+ .stable_stdout = stable_stdout,
+ .stable_stderr = stable_stderr,
+ .passed = passed,
+ };
+ const json = try std.json.Stringify.valueAlloc(arena, measurement, .{ .whitespace = .indent_2 });
+ const path = try std.fmt.allocPrint(arena, "{s}.json", .{stem});
+ try writeNew(io, path, json);
+ const report = try std.fmt.allocPrint(arena, "{s}: first {d:.3}s, warm median {d:.3}s, passed={}\n", .{
+ path, @as(f64, @floatFromInt(times[0])) / 1e9, @as(f64, @floatFromInt(warmed[1])) / 1e9, passed,
+ });
+ try std.Io.File.stdout().writeStreamingAll(io, report);
+ if (!passed) return error.CommandFailed;
+}
+
+fn compare(io: std.Io, arena: std.mem.Allocator, values: [2]Measurement, options: Compare, report: *std.Io.Writer) !void {
+ for (values) |value| if (value.schema != schema_version) return error.UnsupportedMeasurementVersion;
+ if (!std.mem.eql(u8, values[0].harness, values[1].harness) or
+ !std.mem.eql(u8, values[0].zig_version, values[1].zig_version)) return error.DifferentHarness;
+ if (!values[0].passed or !values[1].passed) return error.FailedMeasurement;
+ const a = try std.json.Stringify.valueAlloc(arena, values[0].command, .{});
+ const b = try std.json.Stringify.valueAlloc(arena, values[1].command, .{});
+ if (!std.mem.eql(u8, a, b)) return error.DifferentCommands;
+ const ratio = @as(f64, @floatFromInt(values[1].median_ns)) / @as(f64, @floatFromInt(@max(1, values[0].median_ns)));
+ const before = try Dir.cwd().readFileAlloc(io, values[0].stdout, arena, .limited(64 * 1024 * 1024));
+ const after = try Dir.cwd().readFileAlloc(io, values[1].stdout, arena, .limited(64 * 1024 * 1024));
+ const before_error = try Dir.cwd().readFileAlloc(io, values[0].stderr, arena, .limited(64 * 1024 * 1024));
+ const after_error = try Dir.cwd().readFileAlloc(io, values[1].stderr, arena, .limited(64 * 1024 * 1024));
+ const same_stdout = std.mem.eql(u8, before, after);
+ const same_stderr = std.mem.eql(u8, before_error, after_error);
+ const stable = values[0].stable_stdout and values[1].stable_stdout and values[0].stable_stderr and values[1].stable_stderr;
+ try report.print("warm runtime {d:.3}x; stdout {s}; stderr {s}; repeated captures stable={}\n", .{
+ ratio, if (same_stdout) "identical" else "DIFFERS", if (same_stderr) "identical" else "DIFFERS", stable,
+ });
+ if (options.snapshots and (!same_stdout or !same_stderr or !stable)) return error.SnapshotMismatch;
+ if (options.benchmarks) {
+ const before_runs = try readBenchmarkRuns(io, arena, values[0]);
+ const after_runs = try readBenchmarkRuns(io, arena, values[1]);
+ if (try compareBenchmarks(before_runs, after_runs, options.max_ratio, report)) return error.PerformanceRegression;
+ }
+ if (options.max_ratio) |limit| if (ratio > limit) return error.PerformanceRegression;
+}
+
+const Benchmark = struct {
+ case: []const u8,
+ bytes: u64,
+ median_ns: u64,
+ cold_ns: u64,
+ allocations: u64,
+ allocated_bytes: u64,
+};
+
+fn readBenchmarks(arena: std.mem.Allocator, text: []const u8) ![]Benchmark {
+ var result: std.ArrayList(Benchmark) = .empty;
+ var lines = std.mem.splitScalar(u8, text, '\n');
+ while (lines.next()) |line| {
+ if (std.mem.trim(u8, line, " \t\r").len == 0) continue;
+ const value = (try std.json.parseFromSlice(Benchmark, arena, line, .{ .ignore_unknown_fields = true })).value;
+ if (value.case.len == 0) return error.EmptyBenchmarkName;
+ for (result.items) |old| if (std.mem.eql(u8, old.case, value.case)) return error.DuplicateBenchmark;
+ try result.append(arena, value);
+ }
+ if (result.items.len == 0) return error.NoBenchmarks;
+ return result.toOwnedSlice(arena);
+}
+
+fn readBenchmarkRuns(io: std.Io, arena: std.mem.Allocator, value: Measurement) ![4][]Benchmark {
+ const suffix = ".0.stdout";
+ if (value.samples_ns.len != 4 or !std.mem.endsWith(u8, value.stdout, suffix)) return error.InvalidBenchmarkCaptures;
+ const stem = value.stdout[0 .. value.stdout.len - suffix.len];
+ var runs: [4][]Benchmark = undefined;
+ for (&runs, 0..) |*run, index| {
+ const path = try std.fmt.allocPrint(arena, "{s}.{d}.stdout", .{ stem, index });
+ const bytes = try Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024 * 1024));
+ run.* = try readBenchmarks(arena, bytes);
+ }
+ return runs;
+}
+
+fn compareBenchmarks(before: [4][]Benchmark, after: [4][]Benchmark, max_ratio: ?f64, report: *std.Io.Writer) !bool {
+ const runs = [2][4][]Benchmark{ before, after };
+ for (runs) |side| for (side) |run| {
+ if (run.len != before[0].len) return error.DifferentBenchmarks;
+ };
+ var regressed = false;
+ for (before[0]) |first| {
+ var matched: [2][4]Benchmark = undefined;
+ for (runs, &matched) |side, *samples| for (side, samples) |run, *sample| {
+ sample.* = for (run) |value| {
+ if (std.mem.eql(u8, first.case, value.case)) break value;
+ } else return error.DifferentBenchmarks;
+ if (sample.bytes != first.bytes) return error.DifferentBenchmarkInputs;
+ };
+ var medians = [2]Benchmark{ first, first };
+ for (matched, &medians) |samples, *median| {
+ inline for (.{ "cold_ns", "median_ns", "allocations", "allocated_bytes" }) |field| {
+ var warm: [3]u64 = undefined;
+ for (samples[1..], &warm) |sample, *value| value.* = @field(sample, field);
+ std.mem.sort(u64, &warm, {}, std.sort.asc(u64));
+ @field(median, field) = warm[1];
+ }
+ }
+ const old = medians[0];
+ const current = medians[1];
+ const first_cold = @as(f64, @floatFromInt(matched[1][0].cold_ns)) / @as(f64, @floatFromInt(@max(1, matched[0][0].cold_ns)));
+ const cold = @as(f64, @floatFromInt(current.cold_ns)) / @as(f64, @floatFromInt(@max(1, old.cold_ns)));
+ const median = @as(f64, @floatFromInt(current.median_ns)) / @as(f64, @floatFromInt(@max(1, old.median_ns)));
+ const case_regression = if (max_ratio) |limit| cold > limit or median > limit else false;
+ regressed = regressed or case_regression;
+ try report.print("{s}: first cold {d:.3}x; cold {d:.3}x; median {d:.3}x; allocations {d}->{d}; bytes {d}->{d}{s}\n", .{
+ old.case,
+ first_cold,
+ cold,
+ median,
+ old.allocations,
+ current.allocations,
+ old.allocated_bytes,
+ current.allocated_bytes,
+ if (case_regression) " REGRESSION" else "",
+ });
+ }
+ return regressed;
+}
+
+fn usage(io: std.Io) !void {
+ try std.Io.File.stderr().writeStreamingAll(io, "history run [--allow-immutable] REVSET OUT -- COMMAND...\n" ++
+ "history record OUT -- COMMAND...\n" ++
+ "history compare BEFORE.json AFTER.json [MAX_RATIO] [--snapshots|--benchmarks]\n" ++
+ "run uses isolated jj workspaces without integrating history changes.\n" ++
+ "Commands still have normal filesystem and network access.\n");
+ return error.InvalidArguments;
+}
+
+test "history output collisions preserve completed and partial recordings before command dispatch" {
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ const gpa = arena.allocator();
+ const io = std.testing.io;
+ for ([_][]const u8{ ".reserved", ".json", ".0.stdout", ".0.stderr", ".1.stdout", ".1.stderr", ".2.stdout", ".2.stderr", ".3.stdout", ".3.stderr" }) |suffix| {
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ const output = try tmp.dir.realPathFileAlloc(io, ".", gpa);
+ const stem = try std.fmt.allocPrint(gpa, "{s}/measurement", .{output});
+ const path = try std.fmt.allocPrint(gpa, "{s}{s}", .{ stem, suffix });
+ try writeNew(io, path, "previous recording\n");
+ const missing = try std.fmt.allocPrint(gpa, "{s}/must-not-be-dispatched", .{output});
+ try std.testing.expectError(error.MeasurementExists, record(io, gpa, stem, "fixture", &.{missing}));
+ const preserved = try Dir.cwd().readFileAlloc(io, path, gpa, .limited(1024));
+ try std.testing.expectEqualStrings("previous recording\n", preserved);
+ try std.testing.expectError(error.MeasurementExists, record(io, gpa, stem, "fixture", &.{missing}));
+ }
+}
+
+const measurement_fixture: Measurement = .{
+ .schema = schema_version,
+ .harness = harness_id,
+ .zig_version = builtin.zig_version_string,
+ .revision = "fixture",
+ .cwd = "",
+ .command = &.{ "fixture", "--argument" },
+ .first_ns = 100,
+ .median_ns = 100,
+ .samples_ns = &.{ 100, 100, 100, 100 },
+ .stdout = "",
+ .stderr = "",
+ .stable_stdout = true,
+ .stable_stderr = true,
+ .passed = true,
+};
+
+test "history rejects incomparable and failed measurements before reading captures" {
+ var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer memory.deinit();
+ const arena = memory.allocator();
+ var report: std.Io.Writer.Allocating = .init(arena);
+ defer report.deinit();
+ var values = [2]Measurement{ measurement_fixture, measurement_fixture };
+
+ values[1].schema = 0;
+ try std.testing.expectError(error.UnsupportedMeasurementVersion, compare(std.testing.io, arena, values, .{}, &report.writer));
+ values[1] = measurement_fixture;
+ values[0].schema = schema_version + 1;
+ try std.testing.expectError(error.UnsupportedMeasurementVersion, compare(std.testing.io, arena, values, .{}, &report.writer));
+ values[0] = measurement_fixture;
+ values[1].harness = "other harness";
+ try std.testing.expectError(error.DifferentHarness, compare(std.testing.io, arena, values, .{}, &report.writer));
+ values[1] = measurement_fixture;
+ values[1].zig_version = "other compiler";
+ try std.testing.expectError(error.DifferentHarness, compare(std.testing.io, arena, values, .{}, &report.writer));
+ values[1] = measurement_fixture;
+ values[1].command = &.{"fixture --argument"};
+ try std.testing.expectError(error.DifferentCommands, compare(std.testing.io, arena, values, .{}, &report.writer));
+ values[1] = measurement_fixture;
+ values[1].passed = false;
+ try std.testing.expectError(error.FailedMeasurement, compare(std.testing.io, arena, values, .{}, &report.writer));
+ values[1] = measurement_fixture;
+ values[0].passed = false;
+ try std.testing.expectError(error.FailedMeasurement, compare(std.testing.io, arena, values, .{}, &report.writer));
+ try std.testing.expectEqualStrings("", report.written());
+}
+
+const benchmark_fixture =
+ \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
+ \\{"case":"last","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
+;
+
+test "history capture and performance gates reject false passes" {
+ var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer memory.deinit();
+ const arena = memory.allocator();
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ for (0..4) |run| {
+ const path = try std.fmt.allocPrint(arena, "before.{d}.stdout", .{run});
+ try tmp.dir.writeFile(io, .{ .sub_path = path, .data = benchmark_fixture });
+ }
+ try tmp.dir.writeFile(io, .{ .sub_path = "stderr", .data = "diagnostic\n" });
+ try tmp.dir.writeFile(io, .{ .sub_path = "changed.0.stdout", .data = "changed\n" });
+ var original = measurement_fixture;
+ original.stdout = try tmp.dir.realPathFileAlloc(io, "before.0.stdout", arena);
+ original.stderr = try tmp.dir.realPathFileAlloc(io, "stderr", arena);
+ const changed = try tmp.dir.realPathFileAlloc(io, "changed.0.stdout", arena);
+ var values = [2]Measurement{ original, original };
+ var report: std.Io.Writer.Allocating = .init(arena);
+ defer report.deinit();
+
+ try compare(io, arena, values, .{ .snapshots = true }, &report.writer);
+ values[1].stdout = changed;
+ try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
+ values[1] = original;
+ values[1].stderr = changed;
+ try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
+ for (0..2) |index| {
+ values = .{ original, original };
+ values[index].stable_stdout = false;
+ try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
+ values[index].stable_stdout = true;
+ values[index].stable_stderr = false;
+ try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
+ try compare(io, arena, values, .{}, &report.writer);
+ }
+ values = .{ original, original };
+ values[1].median_ns = 126;
+ try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .max_ratio = 1.25 }, &report.writer));
+ values[1].median_ns = 125;
+ try compare(io, arena, values, .{ .max_ratio = 1.25 }, &report.writer);
+
+ const slower_cold =
+ \\{"case":"last","bytes":20,"median_ns":50,"cold_ns":50,"allocations":5,"allocated_bytes":500}
+ \\{"case":"first","bytes":10,"median_ns":50,"cold_ns":126,"allocations":5,"allocated_bytes":500}
+ ;
+ const slower_warm =
+ \\{"case":"first","bytes":10,"median_ns":126,"cold_ns":50,"allocations":5,"allocated_bytes":500}
+ \\{"case":"last","bytes":20,"median_ns":50,"cold_ns":50,"allocations":5,"allocated_bytes":500}
+ ;
+ values[1].median_ns = 50;
+ values[1].stdout = changed;
+ for ([_][]const u8{ slower_cold, slower_warm }) |text| {
+ for (0..4) |run| {
+ const path = try std.fmt.allocPrint(arena, "changed.{d}.stdout", .{run});
+ try tmp.dir.writeFile(io, .{ .sub_path = path, .data = text });
+ }
+ const begin = report.written().len;
+ try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer));
+ const output = report.written()[begin..];
+ try std.testing.expect(std.mem.indexOf(u8, output, "warm runtime 0.500x") != null);
+ try std.testing.expect(std.mem.indexOf(u8, output, " REGRESSION\nlast:") != null);
+ try compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.26 }, &report.writer);
+ try compare(io, arena, values, .{ .benchmarks = true }, &report.writer);
+ }
+ values[1] = original;
+ values[1].median_ns = 200;
+ try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer));
+}
+
+test "history benchmark cases are nonempty unique and matched by name" {
+ var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer memory.deinit();
+ const arena = memory.allocator();
+ var report: std.Io.Writer.Allocating = .init(arena);
+ defer report.deinit();
+ try std.testing.expectError(error.NoBenchmarks, readBenchmarks(arena, " \n\t\n"));
+ try std.testing.expectError(error.DuplicateBenchmark, readBenchmarks(arena, benchmark_fixture ++ "\n" ++ benchmark_fixture));
+ try std.testing.expectError(error.MissingField, readBenchmarks(arena, "{\"case\":\"first\"}"));
+ const before = try readBenchmarks(arena, benchmark_fixture);
+ const missing = try readBenchmarks(arena, "{\"case\":\"first\",\"bytes\":10,\"median_ns\":100,\"cold_ns\":100,\"allocations\":10,\"allocated_bytes\":1000}");
+ try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, .{missing} ** 4, null, &report.writer));
+ const renamed =
+ \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
+ \\{"case":"renamed","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
+ ;
+ const different = try readBenchmarks(arena, renamed);
+ try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, .{different} ** 4, null, &report.writer));
+ const reordered =
+ \\{"case":"last","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
+ \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
+ ;
+ const same = try readBenchmarks(arena, reordered);
+ try std.testing.expect(!try compareBenchmarks(.{before} ** 4, .{same} ** 4, 1, &report.writer));
+ var samples = [4][]Benchmark{ before, before, before, different };
+ try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, samples, null, &report.writer));
+ samples[3] = try arena.dupe(Benchmark, before);
+ samples[3][0].bytes += 1;
+ try std.testing.expectError(error.DifferentBenchmarkInputs, compareBenchmarks(.{before} ** 4, samples, null, &report.writer));
+}
+
+test "history benchmark gate uses repeated process captures rather than the first process alone" {
+ var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer memory.deinit();
+ const arena = memory.allocator();
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var values = [2]Measurement{ measurement_fixture, measurement_fixture };
+ const root = try tmp.dir.realPathFileAlloc(io, ".", arena);
+ for (&values, 0..) |*value, side| {
+ value.stdout = try std.fmt.allocPrint(arena, "{s}/{d}.0.stdout", .{ root, side });
+ value.stderr = try std.fmt.allocPrint(arena, "{s}/{d}.0.stderr", .{ root, side });
+ try writeNew(io, value.stderr, "");
+ for (0..4) |run| {
+ const path = try std.fmt.allocPrint(arena, "{s}/{d}.{d}.stdout", .{ root, side, run });
+ const duration: u64 = if (side == 1 and run != 0) 150 else 100;
+ const line = try std.fmt.allocPrint(
+ arena,
+ "{{\"case\":\"one\",\"bytes\":10,\"median_ns\":{d},\"cold_ns\":{d},\"allocations\":1,\"allocated_bytes\":10}}\n",
+ .{ duration, duration },
+ );
+ try writeNew(io, path, line);
+ }
+ }
+ var report: std.Io.Writer.Allocating = .init(arena);
+ defer report.deinit();
+ try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer));
+ try std.testing.expect(std.mem.indexOf(u8, report.written(), "first cold 1.000x; cold 1.500x; median 1.500x") != null);
+ for ([_]u64{ 1000, 1000, 100, 100 }, 0..) |duration, run| {
+ const path = try std.fmt.allocPrint(arena, "1.{d}.stdout", .{run});
+ const line = try std.fmt.allocPrint(
+ arena,
+ "{{\"case\":\"one\",\"bytes\":10,\"median_ns\":{d},\"cold_ns\":{d},\"allocations\":1,\"allocated_bytes\":10}}\n",
+ .{ duration, duration },
+ );
+ try tmp.dir.writeFile(io, .{ .sub_path = path, .data = line });
+ }
+ const begin = report.written().len;
+ try compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer);
+ try std.testing.expect(std.mem.indexOf(u8, report.written()[begin..], "first cold 10.000x; cold 1.000x; median 1.000x") != null);
+ try tmp.dir.deleteFile(io, "1.3.stdout");
+ try std.testing.expectError(error.FileNotFound, compare(io, arena, values, .{ .benchmarks = true }, &report.writer));
+ values[1].samples_ns = &.{ 100, 100, 100 };
+ try std.testing.expectError(error.InvalidBenchmarkCaptures, compare(io, arena, values, .{ .benchmarks = true }, &report.writer));
+ values[1].samples_ns = measurement_fixture.samples_ns;
+ values[1].stdout = values[1].stderr;
+ try std.testing.expectError(error.InvalidBenchmarkCaptures, compare(io, arena, values, .{ .benchmarks = true }, &report.writer));
+}