diff options
| author | Gabriel Schneider <[email protected]> | 2026-09-06 18:11:36 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-09-07 13:59:12 -0300 |
| commit | 60367d8fe23f6af98ec28e3cf6c2094dfe332df0 (patch) | |
| tree | 310fc734173cf771881f4691c71909135fadde97 /test/history.zig | |
| parent | fa82cac885cb4738fe36d1e49b4749b5a3e31a4a (diff) | |
| download | pardes-60367d8fe23f6af98ec28e3cf6c2094dfe332df0.tar.gz pardes-60367d8fe23f6af98ec28e3cf6c2094dfe332df0.zip | |
Refactor panes and filesystem; replace FUSE with 9P
Consolidate pane, layout, memory and host code. Serve 9P by default over Unix sockets, with runtime mounts and optional TCP/QUIC transports. Remove FUSE and obsolete proof-of-concept examples.
Fix highlighting and terminal-history performance, expand differential and stress-test infrastructure, sort navigation results while preserving the next occurrence, add syntax-colored Braille minimaps, remove SPC-k, and document 9P interaction as a repository skill.
Diffstat (limited to 'test/history.zig')
| -rw-r--r-- | test/history.zig | 502 |
1 files changed, 502 insertions, 0 deletions
diff --git a/test/history.zig b/test/history.zig new file mode 100644 index 00000000..5ce45239 --- /dev/null +++ b/test/history.zig @@ -0,0 +1,502 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const Dir = std.Io.Dir; +const schema_version = 2; +const harness_id = blk: { + @setEvalBranchQuota(100_000); + break :blk std.fmt.comptimePrint("{x}", .{std.hash.Wyhash.hash(0, @embedFile("history.zig"))}); +}; + +const Measurement = struct { + schema: u32 = 0, + harness: []const u8 = "", + zig_version: []const u8 = "", + revision: []const u8, + cwd: []const u8, + command: []const []const u8, + first_ns: u64, + median_ns: u64, + samples_ns: []const u64, + stdout: []const u8, + stderr: []const u8, + stable_stdout: bool, + stable_stderr: bool = false, + passed: bool, +}; + +const Compare = struct { + snapshots: bool = false, + benchmarks: bool = false, + max_ratio: ?f64 = null, +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + const args = try init.minimal.args.toSlice(arena); + if (args.len < 2) return usage(io); + if (std.mem.eql(u8, args[1], "run")) { + const immutable = args.len > 2 and std.mem.eql(u8, args[2], "--allow-immutable"); + const pos: usize = if (immutable) 3 else 2; + if (args.len < pos + 4 or !std.mem.eql(u8, args[pos + 2], "--")) return usage(io); + try Dir.cwd().createDirPath(io, args[pos + 1]); + const output = try Dir.cwd().realPathFileAlloc(io, args[pos + 1], arena); + const exe = try std.process.executablePathAlloc(io, arena); + var command: std.ArrayList([]const u8) = .empty; + try command.appendSlice(arena, &.{ "jj", "--ignore-working-copy", "--no-integrate-operation" }); + if (immutable) try command.append(arena, "--ignore-immutable"); + try command.appendSlice(arena, &.{ "run", "--root", "-j", "1", "-r", args[pos], "--", exe, "record", output, "--" }); + try command.appendSlice(arena, args[pos + 3 ..]); + const result = try std.process.run(arena, io, .{ .argv = command.items }); + try std.Io.File.stdout().writeStreamingAll(io, result.stdout); + try std.Io.File.stderr().writeStreamingAll(io, result.stderr); + if (result.term != .exited or result.term.exited != 0) return error.HistoryRunFailed; + } else if (std.mem.eql(u8, args[1], "record")) { + if (args.len < 5 or !std.mem.eql(u8, args[3], "--")) return usage(io); + const command = args[4..]; + try Dir.cwd().createDirPath(io, args[2]); + const output = try Dir.cwd().realPathFileAlloc(io, args[2], arena); + const revision = init.environ_map.get("JJ_COMMIT_ID") orelse "working"; + const encoded_command = try std.json.Stringify.valueAlloc(arena, command, .{}); + const stem = try std.fmt.allocPrint(arena, "{s}/{s}-{x}", .{ + output, revision, std.hash.Wyhash.hash(0, encoded_command), + }); + try record(io, arena, stem, revision, command); + } else if (std.mem.eql(u8, args[1], "compare")) { + if (args.len < 4) return usage(io); + var options: Compare = .{}; + for (args[4..]) |arg| { + if (std.mem.eql(u8, arg, "--snapshots")) { + options.snapshots = true; + } else if (std.mem.eql(u8, arg, "--benchmarks")) { + options.benchmarks = true; + } else if (options.max_ratio == null) { + const ratio = std.fmt.parseFloat(f64, arg) catch return usage(io); + if (!std.math.isFinite(ratio) or ratio <= 0) return error.InvalidRatio; + options.max_ratio = ratio; + } else return usage(io); + } + if (options.snapshots and options.benchmarks) return usage(io); + var values: [2]Measurement = undefined; + for (&values, args[2..4]) |*value, path| { + const bytes = try Dir.cwd().readFileAlloc(io, path, arena, .limited(1024 * 1024)); + value.* = (try std.json.parseFromSlice(Measurement, arena, bytes, .{})).value; + } + var report: std.Io.Writer.Allocating = .init(arena); + defer report.deinit(); + const result = compare(io, arena, values, options, &report.writer); + try std.Io.File.stdout().writeStreamingAll(io, report.written()); + try result; + } else return usage(io); +} + +fn writeNew(io: std.Io, path: []const u8, bytes: []const u8) !void { + const file = try Dir.cwd().createFile(io, path, .{ .exclusive = true }); + defer file.close(io); + try file.writeStreamingAll(io, bytes); +} + +fn record(io: std.Io, arena: std.mem.Allocator, stem: []const u8, revision: []const u8, command: []const []const u8) !void { + const reservation = try std.fmt.allocPrint(arena, "{s}.reserved", .{stem}); + writeNew(io, reservation, "") catch |err| return if (err == error.PathAlreadyExists) error.MeasurementExists else err; + for ([_][]const u8{ ".json", ".0.stdout", ".0.stderr", ".1.stdout", ".1.stderr", ".2.stdout", ".2.stderr", ".3.stdout", ".3.stderr" }) |suffix| { + const path = try std.fmt.allocPrint(arena, "{s}{s}", .{ stem, suffix }); + _ = Dir.cwd().statFile(io, path, .{ .follow_symlinks = false }) catch |err| { + if (err == error.FileNotFound) continue; + return err; + }; + return error.MeasurementExists; + } + var times: [4]u64 = undefined; + var first_output: []const u8 = ""; + var first_error: []const u8 = ""; + var stable_stdout = true; + var stable_stderr = true; + var passed = true; + for (×, 0..) |*elapsed, index| { + const started = std.Io.Clock.awake.now(io); + const result = try std.process.run(arena, io, .{ + .argv = command, + .stdout_limit = .limited(64 * 1024 * 1024), + .stderr_limit = .limited(64 * 1024 * 1024), + }); + elapsed.* = @intCast(started.durationTo(std.Io.Clock.awake.now(io)).toNanoseconds()); + passed = passed and result.term == .exited and result.term.exited == 0; + if (index == 0) { + first_output = result.stdout; + first_error = result.stderr; + } else { + stable_stdout = stable_stdout and std.mem.eql(u8, first_output, result.stdout); + stable_stderr = stable_stderr and std.mem.eql(u8, first_error, result.stderr); + } + try writeNew(io, try std.fmt.allocPrint(arena, "{s}.{d}.stdout", .{ stem, index }), result.stdout); + try writeNew(io, try std.fmt.allocPrint(arena, "{s}.{d}.stderr", .{ stem, index }), result.stderr); + } + var warmed = times[1..].*; + std.mem.sort(u64, &warmed, {}, std.sort.asc(u64)); + const measurement: Measurement = .{ + .schema = schema_version, + .harness = harness_id, + .zig_version = builtin.zig_version_string, + .revision = revision, + .cwd = try std.process.currentPathAlloc(io, arena), + .command = command, + .first_ns = times[0], + .median_ns = warmed[1], + .samples_ns = ×, + .stdout = try std.fmt.allocPrint(arena, "{s}.0.stdout", .{stem}), + .stderr = try std.fmt.allocPrint(arena, "{s}.0.stderr", .{stem}), + .stable_stdout = stable_stdout, + .stable_stderr = stable_stderr, + .passed = passed, + }; + const json = try std.json.Stringify.valueAlloc(arena, measurement, .{ .whitespace = .indent_2 }); + const path = try std.fmt.allocPrint(arena, "{s}.json", .{stem}); + try writeNew(io, path, json); + const report = try std.fmt.allocPrint(arena, "{s}: first {d:.3}s, warm median {d:.3}s, passed={}\n", .{ + path, @as(f64, @floatFromInt(times[0])) / 1e9, @as(f64, @floatFromInt(warmed[1])) / 1e9, passed, + }); + try std.Io.File.stdout().writeStreamingAll(io, report); + if (!passed) return error.CommandFailed; +} + +fn compare(io: std.Io, arena: std.mem.Allocator, values: [2]Measurement, options: Compare, report: *std.Io.Writer) !void { + for (values) |value| if (value.schema != schema_version) return error.UnsupportedMeasurementVersion; + if (!std.mem.eql(u8, values[0].harness, values[1].harness) or + !std.mem.eql(u8, values[0].zig_version, values[1].zig_version)) return error.DifferentHarness; + if (!values[0].passed or !values[1].passed) return error.FailedMeasurement; + const a = try std.json.Stringify.valueAlloc(arena, values[0].command, .{}); + const b = try std.json.Stringify.valueAlloc(arena, values[1].command, .{}); + if (!std.mem.eql(u8, a, b)) return error.DifferentCommands; + const ratio = @as(f64, @floatFromInt(values[1].median_ns)) / @as(f64, @floatFromInt(@max(1, values[0].median_ns))); + const before = try Dir.cwd().readFileAlloc(io, values[0].stdout, arena, .limited(64 * 1024 * 1024)); + const after = try Dir.cwd().readFileAlloc(io, values[1].stdout, arena, .limited(64 * 1024 * 1024)); + const before_error = try Dir.cwd().readFileAlloc(io, values[0].stderr, arena, .limited(64 * 1024 * 1024)); + const after_error = try Dir.cwd().readFileAlloc(io, values[1].stderr, arena, .limited(64 * 1024 * 1024)); + const same_stdout = std.mem.eql(u8, before, after); + const same_stderr = std.mem.eql(u8, before_error, after_error); + const stable = values[0].stable_stdout and values[1].stable_stdout and values[0].stable_stderr and values[1].stable_stderr; + try report.print("warm runtime {d:.3}x; stdout {s}; stderr {s}; repeated captures stable={}\n", .{ + ratio, if (same_stdout) "identical" else "DIFFERS", if (same_stderr) "identical" else "DIFFERS", stable, + }); + if (options.snapshots and (!same_stdout or !same_stderr or !stable)) return error.SnapshotMismatch; + if (options.benchmarks) { + const before_runs = try readBenchmarkRuns(io, arena, values[0]); + const after_runs = try readBenchmarkRuns(io, arena, values[1]); + if (try compareBenchmarks(before_runs, after_runs, options.max_ratio, report)) return error.PerformanceRegression; + } + if (options.max_ratio) |limit| if (ratio > limit) return error.PerformanceRegression; +} + +const Benchmark = struct { + case: []const u8, + bytes: u64, + median_ns: u64, + cold_ns: u64, + allocations: u64, + allocated_bytes: u64, +}; + +fn readBenchmarks(arena: std.mem.Allocator, text: []const u8) ![]Benchmark { + var result: std.ArrayList(Benchmark) = .empty; + var lines = std.mem.splitScalar(u8, text, '\n'); + while (lines.next()) |line| { + if (std.mem.trim(u8, line, " \t\r").len == 0) continue; + const value = (try std.json.parseFromSlice(Benchmark, arena, line, .{ .ignore_unknown_fields = true })).value; + if (value.case.len == 0) return error.EmptyBenchmarkName; + for (result.items) |old| if (std.mem.eql(u8, old.case, value.case)) return error.DuplicateBenchmark; + try result.append(arena, value); + } + if (result.items.len == 0) return error.NoBenchmarks; + return result.toOwnedSlice(arena); +} + +fn readBenchmarkRuns(io: std.Io, arena: std.mem.Allocator, value: Measurement) ![4][]Benchmark { + const suffix = ".0.stdout"; + if (value.samples_ns.len != 4 or !std.mem.endsWith(u8, value.stdout, suffix)) return error.InvalidBenchmarkCaptures; + const stem = value.stdout[0 .. value.stdout.len - suffix.len]; + var runs: [4][]Benchmark = undefined; + for (&runs, 0..) |*run, index| { + const path = try std.fmt.allocPrint(arena, "{s}.{d}.stdout", .{ stem, index }); + const bytes = try Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024 * 1024)); + run.* = try readBenchmarks(arena, bytes); + } + return runs; +} + +fn compareBenchmarks(before: [4][]Benchmark, after: [4][]Benchmark, max_ratio: ?f64, report: *std.Io.Writer) !bool { + const runs = [2][4][]Benchmark{ before, after }; + for (runs) |side| for (side) |run| { + if (run.len != before[0].len) return error.DifferentBenchmarks; + }; + var regressed = false; + for (before[0]) |first| { + var matched: [2][4]Benchmark = undefined; + for (runs, &matched) |side, *samples| for (side, samples) |run, *sample| { + sample.* = for (run) |value| { + if (std.mem.eql(u8, first.case, value.case)) break value; + } else return error.DifferentBenchmarks; + if (sample.bytes != first.bytes) return error.DifferentBenchmarkInputs; + }; + var medians = [2]Benchmark{ first, first }; + for (matched, &medians) |samples, *median| { + inline for (.{ "cold_ns", "median_ns", "allocations", "allocated_bytes" }) |field| { + var warm: [3]u64 = undefined; + for (samples[1..], &warm) |sample, *value| value.* = @field(sample, field); + std.mem.sort(u64, &warm, {}, std.sort.asc(u64)); + @field(median, field) = warm[1]; + } + } + const old = medians[0]; + const current = medians[1]; + const first_cold = @as(f64, @floatFromInt(matched[1][0].cold_ns)) / @as(f64, @floatFromInt(@max(1, matched[0][0].cold_ns))); + const cold = @as(f64, @floatFromInt(current.cold_ns)) / @as(f64, @floatFromInt(@max(1, old.cold_ns))); + const median = @as(f64, @floatFromInt(current.median_ns)) / @as(f64, @floatFromInt(@max(1, old.median_ns))); + const case_regression = if (max_ratio) |limit| cold > limit or median > limit else false; + regressed = regressed or case_regression; + try report.print("{s}: first cold {d:.3}x; cold {d:.3}x; median {d:.3}x; allocations {d}->{d}; bytes {d}->{d}{s}\n", .{ + old.case, + first_cold, + cold, + median, + old.allocations, + current.allocations, + old.allocated_bytes, + current.allocated_bytes, + if (case_regression) " REGRESSION" else "", + }); + } + return regressed; +} + +fn usage(io: std.Io) !void { + try std.Io.File.stderr().writeStreamingAll(io, "history run [--allow-immutable] REVSET OUT -- COMMAND...\n" ++ + "history record OUT -- COMMAND...\n" ++ + "history compare BEFORE.json AFTER.json [MAX_RATIO] [--snapshots|--benchmarks]\n" ++ + "run uses isolated jj workspaces without integrating history changes.\n" ++ + "Commands still have normal filesystem and network access.\n"); + return error.InvalidArguments; +} + +test "history output collisions preserve completed and partial recordings before command dispatch" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const gpa = arena.allocator(); + const io = std.testing.io; + for ([_][]const u8{ ".reserved", ".json", ".0.stdout", ".0.stderr", ".1.stdout", ".1.stderr", ".2.stdout", ".2.stderr", ".3.stdout", ".3.stderr" }) |suffix| { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const output = try tmp.dir.realPathFileAlloc(io, ".", gpa); + const stem = try std.fmt.allocPrint(gpa, "{s}/measurement", .{output}); + const path = try std.fmt.allocPrint(gpa, "{s}{s}", .{ stem, suffix }); + try writeNew(io, path, "previous recording\n"); + const missing = try std.fmt.allocPrint(gpa, "{s}/must-not-be-dispatched", .{output}); + try std.testing.expectError(error.MeasurementExists, record(io, gpa, stem, "fixture", &.{missing})); + const preserved = try Dir.cwd().readFileAlloc(io, path, gpa, .limited(1024)); + try std.testing.expectEqualStrings("previous recording\n", preserved); + try std.testing.expectError(error.MeasurementExists, record(io, gpa, stem, "fixture", &.{missing})); + } +} + +const measurement_fixture: Measurement = .{ + .schema = schema_version, + .harness = harness_id, + .zig_version = builtin.zig_version_string, + .revision = "fixture", + .cwd = "", + .command = &.{ "fixture", "--argument" }, + .first_ns = 100, + .median_ns = 100, + .samples_ns = &.{ 100, 100, 100, 100 }, + .stdout = "", + .stderr = "", + .stable_stdout = true, + .stable_stderr = true, + .passed = true, +}; + +test "history rejects incomparable and failed measurements before reading captures" { + var memory: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer memory.deinit(); + const arena = memory.allocator(); + var report: std.Io.Writer.Allocating = .init(arena); + defer report.deinit(); + var values = [2]Measurement{ measurement_fixture, measurement_fixture }; + + values[1].schema = 0; + try std.testing.expectError(error.UnsupportedMeasurementVersion, compare(std.testing.io, arena, values, .{}, &report.writer)); + values[1] = measurement_fixture; + values[0].schema = schema_version + 1; + try std.testing.expectError(error.UnsupportedMeasurementVersion, compare(std.testing.io, arena, values, .{}, &report.writer)); + values[0] = measurement_fixture; + values[1].harness = "other harness"; + try std.testing.expectError(error.DifferentHarness, compare(std.testing.io, arena, values, .{}, &report.writer)); + values[1] = measurement_fixture; + values[1].zig_version = "other compiler"; + try std.testing.expectError(error.DifferentHarness, compare(std.testing.io, arena, values, .{}, &report.writer)); + values[1] = measurement_fixture; + values[1].command = &.{"fixture --argument"}; + try std.testing.expectError(error.DifferentCommands, compare(std.testing.io, arena, values, .{}, &report.writer)); + values[1] = measurement_fixture; + values[1].passed = false; + try std.testing.expectError(error.FailedMeasurement, compare(std.testing.io, arena, values, .{}, &report.writer)); + values[1] = measurement_fixture; + values[0].passed = false; + try std.testing.expectError(error.FailedMeasurement, compare(std.testing.io, arena, values, .{}, &report.writer)); + try std.testing.expectEqualStrings("", report.written()); +} + +const benchmark_fixture = + \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000} + \\{"case":"last","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000} +; + +test "history capture and performance gates reject false passes" { + var memory: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer memory.deinit(); + const arena = memory.allocator(); + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + for (0..4) |run| { + const path = try std.fmt.allocPrint(arena, "before.{d}.stdout", .{run}); + try tmp.dir.writeFile(io, .{ .sub_path = path, .data = benchmark_fixture }); + } + try tmp.dir.writeFile(io, .{ .sub_path = "stderr", .data = "diagnostic\n" }); + try tmp.dir.writeFile(io, .{ .sub_path = "changed.0.stdout", .data = "changed\n" }); + var original = measurement_fixture; + original.stdout = try tmp.dir.realPathFileAlloc(io, "before.0.stdout", arena); + original.stderr = try tmp.dir.realPathFileAlloc(io, "stderr", arena); + const changed = try tmp.dir.realPathFileAlloc(io, "changed.0.stdout", arena); + var values = [2]Measurement{ original, original }; + var report: std.Io.Writer.Allocating = .init(arena); + defer report.deinit(); + + try compare(io, arena, values, .{ .snapshots = true }, &report.writer); + values[1].stdout = changed; + try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer)); + values[1] = original; + values[1].stderr = changed; + try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer)); + for (0..2) |index| { + values = .{ original, original }; + values[index].stable_stdout = false; + try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer)); + values[index].stable_stdout = true; + values[index].stable_stderr = false; + try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer)); + try compare(io, arena, values, .{}, &report.writer); + } + values = .{ original, original }; + values[1].median_ns = 126; + try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .max_ratio = 1.25 }, &report.writer)); + values[1].median_ns = 125; + try compare(io, arena, values, .{ .max_ratio = 1.25 }, &report.writer); + + const slower_cold = + \\{"case":"last","bytes":20,"median_ns":50,"cold_ns":50,"allocations":5,"allocated_bytes":500} + \\{"case":"first","bytes":10,"median_ns":50,"cold_ns":126,"allocations":5,"allocated_bytes":500} + ; + const slower_warm = + \\{"case":"first","bytes":10,"median_ns":126,"cold_ns":50,"allocations":5,"allocated_bytes":500} + \\{"case":"last","bytes":20,"median_ns":50,"cold_ns":50,"allocations":5,"allocated_bytes":500} + ; + values[1].median_ns = 50; + values[1].stdout = changed; + for ([_][]const u8{ slower_cold, slower_warm }) |text| { + for (0..4) |run| { + const path = try std.fmt.allocPrint(arena, "changed.{d}.stdout", .{run}); + try tmp.dir.writeFile(io, .{ .sub_path = path, .data = text }); + } + const begin = report.written().len; + try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer)); + const output = report.written()[begin..]; + try std.testing.expect(std.mem.indexOf(u8, output, "warm runtime 0.500x") != null); + try std.testing.expect(std.mem.indexOf(u8, output, " REGRESSION\nlast:") != null); + try compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.26 }, &report.writer); + try compare(io, arena, values, .{ .benchmarks = true }, &report.writer); + } + values[1] = original; + values[1].median_ns = 200; + try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer)); +} + +test "history benchmark cases are nonempty unique and matched by name" { + var memory: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer memory.deinit(); + const arena = memory.allocator(); + var report: std.Io.Writer.Allocating = .init(arena); + defer report.deinit(); + try std.testing.expectError(error.NoBenchmarks, readBenchmarks(arena, " \n\t\n")); + try std.testing.expectError(error.DuplicateBenchmark, readBenchmarks(arena, benchmark_fixture ++ "\n" ++ benchmark_fixture)); + try std.testing.expectError(error.MissingField, readBenchmarks(arena, "{\"case\":\"first\"}")); + const before = try readBenchmarks(arena, benchmark_fixture); + const missing = try readBenchmarks(arena, "{\"case\":\"first\",\"bytes\":10,\"median_ns\":100,\"cold_ns\":100,\"allocations\":10,\"allocated_bytes\":1000}"); + try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, .{missing} ** 4, null, &report.writer)); + const renamed = + \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000} + \\{"case":"renamed","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000} + ; + const different = try readBenchmarks(arena, renamed); + try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, .{different} ** 4, null, &report.writer)); + const reordered = + \\{"case":"last","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000} + \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000} + ; + const same = try readBenchmarks(arena, reordered); + try std.testing.expect(!try compareBenchmarks(.{before} ** 4, .{same} ** 4, 1, &report.writer)); + var samples = [4][]Benchmark{ before, before, before, different }; + try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, samples, null, &report.writer)); + samples[3] = try arena.dupe(Benchmark, before); + samples[3][0].bytes += 1; + try std.testing.expectError(error.DifferentBenchmarkInputs, compareBenchmarks(.{before} ** 4, samples, null, &report.writer)); +} + +test "history benchmark gate uses repeated process captures rather than the first process alone" { + var memory: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer memory.deinit(); + const arena = memory.allocator(); + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var values = [2]Measurement{ measurement_fixture, measurement_fixture }; + const root = try tmp.dir.realPathFileAlloc(io, ".", arena); + for (&values, 0..) |*value, side| { + value.stdout = try std.fmt.allocPrint(arena, "{s}/{d}.0.stdout", .{ root, side }); + value.stderr = try std.fmt.allocPrint(arena, "{s}/{d}.0.stderr", .{ root, side }); + try writeNew(io, value.stderr, ""); + for (0..4) |run| { + const path = try std.fmt.allocPrint(arena, "{s}/{d}.{d}.stdout", .{ root, side, run }); + const duration: u64 = if (side == 1 and run != 0) 150 else 100; + const line = try std.fmt.allocPrint( + arena, + "{{\"case\":\"one\",\"bytes\":10,\"median_ns\":{d},\"cold_ns\":{d},\"allocations\":1,\"allocated_bytes\":10}}\n", + .{ duration, duration }, + ); + try writeNew(io, path, line); + } + } + var report: std.Io.Writer.Allocating = .init(arena); + defer report.deinit(); + try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer)); + try std.testing.expect(std.mem.indexOf(u8, report.written(), "first cold 1.000x; cold 1.500x; median 1.500x") != null); + for ([_]u64{ 1000, 1000, 100, 100 }, 0..) |duration, run| { + const path = try std.fmt.allocPrint(arena, "1.{d}.stdout", .{run}); + const line = try std.fmt.allocPrint( + arena, + "{{\"case\":\"one\",\"bytes\":10,\"median_ns\":{d},\"cold_ns\":{d},\"allocations\":1,\"allocated_bytes\":10}}\n", + .{ duration, duration }, + ); + try tmp.dir.writeFile(io, .{ .sub_path = path, .data = line }); + } + const begin = report.written().len; + try compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer); + try std.testing.expect(std.mem.indexOf(u8, report.written()[begin..], "first cold 10.000x; cold 1.000x; median 1.000x") != null); + try tmp.dir.deleteFile(io, "1.3.stdout"); + try std.testing.expectError(error.FileNotFound, compare(io, arena, values, .{ .benchmarks = true }, &report.writer)); + values[1].samples_ns = &.{ 100, 100, 100 }; + try std.testing.expectError(error.InvalidBenchmarkCaptures, compare(io, arena, values, .{ .benchmarks = true }, &report.writer)); + values[1].samples_ns = measurement_fixture.samples_ns; + values[1].stdout = values[1].stderr; + try std.testing.expectError(error.InvalidBenchmarkCaptures, compare(io, arena, values, .{ .benchmarks = true }, &report.writer)); +} |
