path: root/test/history.zig

const std = @import("std");
const builtin = @import("builtin");
const Dir = std.Io.Dir;
const schema_version = 2;
const harness_id = blk: {
    @setEvalBranchQuota(100_000);
    break :blk std.fmt.comptimePrint("{x}", .{std.hash.Wyhash.hash(0, @embedFile("history.zig"))});
};

const Measurement = struct {
    schema: u32 = 0,
    harness: []const u8 = "",
    zig_version: []const u8 = "",
    revision: []const u8,
    cwd: []const u8,
    command: []const []const u8,
    first_ns: u64,
    median_ns: u64,
    samples_ns: []const u64,
    stdout: []const u8,
    stderr: []const u8,
    stable_stdout: bool,
    stable_stderr: bool = false,
    passed: bool,
};

const Compare = struct {
    snapshots: bool = false,
    benchmarks: bool = false,
    max_ratio: ?f64 = null,
};

pub fn main(init: std.process.Init) !void {
    const arena = init.arena.allocator();
    const io = init.io;
    const args = try init.minimal.args.toSlice(arena);
    if (args.len < 2) return usage(io);
    if (std.mem.eql(u8, args[1], "run")) {
        const immutable = args.len > 2 and std.mem.eql(u8, args[2], "--allow-immutable");
        const pos: usize = if (immutable) 3 else 2;
        if (args.len < pos + 4 or !std.mem.eql(u8, args[pos + 2], "--")) return usage(io);
        try Dir.cwd().createDirPath(io, args[pos + 1]);
        const output = try Dir.cwd().realPathFileAlloc(io, args[pos + 1], arena);
        const exe = try std.process.executablePathAlloc(io, arena);
        var command: std.ArrayList([]const u8) = .empty;
        try command.appendSlice(arena, &.{ "jj", "--ignore-working-copy", "--no-integrate-operation" });
        if (immutable) try command.append(arena, "--ignore-immutable");
        try command.appendSlice(arena, &.{ "run", "--root", "-j", "1", "-r", args[pos], "--", exe, "record", output, "--" });
        try command.appendSlice(arena, args[pos + 3 ..]);
        const result = try std.process.run(arena, io, .{ .argv = command.items });
        try std.Io.File.stdout().writeStreamingAll(io, result.stdout);
        try std.Io.File.stderr().writeStreamingAll(io, result.stderr);
        if (result.term != .exited or result.term.exited != 0) return error.HistoryRunFailed;
    } else if (std.mem.eql(u8, args[1], "record")) {
        if (args.len < 5 or !std.mem.eql(u8, args[3], "--")) return usage(io);
        const command = args[4..];
        try Dir.cwd().createDirPath(io, args[2]);
        const output = try Dir.cwd().realPathFileAlloc(io, args[2], arena);
        const revision = init.environ_map.get("JJ_COMMIT_ID") orelse "working";
        const encoded_command = try std.json.Stringify.valueAlloc(arena, command, .{});
        const stem = try std.fmt.allocPrint(arena, "{s}/{s}-{x}", .{
            output, revision, std.hash.Wyhash.hash(0, encoded_command),
        });
        try record(io, arena, stem, revision, command);
    } else if (std.mem.eql(u8, args[1], "compare")) {
        if (args.len < 4) return usage(io);
        var options: Compare = .{};
        for (args[4..]) |arg| {
            if (std.mem.eql(u8, arg, "--snapshots")) {
                options.snapshots = true;
            } else if (std.mem.eql(u8, arg, "--benchmarks")) {
                options.benchmarks = true;
            } else if (options.max_ratio == null) {
                const ratio = std.fmt.parseFloat(f64, arg) catch return usage(io);
                if (!std.math.isFinite(ratio) or ratio <= 0) return error.InvalidRatio;
                options.max_ratio = ratio;
            } else return usage(io);
        }
        if (options.snapshots and options.benchmarks) return usage(io);
        var values: [2]Measurement = undefined;
        for (&values, args[2..4]) |*value, path| {
            const bytes = try Dir.cwd().readFileAlloc(io, path, arena, .limited(1024 * 1024));
            value.* = (try std.json.parseFromSlice(Measurement, arena, bytes, .{})).value;
        }
        var report: std.Io.Writer.Allocating = .init(arena);
        defer report.deinit();
        const result = compare(io, arena, values, options, &report.writer);
        try std.Io.File.stdout().writeStreamingAll(io, report.written());
        try result;
    } else return usage(io);
}

fn writeNew(io: std.Io, path: []const u8, bytes: []const u8) !void {
    const file = try Dir.cwd().createFile(io, path, .{ .exclusive = true });
    defer file.close(io);
    try file.writeStreamingAll(io, bytes);
}

fn record(io: std.Io, arena: std.mem.Allocator, stem: []const u8, revision: []const u8, command: []const []const u8) !void {
    const reservation = try std.fmt.allocPrint(arena, "{s}.reserved", .{stem});
    writeNew(io, reservation, "") catch |err| return if (err == error.PathAlreadyExists) error.MeasurementExists else err;
    for ([_][]const u8{ ".json", ".0.stdout", ".0.stderr", ".1.stdout", ".1.stderr", ".2.stdout", ".2.stderr", ".3.stdout", ".3.stderr" }) |suffix| {
        const path = try std.fmt.allocPrint(arena, "{s}{s}", .{ stem, suffix });
        _ = Dir.cwd().statFile(io, path, .{ .follow_symlinks = false }) catch |err| {
            if (err == error.FileNotFound) continue;
            return err;
        };
        return error.MeasurementExists;
    }
    var times: [4]u64 = undefined;
    var first_output: []const u8 = "";
    var first_error: []const u8 = "";
    var stable_stdout = true;
    var stable_stderr = true;
    var passed = true;
    for (&times, 0..) |*elapsed, index| {
        const started = std.Io.Clock.awake.now(io);
        const result = try std.process.run(arena, io, .{
            .argv = command,
            .stdout_limit = .limited(64 * 1024 * 1024),
            .stderr_limit = .limited(64 * 1024 * 1024),
        });
        elapsed.* = @intCast(started.durationTo(std.Io.Clock.awake.now(io)).toNanoseconds());
        passed = passed and result.term == .exited and result.term.exited == 0;
        if (index == 0) {
            first_output = result.stdout;
            first_error = result.stderr;
        } else {
            stable_stdout = stable_stdout and std.mem.eql(u8, first_output, result.stdout);
            stable_stderr = stable_stderr and std.mem.eql(u8, first_error, result.stderr);
        }
        try writeNew(io, try std.fmt.allocPrint(arena, "{s}.{d}.stdout", .{ stem, index }), result.stdout);
        try writeNew(io, try std.fmt.allocPrint(arena, "{s}.{d}.stderr", .{ stem, index }), result.stderr);
    }
    var warmed = times[1..].*;
    std.mem.sort(u64, &warmed, {}, std.sort.asc(u64));
    const measurement: Measurement = .{
        .schema = schema_version,
        .harness = harness_id,
        .zig_version = builtin.zig_version_string,
        .revision = revision,
        .cwd = try std.process.currentPathAlloc(io, arena),
        .command = command,
        .first_ns = times[0],
        .median_ns = warmed[1],
        .samples_ns = &times,
        .stdout = try std.fmt.allocPrint(arena, "{s}.0.stdout", .{stem}),
        .stderr = try std.fmt.allocPrint(arena, "{s}.0.stderr", .{stem}),
        .stable_stdout = stable_stdout,
        .stable_stderr = stable_stderr,
        .passed = passed,
    };
    const json = try std.json.Stringify.valueAlloc(arena, measurement, .{ .whitespace = .indent_2 });
    const path = try std.fmt.allocPrint(arena, "{s}.json", .{stem});
    try writeNew(io, path, json);
    const report = try std.fmt.allocPrint(arena, "{s}: first {d:.3}s, warm median {d:.3}s, passed={}\n", .{
        path, @as(f64, @floatFromInt(times[0])) / 1e9, @as(f64, @floatFromInt(warmed[1])) / 1e9, passed,
    });
    try std.Io.File.stdout().writeStreamingAll(io, report);
    if (!passed) return error.CommandFailed;
}

fn compare(io: std.Io, arena: std.mem.Allocator, values: [2]Measurement, options: Compare, report: *std.Io.Writer) !void {
    for (values) |value| if (value.schema != schema_version) return error.UnsupportedMeasurementVersion;
    if (!std.mem.eql(u8, values[0].harness, values[1].harness) or
        !std.mem.eql(u8, values[0].zig_version, values[1].zig_version)) return error.DifferentHarness;
    if (!values[0].passed or !values[1].passed) return error.FailedMeasurement;
    const a = try std.json.Stringify.valueAlloc(arena, values[0].command, .{});
    const b = try std.json.Stringify.valueAlloc(arena, values[1].command, .{});
    if (!std.mem.eql(u8, a, b)) return error.DifferentCommands;
    const ratio = @as(f64, @floatFromInt(values[1].median_ns)) / @as(f64, @floatFromInt(@max(1, values[0].median_ns)));
    const before = try Dir.cwd().readFileAlloc(io, values[0].stdout, arena, .limited(64 * 1024 * 1024));
    const after = try Dir.cwd().readFileAlloc(io, values[1].stdout, arena, .limited(64 * 1024 * 1024));
    const before_error = try Dir.cwd().readFileAlloc(io, values[0].stderr, arena, .limited(64 * 1024 * 1024));
    const after_error = try Dir.cwd().readFileAlloc(io, values[1].stderr, arena, .limited(64 * 1024 * 1024));
    const same_stdout = std.mem.eql(u8, before, after);
    const same_stderr = std.mem.eql(u8, before_error, after_error);
    const stable = values[0].stable_stdout and values[1].stable_stdout and values[0].stable_stderr and values[1].stable_stderr;
    try report.print("warm runtime {d:.3}x; stdout {s}; stderr {s}; repeated captures stable={}\n", .{
        ratio, if (same_stdout) "identical" else "DIFFERS", if (same_stderr) "identical" else "DIFFERS", stable,
    });
    if (options.snapshots and (!same_stdout or !same_stderr or !stable)) return error.SnapshotMismatch;
    if (options.benchmarks) {
        const before_runs = try readBenchmarkRuns(io, arena, values[0]);
        const after_runs = try readBenchmarkRuns(io, arena, values[1]);
        if (try compareBenchmarks(before_runs, after_runs, options.max_ratio, report)) return error.PerformanceRegression;
    }
    if (options.max_ratio) |limit| if (ratio > limit) return error.PerformanceRegression;
}

const Benchmark = struct {
    case: []const u8,
    bytes: u64,
    median_ns: u64,
    cold_ns: u64,
    allocations: u64,
    allocated_bytes: u64,
};

fn readBenchmarks(arena: std.mem.Allocator, text: []const u8) ![]Benchmark {
    var result: std.ArrayList(Benchmark) = .empty;
    var lines = std.mem.splitScalar(u8, text, '\n');
    while (lines.next()) |line| {
        if (std.mem.trim(u8, line, " \t\r").len == 0) continue;
        const value = (try std.json.parseFromSlice(Benchmark, arena, line, .{ .ignore_unknown_fields = true })).value;
        if (value.case.len == 0) return error.EmptyBenchmarkName;
        for (result.items) |old| if (std.mem.eql(u8, old.case, value.case)) return error.DuplicateBenchmark;
        try result.append(arena, value);
    }
    if (result.items.len == 0) return error.NoBenchmarks;
    return result.toOwnedSlice(arena);
}

fn readBenchmarkRuns(io: std.Io, arena: std.mem.Allocator, value: Measurement) ![4][]Benchmark {
    const suffix = ".0.stdout";
    if (value.samples_ns.len != 4 or !std.mem.endsWith(u8, value.stdout, suffix)) return error.InvalidBenchmarkCaptures;
    const stem = value.stdout[0 .. value.stdout.len - suffix.len];
    var runs: [4][]Benchmark = undefined;
    for (&runs, 0..) |*run, index| {
        const path = try std.fmt.allocPrint(arena, "{s}.{d}.stdout", .{ stem, index });
        const bytes = try Dir.cwd().readFileAlloc(io, path, arena, .limited(64 * 1024 * 1024));
        run.* = try readBenchmarks(arena, bytes);
    }
    return runs;
}

fn compareBenchmarks(before: [4][]Benchmark, after: [4][]Benchmark, max_ratio: ?f64, report: *std.Io.Writer) !bool {
    const runs = [2][4][]Benchmark{ before, after };
    for (runs) |side| for (side) |run| {
        if (run.len != before[0].len) return error.DifferentBenchmarks;
    };
    var regressed = false;
    for (before[0]) |first| {
        var matched: [2][4]Benchmark = undefined;
        for (runs, &matched) |side, *samples| for (side, samples) |run, *sample| {
            sample.* = for (run) |value| {
                if (std.mem.eql(u8, first.case, value.case)) break value;
            } else return error.DifferentBenchmarks;
            if (sample.bytes != first.bytes) return error.DifferentBenchmarkInputs;
        };
        var medians = [2]Benchmark{ first, first };
        for (matched, &medians) |samples, *median| {
            inline for (.{ "cold_ns", "median_ns", "allocations", "allocated_bytes" }) |field| {
                var warm: [3]u64 = undefined;
                for (samples[1..], &warm) |sample, *value| value.* = @field(sample, field);
                std.mem.sort(u64, &warm, {}, std.sort.asc(u64));
                @field(median, field) = warm[1];
            }
        }
        const old = medians[0];
        const current = medians[1];
        const first_cold = @as(f64, @floatFromInt(matched[1][0].cold_ns)) / @as(f64, @floatFromInt(@max(1, matched[0][0].cold_ns)));
        const cold = @as(f64, @floatFromInt(current.cold_ns)) / @as(f64, @floatFromInt(@max(1, old.cold_ns)));
        const median = @as(f64, @floatFromInt(current.median_ns)) / @as(f64, @floatFromInt(@max(1, old.median_ns)));
        const case_regression = if (max_ratio) |limit| cold > limit or median > limit else false;
        regressed = regressed or case_regression;
        try report.print("{s}: first cold {d:.3}x; cold {d:.3}x; median {d:.3}x; allocations {d}->{d}; bytes {d}->{d}{s}\n", .{
            old.case,
            first_cold,
            cold,
            median,
            old.allocations,
            current.allocations,
            old.allocated_bytes,
            current.allocated_bytes,
            if (case_regression) " REGRESSION" else "",
        });
    }
    return regressed;
}

fn usage(io: std.Io) !void {
    try std.Io.File.stderr().writeStreamingAll(io, "history run [--allow-immutable] REVSET OUT -- COMMAND...\n" ++
        "history record OUT -- COMMAND...\n" ++
        "history compare BEFORE.json AFTER.json [MAX_RATIO] [--snapshots|--benchmarks]\n" ++
        "run uses isolated jj workspaces without integrating history changes.\n" ++
        "Commands still have normal filesystem and network access.\n");
    return error.InvalidArguments;
}

test "history output collisions preserve completed and partial recordings before command dispatch" {
    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena.deinit();
    const gpa = arena.allocator();
    const io = std.testing.io;
    for ([_][]const u8{ ".reserved", ".json", ".0.stdout", ".0.stderr", ".1.stdout", ".1.stderr", ".2.stdout", ".2.stderr", ".3.stdout", ".3.stderr" }) |suffix| {
        var tmp = std.testing.tmpDir(.{});
        defer tmp.cleanup();
        const output = try tmp.dir.realPathFileAlloc(io, ".", gpa);
        const stem = try std.fmt.allocPrint(gpa, "{s}/measurement", .{output});
        const path = try std.fmt.allocPrint(gpa, "{s}{s}", .{ stem, suffix });
        try writeNew(io, path, "previous recording\n");
        const missing = try std.fmt.allocPrint(gpa, "{s}/must-not-be-dispatched", .{output});
        try std.testing.expectError(error.MeasurementExists, record(io, gpa, stem, "fixture", &.{missing}));
        const preserved = try Dir.cwd().readFileAlloc(io, path, gpa, .limited(1024));
        try std.testing.expectEqualStrings("previous recording\n", preserved);
        try std.testing.expectError(error.MeasurementExists, record(io, gpa, stem, "fixture", &.{missing}));
    }
}

const measurement_fixture: Measurement = .{
    .schema = schema_version,
    .harness = harness_id,
    .zig_version = builtin.zig_version_string,
    .revision = "fixture",
    .cwd = "",
    .command = &.{ "fixture", "--argument" },
    .first_ns = 100,
    .median_ns = 100,
    .samples_ns = &.{ 100, 100, 100, 100 },
    .stdout = "",
    .stderr = "",
    .stable_stdout = true,
    .stable_stderr = true,
    .passed = true,
};

test "history rejects incomparable and failed measurements before reading captures" {
    var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
    defer memory.deinit();
    const arena = memory.allocator();
    var report: std.Io.Writer.Allocating = .init(arena);
    defer report.deinit();
    var values = [2]Measurement{ measurement_fixture, measurement_fixture };

    values[1].schema = 0;
    try std.testing.expectError(error.UnsupportedMeasurementVersion, compare(std.testing.io, arena, values, .{}, &report.writer));
    values[1] = measurement_fixture;
    values[0].schema = schema_version + 1;
    try std.testing.expectError(error.UnsupportedMeasurementVersion, compare(std.testing.io, arena, values, .{}, &report.writer));
    values[0] = measurement_fixture;
    values[1].harness = "other harness";
    try std.testing.expectError(error.DifferentHarness, compare(std.testing.io, arena, values, .{}, &report.writer));
    values[1] = measurement_fixture;
    values[1].zig_version = "other compiler";
    try std.testing.expectError(error.DifferentHarness, compare(std.testing.io, arena, values, .{}, &report.writer));
    values[1] = measurement_fixture;
    values[1].command = &.{"fixture --argument"};
    try std.testing.expectError(error.DifferentCommands, compare(std.testing.io, arena, values, .{}, &report.writer));
    values[1] = measurement_fixture;
    values[1].passed = false;
    try std.testing.expectError(error.FailedMeasurement, compare(std.testing.io, arena, values, .{}, &report.writer));
    values[1] = measurement_fixture;
    values[0].passed = false;
    try std.testing.expectError(error.FailedMeasurement, compare(std.testing.io, arena, values, .{}, &report.writer));
    try std.testing.expectEqualStrings("", report.written());
}

const benchmark_fixture =
    \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
    \\{"case":"last","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
;

test "history capture and performance gates reject false passes" {
    var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
    defer memory.deinit();
    const arena = memory.allocator();
    const io = std.testing.io;
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    for (0..4) |run| {
        const path = try std.fmt.allocPrint(arena, "before.{d}.stdout", .{run});
        try tmp.dir.writeFile(io, .{ .sub_path = path, .data = benchmark_fixture });
    }
    try tmp.dir.writeFile(io, .{ .sub_path = "stderr", .data = "diagnostic\n" });
    try tmp.dir.writeFile(io, .{ .sub_path = "changed.0.stdout", .data = "changed\n" });
    var original = measurement_fixture;
    original.stdout = try tmp.dir.realPathFileAlloc(io, "before.0.stdout", arena);
    original.stderr = try tmp.dir.realPathFileAlloc(io, "stderr", arena);
    const changed = try tmp.dir.realPathFileAlloc(io, "changed.0.stdout", arena);
    var values = [2]Measurement{ original, original };
    var report: std.Io.Writer.Allocating = .init(arena);
    defer report.deinit();

    try compare(io, arena, values, .{ .snapshots = true }, &report.writer);
    values[1].stdout = changed;
    try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
    values[1] = original;
    values[1].stderr = changed;
    try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
    for (0..2) |index| {
        values = .{ original, original };
        values[index].stable_stdout = false;
        try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
        values[index].stable_stdout = true;
        values[index].stable_stderr = false;
        try std.testing.expectError(error.SnapshotMismatch, compare(io, arena, values, .{ .snapshots = true }, &report.writer));
        try compare(io, arena, values, .{}, &report.writer);
    }
    values = .{ original, original };
    values[1].median_ns = 126;
    try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .max_ratio = 1.25 }, &report.writer));
    values[1].median_ns = 125;
    try compare(io, arena, values, .{ .max_ratio = 1.25 }, &report.writer);

    const slower_cold =
        \\{"case":"last","bytes":20,"median_ns":50,"cold_ns":50,"allocations":5,"allocated_bytes":500}
        \\{"case":"first","bytes":10,"median_ns":50,"cold_ns":126,"allocations":5,"allocated_bytes":500}
    ;
    const slower_warm =
        \\{"case":"first","bytes":10,"median_ns":126,"cold_ns":50,"allocations":5,"allocated_bytes":500}
        \\{"case":"last","bytes":20,"median_ns":50,"cold_ns":50,"allocations":5,"allocated_bytes":500}
    ;
    values[1].median_ns = 50;
    values[1].stdout = changed;
    for ([_][]const u8{ slower_cold, slower_warm }) |text| {
        for (0..4) |run| {
            const path = try std.fmt.allocPrint(arena, "changed.{d}.stdout", .{run});
            try tmp.dir.writeFile(io, .{ .sub_path = path, .data = text });
        }
        const begin = report.written().len;
        try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer));
        const output = report.written()[begin..];
        try std.testing.expect(std.mem.indexOf(u8, output, "warm runtime 0.500x") != null);
        try std.testing.expect(std.mem.indexOf(u8, output, " REGRESSION\nlast:") != null);
        try compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.26 }, &report.writer);
        try compare(io, arena, values, .{ .benchmarks = true }, &report.writer);
    }
    values[1] = original;
    values[1].median_ns = 200;
    try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer));
}

test "history benchmark cases are nonempty unique and matched by name" {
    var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
    defer memory.deinit();
    const arena = memory.allocator();
    var report: std.Io.Writer.Allocating = .init(arena);
    defer report.deinit();
    try std.testing.expectError(error.NoBenchmarks, readBenchmarks(arena, " \n\t\n"));
    try std.testing.expectError(error.DuplicateBenchmark, readBenchmarks(arena, benchmark_fixture ++ "\n" ++ benchmark_fixture));
    try std.testing.expectError(error.MissingField, readBenchmarks(arena, "{\"case\":\"first\"}"));
    const before = try readBenchmarks(arena, benchmark_fixture);
    const missing = try readBenchmarks(arena, "{\"case\":\"first\",\"bytes\":10,\"median_ns\":100,\"cold_ns\":100,\"allocations\":10,\"allocated_bytes\":1000}");
    try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, .{missing} ** 4, null, &report.writer));
    const renamed =
        \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
        \\{"case":"renamed","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
    ;
    const different = try readBenchmarks(arena, renamed);
    try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, .{different} ** 4, null, &report.writer));
    const reordered =
        \\{"case":"last","bytes":20,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
        \\{"case":"first","bytes":10,"median_ns":100,"cold_ns":100,"allocations":10,"allocated_bytes":1000}
    ;
    const same = try readBenchmarks(arena, reordered);
    try std.testing.expect(!try compareBenchmarks(.{before} ** 4, .{same} ** 4, 1, &report.writer));
    var samples = [4][]Benchmark{ before, before, before, different };
    try std.testing.expectError(error.DifferentBenchmarks, compareBenchmarks(.{before} ** 4, samples, null, &report.writer));
    samples[3] = try arena.dupe(Benchmark, before);
    samples[3][0].bytes += 1;
    try std.testing.expectError(error.DifferentBenchmarkInputs, compareBenchmarks(.{before} ** 4, samples, null, &report.writer));
}

test "history benchmark gate uses repeated process captures rather than the first process alone" {
    var memory: std.heap.ArenaAllocator = .init(std.testing.allocator);
    defer memory.deinit();
    const arena = memory.allocator();
    const io = std.testing.io;
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    var values = [2]Measurement{ measurement_fixture, measurement_fixture };
    const root = try tmp.dir.realPathFileAlloc(io, ".", arena);
    for (&values, 0..) |*value, side| {
        value.stdout = try std.fmt.allocPrint(arena, "{s}/{d}.0.stdout", .{ root, side });
        value.stderr = try std.fmt.allocPrint(arena, "{s}/{d}.0.stderr", .{ root, side });
        try writeNew(io, value.stderr, "");
        for (0..4) |run| {
            const path = try std.fmt.allocPrint(arena, "{s}/{d}.{d}.stdout", .{ root, side, run });
            const duration: u64 = if (side == 1 and run != 0) 150 else 100;
            const line = try std.fmt.allocPrint(
                arena,
                "{{\"case\":\"one\",\"bytes\":10,\"median_ns\":{d},\"cold_ns\":{d},\"allocations\":1,\"allocated_bytes\":10}}\n",
                .{ duration, duration },
            );
            try writeNew(io, path, line);
        }
    }
    var report: std.Io.Writer.Allocating = .init(arena);
    defer report.deinit();
    try std.testing.expectError(error.PerformanceRegression, compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer));
    try std.testing.expect(std.mem.indexOf(u8, report.written(), "first cold 1.000x; cold 1.500x; median 1.500x") != null);
    for ([_]u64{ 1000, 1000, 100, 100 }, 0..) |duration, run| {
        const path = try std.fmt.allocPrint(arena, "1.{d}.stdout", .{run});
        const line = try std.fmt.allocPrint(
            arena,
            "{{\"case\":\"one\",\"bytes\":10,\"median_ns\":{d},\"cold_ns\":{d},\"allocations\":1,\"allocated_bytes\":10}}\n",
            .{ duration, duration },
        );
        try tmp.dir.writeFile(io, .{ .sub_path = path, .data = line });
    }
    const begin = report.written().len;
    try compare(io, arena, values, .{ .benchmarks = true, .max_ratio = 1.25 }, &report.writer);
    try std.testing.expect(std.mem.indexOf(u8, report.written()[begin..], "first cold 10.000x; cold 1.000x; median 1.000x") != null);
    try tmp.dir.deleteFile(io, "1.3.stdout");
    try std.testing.expectError(error.FileNotFound, compare(io, arena, values, .{ .benchmarks = true }, &report.writer));
    values[1].samples_ns = &.{ 100, 100, 100 };
    try std.testing.expectError(error.InvalidBenchmarkCaptures, compare(io, arena, values, .{ .benchmarks = true }, &report.writer));
    values[1].samples_ns = measurement_fixture.samples_ns;
    values[1].stdout = values[1].stderr;
    try std.testing.expectError(error.InvalidBenchmarkCaptures, compare(io, arena, values, .{ .benchmarks = true }, &report.writer));
}