diff options
Diffstat (limited to 'test/unit_profile_test.zig')
| -rw-r--r-- | test/unit_profile_test.zig | 236 |
1 files changed, 236 insertions, 0 deletions
diff --git a/test/unit_profile_test.zig b/test/unit_profile_test.zig new file mode 100644 index 00000000..721f1212 --- /dev/null +++ b/test/unit_profile_test.zig @@ -0,0 +1,236 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const expect = std.testing.expect; + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const gpa = init.arena.allocator(); + const args = try init.minimal.args.toSlice(gpa); + if (args.len == 2 and std.mem.eql(u8, args[1], "--listen=-")) return fixture(init); + if (args.len != 2) return error.ExpectedProfilerPath; + const self = try std.process.executablePathAlloc(io, gpa); + const cases = .{ + .{ "pass", "", "", 2, 0, 0 }, + .{ "delayed-start", "", "", 2, 0, 0 }, + .{ "skip", "", "", 1, 0, 1 }, + .{ "fail", "", "TestFailed", 1, 1, 0 }, + .{ "log", "", "TestFailed", 1, 1, 0 }, + .{ "leak", "", "TestFailed", 1, 1, 0 }, + .{ "pass", "fixture.two", "", 1, 0, 0 }, + .{ "pass", "absent", "NoMatchingTests", 0, 0, 0 }, + .{ "version", "", "DifferentZigVersion", 0, 0, 0 }, + .{ "metadata", "", "InvalidMetadata", 0, 0, 0 }, + .{ "index", "", "UnexpectedTestIndex", 0, 0, 0 }, + .{ "status", "", "InvalidTestStatus", 0, 0, 0 }, + .{ "oversized", "", "MessageTooLarge", 0, 0, 0 }, + .{ "truncated", "", "EndOfStream", 0, 0, 0 }, + .{ "hang-version", "", "Timeout", 0, 0, 0 }, + .{ "hang-metadata", "", "Timeout", 0, 0, 0 }, + .{ "hang-start", "", "Timeout", 0, 0, 0 }, + .{ "hang-result", "", "Timeout", 0, 0, 0 }, + .{ "hang-body", "", "Timeout", 0, 0, 0 }, + .{ "split-deadline", "", "Timeout", 0, 0, 0 }, + .{ "hang-exit", "", "Timeout", 0, 0, 0 }, + .{ "hang-exit-closed", "", "Timeout", 0, 0, 0 }, + .{ "exit", "", "TestRunnerFailed", 2, 0, 0 }, + }; + inline for (cases) |case| { + try init.environ_map.put("PARDES_PROFILE_FIXTURE", case[0]); + const result = try std.process.run(gpa, io, .{ + .argv = &.{ args[1], self, case[1], if (std.mem.eql(u8, case[2], "Timeout")) "--timeout-ms=150" else "--timeout-ms=1500" }, + .environ_map = init.environ_map, + .stdout_limit = .limited(1024 * 1024), + .stderr_limit = .limited(1024 * 1024), + .timeout = .{ .duration = .{ .clock = .awake, .raw = .fromSeconds(5) } }, + }); + errdefer std.debug.print("profile fixture {s}: {s}\n{s}\n", .{ case[0], result.stdout, result.stderr }); + const success = result.term == .exited and result.term.exited == 0; + try expect(success == (case[2].len == 0)); + if (case[2].len != 0) try expect(std.mem.indexOf(u8, result.stderr, "error: " ++ case[2]) != null); + if (std.mem.eql(u8, case[2], "Timeout")) { + const phase = if (std.mem.eql(u8, case[0], "hang-version")) + "fixture reached startup" + else if (std.mem.eql(u8, case[0], "hang-metadata")) + "fixture reached metadata" + else if (std.mem.startsWith(u8, case[0], "hang-exit")) + "fixture reached exit" + else + "running fixture.one"; + try expect(std.mem.indexOf(u8, result.stderr, phase) != null); + } + var lines = std.mem.tokenizeScalar(u8, result.stdout, '\n'); + var sum: i128 = 0; + var count: usize = 0; + var summary = false; + while (lines.next()) |line| { + const parsed = try std.json.parseFromSlice(std.json.Value, gpa, line, .{}); + const object = parsed.value.object; + if (std.mem.eql(u8, object.get("kind").?.string, "case")) { + const elapsed = object.get("request_to_result_ns").?.integer; + try expect(elapsed > 0); + if (std.mem.eql(u8, case[0], "delayed-start")) try expect(elapsed >= 60 * std.time.ns_per_ms); + sum += elapsed; + count += 1; + } else { + try expect(!summary); + summary = true; + try expect(object.get("passed").?.integer == case[3]); + try expect(object.get("failed").?.integer == case[4]); + try expect(object.get("skipped").?.integer == case[5]); + try expect(object.get("request_to_result_ns").?.integer == sum); + try expect(object.get("process_wall_ns").?.integer >= sum); + } + } + if (std.mem.startsWith(u8, case[0], "hang-exit")) { + try expect(count == 2 and !summary); + } else if (case[3] + case[4] + case[5] != 0 or std.mem.eql(u8, case[2], "NoMatchingTests")) { + try expect(summary); + try expect(count == case[3] + case[4] + case[5]); + } else try expect(!summary); + } + var random: [12]u8 = undefined; + io.random(&random); + const path = try std.fmt.allocPrint(gpa, "profile-{x}", .{random}); + var parent = try std.Io.Dir.cwd().createDirPathOpen(io, ".zig-cache/tmp", .{}); + defer parent.close(io); + try parent.createDir(io, path, .default_dir); + defer parent.deleteTree(io, path) catch {}; + var dir = try parent.openDir(io, path, .{}); + defer dir.close(io); + try dir.symLink(io, self, "empty", .{}); + try dir.symLink(io, self, "zero", .{}); + const absolute = try dir.realPathFileAlloc(io, ".", gpa); + const empty = try std.fmt.allocPrint(gpa, "{s}/empty", .{absolute}); + const zero = try std.fmt.allocPrint(gpa, "{s}/zero", .{absolute}); + try init.environ_map.put("PARDES_PROFILE_FIXTURE", "check"); + const checks = [_]struct { filter: []const u8, binaries: []const []const u8, passes: bool }{ + .{ .filter = "fixture.one", .binaries = &.{self}, .passes = true }, + .{ .filter = "fixture.one", .binaries = &.{ empty, self }, .passes = true }, + .{ .filter = "fixture.one", .binaries = &.{ zero, self }, .passes = true }, + .{ .filter = "named.test_0", .binaries = &.{self}, .passes = true }, + .{ .filter = "absent", .binaries = &.{self}, .passes = false }, + .{ .filter = "", .binaries = &.{empty}, .passes = false }, + .{ .filter = "", .binaries = &.{zero}, .passes = false }, + .{ .filter = "test_", .binaries = &.{empty}, .passes = false }, + }; + for (checks) |case| { + var command: std.ArrayList([]const u8) = .empty; + try command.appendSlice(gpa, &.{ args[1], "--check-filter", case.filter }); + try command.appendSlice(gpa, case.binaries); + const result = try std.process.run(gpa, io, .{ + .argv = command.items, + .environ_map = init.environ_map, + .timeout = .{ .duration = .{ .clock = .awake, .raw = .fromSeconds(5) } }, + }); + errdefer std.debug.print("filter fixture '{s}': {s}\n{s}\n", .{ case.filter, result.stdout, result.stderr }); + try expect((result.term == .exited and result.term.exited == 0) == case.passes); + try expect(result.stdout.len == 0); + if (!case.passes) try expect(std.mem.indexOf(u8, result.stderr, "error: NoMatchingTests") != null); + } + std.debug.print("unit-profile: {d} protocol, timing and filter fixtures passed\n", .{cases.len + checks.len}); +} + +fn stall(io: std.Io) !void { + try io.sleep(.fromSeconds(2), .awake); + return error.FixtureWasNotTerminated; +} + +fn fixture(init: std.process.Init) !void { + const io = init.io; + const mode = init.environ_map.get("PARDES_PROFILE_FIXTURE") orelse return error.ExpectedFixtureMode; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + const empty = std.mem.eql(u8, std.fs.path.basename(args[0]), "empty"); + const zero = std.mem.eql(u8, std.fs.path.basename(args[0]), "zero"); + if (std.mem.eql(u8, mode, "hang-version")) { + std.debug.print("fixture reached startup\n", .{}); + return stall(io); + } + var read_buffer: [1024]u8 = undefined; + var write_buffer: [1024]u8 = undefined; + var reader = std.Io.File.stdin().readerStreaming(io, &read_buffer); + var writer = std.Io.File.stdout().writerStreaming(io, &write_buffer); + var server = try std.zig.Server.init(.{ + .in = &reader.interface, + .out = &writer.interface, + .zig_version = if (std.mem.eql(u8, mode, "version")) "wrong-version" else builtin.zig_version_string, + }); + while (true) { + const request = try server.receiveMessage(); + switch (request.tag) { + .query_test_metadata => { + if (std.mem.eql(u8, mode, "hang-metadata")) { + std.debug.print("fixture reached metadata\n", .{}); + return stall(io); + } + if (std.mem.eql(u8, mode, "metadata")) { + try server.serveStringMessage(.test_metadata, "bad"); + continue; + } + try server.serveTestMetadata(.{ + .names = if (zero) &.{} else if (empty) &.{ 1, 16 } else &.{ 1, 13 }, + .expected_panic_msgs = if (zero) &.{} else &.{ 0, 0 }, + .string_bytes = if (zero) + "\x00" + else if (empty) + "\x00fixture.test_0\x00fixture.test_11\x00" + else if (std.mem.eql(u8, mode, "check")) + "\x00fixture.one\x00fixture.test.named.test_0\x00" + else + "\x00fixture.one\x00fixture.two\x00", + }); + }, + .run_test => { + if (std.mem.eql(u8, mode, "check")) return error.TestWasRunDuringMatchValidation; + const index = try server.receiveBody_u32(); + if (std.mem.eql(u8, mode, "hang-start")) return stall(io); + if (std.mem.eql(u8, mode, "delayed-start")) try io.sleep(.fromMilliseconds(70), .awake); + if (std.mem.eql(u8, mode, "split-deadline")) try io.sleep(.fromMilliseconds(100), .awake); + try server.serveStringMessage(.test_started, ""); + if (std.mem.eql(u8, mode, "hang-result")) return stall(io); + if (std.mem.eql(u8, mode, "split-deadline")) try io.sleep(.fromMilliseconds(100), .awake); + if (std.mem.eql(u8, mode, "oversized")) { + try server.serveMessageHeader(.{ .tag = .test_results, .bytes_len = 16 * 1024 * 1024 + 1 }); + try writer.interface.flush(); + return stall(io); + } + if (std.mem.eql(u8, mode, "truncated") or std.mem.eql(u8, mode, "hang-body")) { + try server.serveMessageHeader(.{ .tag = .test_results, .bytes_len = 12 }); + try writer.interface.writeAll("oops"); + try writer.interface.flush(); + if (std.mem.eql(u8, mode, "hang-body")) return stall(io); + return; + } + if (std.mem.eql(u8, mode, "status")) { + var bytes: [12]u8 = @splat(0); + bytes[4] = 3; + try server.serveStringMessage(.test_results, &bytes); + continue; + } + var flags: std.zig.Server.Message.TestResults.Flags = .{ + .status = .pass, + .fuzz = false, + .log_err_count = 0, + .leak_count = 0, + }; + if (index == 0) { + if (std.mem.eql(u8, mode, "fail")) flags.status = .fail; + if (std.mem.eql(u8, mode, "skip")) flags.status = .skip; + if (std.mem.eql(u8, mode, "log")) flags.log_err_count = 1; + if (std.mem.eql(u8, mode, "leak")) flags.leak_count = 1; + } + try server.serveTestResults(.{ .index = if (std.mem.eql(u8, mode, "index")) index + 1 else index, .flags = flags }); + }, + .exit => { + if (std.mem.startsWith(u8, mode, "hang-exit")) std.debug.print("fixture reached exit\n", .{}); + if (std.mem.eql(u8, mode, "hang-exit-closed")) { + std.Io.File.stdout().close(io); + return stall(io); + } + if (std.mem.eql(u8, mode, "hang-exit")) return stall(io); + if (std.mem.eql(u8, mode, "exit")) std.process.exit(7); + return; + }, + else => return error.UnexpectedRequest, + } + } +} |