From 297943b01e18bb3d1dd41a8d3e220811de25228b Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Tue, 10 Nov 2026 11:49:22 -0300 Subject: ztrace probe: BPF on syscalls, scheduling, kprobes, uprobes and USDT Programs built as instruction arrays for one process (no compiler, no libbpf), writing Events into a BPF ring buffer; a drop counter in a map; task_struct offsets from the kernel's BTF; a USDT probe's arguments read in the kernel at the probe. Without CAP_BPF and CAP_PERFMON it says so; --dry-run builds and lists everything. Tested offline by running the programs in a small BPF machine, and for real by test-kernel-vm, which boots this kernel in QEMU where we are root. ztrace perf: moments are no longer cut per event unless --cut; --max-size, and a full recording stops the target and says so. perf.Sorted merges two backends' events by time. Co-Authored-By: Claude Fable 5.2 --- src/cli/perf.zig | 60 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 12 deletions(-) (limited to 'src/cli/perf.zig') diff --git a/src/cli/perf.zig b/src/cli/perf.zig index 57ad11a..58f7d77 100644 --- a/src/cli/perf.zig +++ b/src/cli/perf.zig @@ -30,10 +30,15 @@ pub const usage = \\ size) with the value found there; with :rw every access \\ --break WHAT ... on code: every time it runs, with the registers \\ --threads also the threads and children it creates + \\ --cut end a moment at every fault and every hit of --watch and + \\ --break: the state at each can be gone back to, for more bytes + \\ --max-size BYTES stop when the recording is this large (default 1 GiB) \\ --ring BYTES the kernel's buffer, per CPU (a power of two, default 512 KiB) \\ \\There are four debug registers. Only user space is seen: a page the \\kernel touches first (a read() into fresh memory) faults unseen. + \\Moments end where the recorder ends them anyway (a syscall's exit, a + \\megabyte of events), so that a profile or a storm of faults stays small. \\ ; @@ -44,6 +49,8 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { var out: []const u8 = "out.ztrace"; var options: zt.perf.Options = .{ .counters = &.{ .instructions, .cycles } }; var interval_ms: u32 = 0; + var cut = false; + var max_bytes: ?u64 = null; var specs: std.ArrayList(Spec) = .empty; var i: usize = 0; while (i < args.len) : (i += 1) { @@ -62,6 +69,8 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { options.syscalls = true; } else if (std.mem.eql(u8, arg, "--threads")) { options.follow = true; + } else if (std.mem.eql(u8, arg, "--cut")) { + cut = true; } else if (value == null) { fail(usage, .{}); } else if (std.mem.eql(u8, arg, "-o")) { @@ -79,6 +88,9 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { } else if (std.mem.eql(u8, arg, "--sample")) { options.sample_hz = std.fmt.parseInt(u32, value.?, 0) catch fail("--sample needs a rate\n", .{}); i += 1; + } else if (std.mem.eql(u8, arg, "--max-size")) { + max_bytes = std.fmt.parseInt(u64, value.?, 0) catch fail("--max-size needs bytes\n", .{}); + i += 1; } else if (std.mem.eql(u8, arg, "--ring")) { options.ring_bytes = std.fmt.parseInt(usize, value.?, 0) catch 0; if (!std.math.isPowerOfTwo(options.ring_bytes) or options.ring_bytes < 4096) fail("--ring needs a power of two, 4096 or more\n", .{}); @@ -105,8 +117,9 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { var proc = try zt.ptrace.Process.spawn(path, argv, init.minimal.environ.block.slice.ptr); errdefer proc.kill(); var rec = try zt.Recorder.create(init.gpa, init.io, out); + if (max_bytes) |bytes| rec.max_bytes = bytes; // What changes the state a moment holds (registers, memory) ends one. - rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit(); + if (cut) rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit(); var session = zt.perf.Session.open(init.gpa, proc.pid, options, rec.sink()) catch |err| switch (err) { error.PerfAccess => fail("perf_event_open was refused: kernel.perf_event_paranoid is above 2, or a security module forbids it\n", .{}), error.PerfNoSuchEvent => fail("perf_event_open: this CPU or kernel lacks one of the events asked for (in a VM without a PMU, try -e task-clock; --sample takes at most kernel.perf_event_max_sample_rate)\n", .{}), @@ -133,7 +146,19 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { zt.perf.release(proc); const start = zt.event.now(); - var next_read = start; + const exit_code: ?u64 = pump(&session, &rec, interval_ms) catch |err| switch (err) { + error.RecordingFull => null, + else => |e| return e, + }; + if (exit_code == null) proc.kill(); + const ns = zt.event.now() - start; + try finish(init, &rec, out, exit_code, session.lost, ns, options.counters, session.counts); +} + +/// Decodes until the target ends; its exit code. `error.RecordingFull` +/// when the recording reaches its limit first. +fn pump(session: *zt.perf.Session, rec: *zt.Recorder, interval_ms: u32) !u64 { + var next_read = zt.event.now(); while (true) { var wait_ms: i32 = 100; if (interval_ms != 0) { @@ -147,22 +172,33 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { if (!try session.poll(wait_ms)) break; } try session.readCounters(); + return exited(session.pid, rec.sink()); +} + +/// Waits for the target's end and records it. The exit code. +pub fn exited(pid: linux.pid_t, sink: zt.Sink) !u64 { var status: u32 = 0; - _ = linux.waitpid(proc.pid, @ptrCast(&status), 0); - const exit_code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status))); - const ns = zt.event.now() - start; - try rec.emit(.{ .ts = zt.event.now(), .tid = @intCast(proc.pid), .kind = .exit, .a = .{ exit_code, 0, 0, 0, 0, 0 } }, ""); + _ = linux.waitpid(pid, @ptrCast(&status), 0); + const code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status))); + try sink.emit(.{ .ts = zt.event.now(), .tid = @intCast(pid), .kind = .exit, .a = .{ code, 0, 0, 0, 0, 0 } }, ""); + return code; +} +/// Closes the recording and says what it holds; `exit_code` is null when +/// it filled up and the target was stopped for it. +pub fn finish(init: std.process.Init, rec: *zt.Recorder, out: []const u8, exit_code: ?u64, lost: u64, ns: u64, counters: []const zt.perf.Counter, counts: []const u64) !void { const events = rec.count; const moments = rec.moments + @intFromBool(rec.delta.items.len > 0); try rec.close(); const size = (try std.Io.Dir.cwd().statFile(init.io, out, .{})).size; - for (options.counters, session.counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c }); - std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes, {d:.1} ms, {d:.0} events/s, exit {d}\n", .{ - out, moments, - events, session.lost, - size, @as(f64, @floatFromInt(ns)) / 1e6, - @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9), exit_code, + const code = exit_code orelse fail("{s} is full at {d} bytes ({d} events) and the program was stopped: record fewer kinds of event, or allow more with --max-size BYTES\n", .{ out, size, events }); + for (counters, counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c }); + std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes ({d:.1} an event), {d:.1} ms, {d:.0} events/s, exit {d}\n", .{ + out, moments, + events, lost, + size, @as(f64, @floatFromInt(size)) / @as(f64, @floatFromInt(@max(events, 1))), + @as(f64, @floatFromInt(ns)) / 1e6, @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9), + code, }); } -- cgit v1.3