summaryrefslogtreecommitdiff
path: root/src/trace/bpf.zig
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-11-10 11:49:22 -0300
committerGabriel Schneider <[email protected]>2026-11-10 12:19:06 -0300
commit297943b01e18bb3d1dd41a8d3e220811de25228b (patch)
tree4938e643abada811d3ab28b14c0e31dc94ebda62 /src/trace/bpf.zig
parentdb923fa823354270217362db06356f2b1a41315e (diff)
downloadcomputeranim-297943b01e18bb3d1dd41a8d3e220811de25228b.tar.gz
computeranim-297943b01e18bb3d1dd41a8d3e220811de25228b.zip
ztrace probe: BPF on syscalls, scheduling, kprobes, uprobes and USDT
Programs built as instruction arrays for one process (no compiler, no libbpf), writing Events into a BPF ring buffer; a drop counter in a map; task_struct offsets from the kernel's BTF; a USDT probe's arguments read in the kernel at the probe. Without CAP_BPF and CAP_PERFMON it says so; --dry-run builds and lists everything. Tested offline by running the programs in a small BPF machine, and for real by test-kernel-vm, which boots this kernel in QEMU where we are root. ztrace perf: moments are no longer cut per event unless --cut; --max-size, and a full recording stops the target and says so. perf.Sorted merges two backends' events by time. Co-Authored-By: Claude Fable 5.2 <[email protected]>
Diffstat (limited to 'src/trace/bpf.zig')
-rw-r--r--src/trace/bpf.zig733
1 files changed, 733 insertions, 0 deletions
diff --git a/src/trace/bpf.zig b/src/trace/bpf.zig
new file mode 100644
index 0000000..4069b01
--- /dev/null
+++ b/src/trace/bpf.zig
@@ -0,0 +1,733 @@
+//! The kernel's view with privilege: BPF programs on the raw tracepoints
+//! sys_enter, sys_exit and sched_switch, and on any kprobe, uprobe or USDT
+//! probe. Needs CAP_BPF and CAP_PERFMON (root). Each program writes an
+//! `Event`, and for a syscall's entry and a probe the registers after it,
+//! into one BPF ring buffer; `Tracer.poll` moves them into a `Sink`.
+//!
+//! The programs are instruction arrays built here, for one process (its id
+//! is a constant in them): no compiler, no object file, no libbpf. What the
+//! kernel does not tell (where `pid` is in `task_struct`) comes from its BTF.
+const std = @import("std");
+const linux = std.os.linux;
+const BPF = linux.BPF;
+const Insn = BPF.Insn;
+const event = @import("event.zig");
+const usdt = @import("usdt.zig");
+const Event = event.Event;
+const Regs = event.Regs;
+const Kind = event.Kind;
+
+/// The kernel's `struct pt_regs` is `Regs` up to here.
+const pt_regs_bytes = @offsetOf(Regs, "fs_base");
+/// On a program's stack, from the frame pointer down: the key of the drop
+/// counter, then the record. An event alone, or an event and registers.
+const key_at = -8;
+const small_at = key_at - @sizeOf(Event);
+const large_at = small_at - pt_regs_bytes;
+
+fn field(comptime name: []const u8) i16 {
+ return @offsetOf(Event, name);
+}
+
+fn arg(i: usize) i16 {
+ return @intCast(@offsetOf(Event, "a") + 8 * i);
+}
+
+fn reg(comptime name: []const u8) i16 {
+ return @offsetOf(Regs, name);
+}
+
+pub const Program = struct {
+ insns: [192]Insn = undefined,
+ len: usize = 0,
+ /// The jump that leaves early, to be pointed at the end.
+ leave: ?usize = null,
+
+ pub fn code(p: *const Program) []const Insn {
+ return p.insns[0..p.len];
+ }
+
+ fn add(p: *Program, insn: Insn) void {
+ p.insns[p.len] = insn;
+ p.len += 1;
+ }
+
+ /// r6 = the context, r7 = thread and process id; goes on only in a
+ /// thread of process `tgid`.
+ fn enter(p: *Program, tgid: u32) void {
+ p.add(Insn.mov(.r6, .r1));
+ p.add(Insn.call(.get_current_pid_tgid));
+ p.add(Insn.mov(.r7, .r0));
+ p.add(Insn.rsh(.r0, 32));
+ p.leave = p.len;
+ p.add(Insn.jne(.r0, @as(i32, @intCast(tgid)), 0));
+ }
+
+ /// The time, the thread and the kind of the event at `at`.
+ fn stamp(p: *Program, at: i16, kind: Kind) void {
+ p.add(Insn.call(.ktime_get_ns));
+ p.add(Insn.stx(.double_word, .r10, at + field("ts"), .r0));
+ p.add(Insn.stx(.word, .r10, at + field("tid"), .r7));
+ p.add(Insn.st(.half_word, .r10, at + field("kind"), @backingInt(kind)));
+ }
+
+ /// bpf_probe_read_kernel(frame + `to`, `size`, r3)
+ fn read(p: *Program, to: i16, size: i32) void {
+ p.add(Insn.mov(.r1, .r10));
+ p.add(Insn.add(.r1, to));
+ p.add(Insn.mov(.r2, size));
+ p.add(Insn.call(.probe_read_kernel));
+ }
+
+ /// The low `bytes` of a register, extended to all of it.
+ fn extend(p: *Program, r: Insn.Reg, bytes: u8, signed: bool) void {
+ if (bytes >= 8) return;
+ const shift: i32 = 64 - 8 * @as(i32, bytes);
+ p.add(Insn.lsh(r, shift));
+ p.add(if (signed) Insn.arsh(r, shift) else Insn.rsh(r, shift));
+ }
+
+ /// A register of the probed thread, from the context in r6.
+ fn probed(p: *Program, to: Insn.Reg, r: anytype) void {
+ p.add(Insn.ldx(.double_word, to, .r6, @as(i16, r.slot) * 8));
+ p.extend(to, r.bytes, false);
+ }
+
+ /// One argument of a USDT probe into the u64 at `to`: what
+ /// `usdt.Arg.fetch` computes, in the kernel, at the probe. Memory that
+ /// cannot be read gives 0.
+ fn usdtArg(p: *Program, a: usdt.Arg, to: i16) void {
+ switch (a.at) {
+ .constant => |c| {
+ p.add(Insn.ld_dw1(.r1, @bitCast(c)));
+ p.add(Insn.ld_dw2(@bitCast(c)));
+ },
+ .register => |r| p.probed(.r1, r),
+ .memory => |m| {
+ p.add(Insn.st(.double_word, .r10, to, 0));
+ p.probed(.r3, m.base);
+ if (m.index) |index| {
+ p.probed(.r1, index);
+ p.add(Insn.lsh(.r1, @as(i32, @ctz(m.scale))));
+ p.add(Insn.add(.r3, .r1));
+ }
+ p.add(Insn.ld_dw1(.r1, @bitCast(m.disp)));
+ p.add(Insn.ld_dw2(@bitCast(m.disp)));
+ p.add(Insn.add(.r3, .r1));
+ p.add(Insn.mov(.r1, .r10));
+ p.add(Insn.add(.r1, to));
+ p.add(Insn.mov(.r2, @as(i32, a.size)));
+ p.add(Insn.call(.probe_read_user));
+ p.add(Insn.ldx(.double_word, .r1, .r10, to));
+ },
+ }
+ p.extend(.r1, a.size, a.signed);
+ p.add(Insn.stx(.double_word, .r10, to, .r1));
+ }
+
+ /// Sends the `size` bytes at `at`, counts the record in `drops` when the
+ /// ring is full, and ends the program.
+ fn send(p: *Program, maps: Maps, at: i16, size: i32) void {
+ p.add(Insn.ld_map_fd1(.r1, maps.ring));
+ p.add(Insn.ld_map_fd2(maps.ring));
+ p.add(Insn.mov(.r2, .r10));
+ p.add(Insn.add(.r2, at));
+ p.add(Insn.mov(.r3, size));
+ p.add(Insn.mov(.r4, 0));
+ p.add(Insn.call(.ringbuf_output));
+ p.add(Insn.jeq(.r0, 0, 9));
+ p.add(Insn.st(.word, .r10, key_at, 0));
+ p.add(Insn.ld_map_fd1(.r1, maps.drops));
+ p.add(Insn.ld_map_fd2(maps.drops));
+ p.add(Insn.mov(.r2, .r10));
+ p.add(Insn.add(.r2, key_at));
+ p.add(Insn.call(.map_lookup_elem));
+ p.add(Insn.jeq(.r0, 0, 2));
+ p.add(Insn.mov(.r1, 1));
+ p.add(Insn.xadd(.r0, .r1));
+ if (p.leave) |at_jump| p.insns[at_jump].off = @intCast(p.len - at_jump - 1);
+ p.add(Insn.mov(.r0, 0));
+ p.add(Insn.exit());
+ }
+};
+
+/// The maps every program writes to: the ring of records, and an array of
+/// one u64 counting the records that found it full.
+pub const Maps = struct { ring: linux.fd_t, drops: linux.fd_t };
+
+/// raw tracepoint sys_enter(struct pt_regs *regs, long id): a
+/// `syscall_enter` and the registers.
+pub fn sysEnter(maps: Maps, tgid: u32) Program {
+ var p: Program = .{};
+ p.enter(tgid);
+ p.add(Insn.ldx(.double_word, .r3, .r6, 0));
+ p.read(large_at + @sizeOf(Event), pt_regs_bytes);
+ p.stamp(large_at, .syscall_enter);
+ p.add(Insn.ldx(.double_word, .r1, .r6, 8));
+ p.add(Insn.stx(.half_word, .r10, large_at + field("aux"), .r1));
+ inline for (.{ "rdi", "rsi", "rdx", "r10", "r8", "r9" }, 0..) |name, i| {
+ p.add(Insn.ldx(.double_word, .r1, .r10, large_at + @sizeOf(Event) + reg(name)));
+ p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1));
+ }
+ p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes);
+ return p;
+}
+
+/// raw tracepoint sys_exit(struct pt_regs *regs, long ret): a `syscall_exit`.
+pub fn sysExit(maps: Maps, tgid: u32) Program {
+ var p: Program = .{};
+ const number_at = small_at - 8;
+ p.enter(tgid);
+ p.add(Insn.ldx(.double_word, .r3, .r6, 0));
+ p.add(Insn.add(.r3, reg("orig_rax")));
+ p.read(number_at, 8);
+ p.stamp(small_at, .syscall_exit);
+ p.add(Insn.ldx(.double_word, .r1, .r10, number_at));
+ p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
+ p.add(Insn.ldx(.double_word, .r1, .r6, 8));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r1));
+ for (1..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0));
+ p.send(maps, small_at, @sizeOf(Event));
+ return p;
+}
+
+/// raw tracepoint sched_switch(bool preempt, struct task_struct *prev,
+/// *next): a `sched` when either is a thread of the process. `pid_at` and
+/// `tgid_at` are where those fields are in `task_struct`.
+pub fn schedSwitch(maps: Maps, tgid: u32, pid_at: u32, tgid_at: u32) Program {
+ var p: Program = .{};
+ // prev's pid and tgid, next's pid and tgid, as u32 below the event.
+ const ids_at = small_at - 16;
+ p.add(Insn.mov(.r6, .r1));
+ for ([_]i16{ 8, 16 }, 0..) |task, t| for ([_]u32{ pid_at, tgid_at }, 0..) |at, i| {
+ p.add(Insn.ldx(.double_word, .r3, .r6, task));
+ p.add(Insn.add(.r3, @as(i32, @intCast(at))));
+ p.read(ids_at + @as(i16, @intCast(8 * t + 4 * i)), 4);
+ };
+ p.add(Insn.ldx(.word, .r1, .r10, ids_at + 4));
+ p.add(Insn.jeq(.r1, @as(i32, @intCast(tgid)), 2));
+ p.add(Insn.ldx(.word, .r1, .r10, ids_at + 12));
+ p.leave = p.len;
+ p.add(Insn.jne(.r1, @as(i32, @intCast(tgid)), 0));
+ p.add(Insn.ldx(.word, .r7, .r10, ids_at));
+ p.stamp(small_at, .sched);
+ p.add(Insn.ldx(.double_word, .r1, .r6, 0));
+ p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r7));
+ p.add(Insn.ldx(.word, .r1, .r10, ids_at + 8));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(1), .r1));
+ p.add(Insn.call(.get_smp_processor_id));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(2), .r0));
+ for (3..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0));
+ p.send(maps, small_at, @sizeOf(Event));
+ return p;
+}
+
+/// A kprobe or uprobe (the context is the registers there): a `probe`
+/// named `id`, and the registers. Its a1..a5 are the first five arguments
+/// of a function call, or with `args` those of a USDT probe, read there
+/// and then.
+pub fn probe(maps: Maps, tgid: u32, id: u64, args: ?[]const usdt.Arg) Program {
+ var p: Program = .{};
+ p.enter(tgid);
+ p.stamp(large_at, .probe);
+ p.add(Insn.st(.half_word, .r10, large_at + field("aux"), 0));
+ p.add(Insn.ld_dw1(.r1, id));
+ p.add(Insn.ld_dw2(id));
+ p.add(Insn.stx(.double_word, .r10, large_at + arg(0), .r1));
+ inline for (.{ "rdi", "rsi", "rdx", "rcx", "r8" }, 1..) |name, i| {
+ if (args == null) {
+ p.add(Insn.ldx(.double_word, .r1, .r6, reg(name)));
+ p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1));
+ } else if (i <= args.?.len) {
+ p.usdtArg(args.?[i - 1], large_at + arg(i));
+ } else p.add(Insn.st(.double_word, .r10, large_at + arg(i), 0));
+ }
+ for (0..pt_regs_bytes / 8) |i| {
+ p.add(Insn.ldx(.double_word, .r1, .r6, @intCast(8 * i)));
+ p.add(Insn.stx(.double_word, .r10, large_at + @sizeOf(Event) + @as(i16, @intCast(8 * i)), .r1));
+ }
+ p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes);
+ return p;
+}
+
+/// `probe` for a USDT probe with the arguments `specs` ("-4@%edi
+/// 8@-24(%rbp)"): the first five become a1..a5.
+pub fn usdtProgram(maps: Maps, tgid: u32, id: u64, specs: []const u8) !Program {
+ var args: [5]usdt.Arg = undefined;
+ var count: usize = 0;
+ var each = std.mem.tokenizeScalar(u8, specs, ' ');
+ while (each.next()) |spec| {
+ if (count == args.len) break;
+ args[count] = try .parse(spec);
+ count += 1;
+ }
+ return probe(maps, tgid, id, args[0..count]);
+}
+
+/// Where `member` is in the struct `name`, in bytes, from the kernel's
+/// type information (the contents of /sys/kernel/btf/vmlinux).
+pub fn fieldOffset(btf: []const u8, name: []const u8, member: []const u8) ?u32 {
+ const T = BPF.btf;
+ if (btf.len < @sizeOf(T.Header)) return null;
+ const header = std.mem.bytesToValue(T.Header, btf[0..@sizeOf(T.Header)]);
+ if (header.magic != T.magic or header.hdr_len + @as(u64, header.str_off) + header.str_len > btf.len) return null;
+ const types = btf[header.hdr_len + header.type_off ..][0..header.type_len];
+ const strings = btf[header.hdr_len + header.str_off ..][0..header.str_len];
+ var at: usize = 0;
+ while (at + @sizeOf(T.Type) <= types.len) {
+ const t = std.mem.bytesToValue(T.Type, types[at..][0..@sizeOf(T.Type)]);
+ at += @sizeOf(T.Type);
+ const count: usize = t.info.vlen;
+ // What follows the type, by kind.
+ const extra: usize = switch (t.info.kind) {
+ .int, .@"var", .decl_tag => 4,
+ .array => 12,
+ .@"struct", .@"union", .datasec, .enum64 => 12 * count,
+ .@"enum", .func_proto => 8 * count,
+ else => 0,
+ };
+ if (at + extra > types.len) return null;
+ defer at += extra;
+ if (t.info.kind != .@"struct" or !std.mem.eql(u8, std.mem.sliceTo(strings[@min(t.name_off, strings.len)..], 0), name)) continue;
+ for (0..count) |i| {
+ const m = std.mem.bytesToValue(T.Member, types[at + 12 * i ..][0..12]);
+ if (std.mem.eql(u8, std.mem.sliceTo(strings[@min(m.name_off, strings.len)..], 0), member)) return m.offset.bit / 8;
+ }
+ }
+ return null;
+}
+
+/// The reading end of a BPF ring buffer: a page with our position, a page
+/// with the kernel's, and the data, which the kernel maps twice in a row so
+/// that no record wraps. A record is a u32 length with two flag bits, four
+/// bytes more, the bytes, padded to 8.
+pub const RingBuf = struct {
+ consumer: *u64,
+ producer: *const u64,
+ data: []const u8,
+ /// Read so far; the kernel is told by `commit`, after the records were used.
+ at: u64 = 0,
+
+ const busy: u32 = 1 << 31;
+ const discarded: u32 = 1 << 30;
+
+ pub fn map(fd: linux.fd_t, size: usize) !RingBuf {
+ const ours = linux.mmap(null, 4096, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0);
+ if (linux.errno(ours) != .SUCCESS) return error.BpfRingMap;
+ const theirs = linux.mmap(null, 4096 + 2 * size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 4096);
+ if (linux.errno(theirs) != .SUCCESS) return error.BpfRingMap;
+ return .{ .consumer = @ptrFromInt(ours), .producer = @ptrFromInt(theirs), .data = @as([*]const u8, @ptrFromInt(theirs + 4096))[0..size] };
+ }
+
+ /// The next finished record, valid until `commit`.
+ pub fn next(r: *RingBuf) ?[]const u8 {
+ while (r.at < @atomicLoad(u64, r.producer, .acquire)) {
+ const start: usize = @intCast(r.at & (r.data.len - 1));
+ const header = @atomicLoad(u32, @as(*const u32, @ptrCast(@alignCast(r.data.ptr + start))), .acquire);
+ if (header & busy != 0) return null;
+ const len = header & ~(busy | discarded);
+ r.at += std.mem.alignForward(u64, len + 8, 8);
+ if (header & discarded == 0) return (r.data.ptr + start + 8)[0..len];
+ }
+ return null;
+ }
+
+ pub fn commit(r: *RingBuf) void {
+ @atomicStore(u64, r.consumer, r.at, .release);
+ }
+};
+
+/// The capability bits of this process (CapEff of /proc/self/status).
+fn capabilities() u64 {
+ var buf: [4096]u8 = undefined;
+ const fd = linux.open("/proc/self/status", .{}, 0);
+ if (linux.errno(fd) != .SUCCESS) return 0;
+ defer _ = linux.close(@intCast(fd));
+ const n = linux.read(@intCast(fd), &buf, buf.len);
+ if (linux.errno(n) != .SUCCESS) return 0;
+ const label = "CapEff:\t";
+ const at = std.mem.indexOf(u8, buf[0..n], label) orelse return 0;
+ return std.fmt.parseInt(u64, buf[at + label.len ..][0..16], 16) catch 0;
+}
+
+/// What this process lacks to trace with BPF, said into `buf`; null when
+/// it has the capabilities (which inside a user namespace still is not
+/// enough: the kernel wants them in the initial one).
+pub fn missing(buf: []u8) ?[]const u8 {
+ const caps = capabilities();
+ const has = struct {
+ fn has(all: u64, cap: u6) bool {
+ return all >> cap & 1 != 0;
+ }
+ }.has;
+ if (has(caps, linux.CAP.SYS_ADMIN) or (has(caps, linux.CAP.BPF) and has(caps, linux.CAP.PERFMON))) return null;
+ return std.fmt.bufPrint(buf, "missing {s}{s}{s} (effective capabilities {x:0>16}, uid {d}). " ++
+ "CAP_BPF is what bpf() wants to create maps and load programs, CAP_PERFMON to load tracing programs and open kprobes and uprobes; CAP_SYS_ADMIN stands for both. " ++
+ "Run as root, or give the binary the two: setcap cap_bpf,cap_perfmon+ep ztrace", .{
+ if (has(caps, linux.CAP.BPF)) "" else "CAP_BPF",
+ if (has(caps, linux.CAP.BPF) or has(caps, linux.CAP.PERFMON)) "" else " and ",
+ if (has(caps, linux.CAP.PERFMON)) "" else "CAP_PERFMON",
+ caps,
+ linux.getuid(),
+ }) catch null;
+}
+
+pub const Tracer = struct {
+ gpa: std.mem.Allocator,
+ /// The process traced: its thread group id, in the initial pid namespace.
+ /// shortcut: one process, and a tracer in a pid namespace of its own sees nothing; keep the ids in a BPF map to follow children.
+ pid: linux.pid_t,
+ sink: event.Sink,
+ maps: Maps,
+ ring: RingBuf,
+ /// Programs and what attaches them.
+ fds: std.ArrayList(linux.fd_t) = .empty,
+ /// What the verifier said about the last program, when it refused one.
+ log: []u8,
+
+ pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, sink: event.Sink, ring_bytes: u32) !Tracer {
+ const ring = try BPF.map_create(.ringbuf, 0, 0, ring_bytes);
+ errdefer _ = linux.close(ring);
+ const drops = try BPF.map_create(.array, 4, 8, 1);
+ errdefer _ = linux.close(drops);
+ const ring_map = try RingBuf.map(ring, ring_bytes);
+ const log = try gpa.alloc(u8, 1 << 16);
+ log[0] = 0;
+ return .{ .gpa = gpa, .pid = pid, .sink = sink, .maps = .{ .ring = ring, .drops = drops }, .ring = ring_map, .log = log };
+ }
+
+ pub fn close(t: *Tracer) void {
+ for (t.fds.items) |fd| _ = linux.close(fd);
+ t.fds.deinit(t.gpa);
+ _ = linux.munmap(@ptrCast(@alignCast(t.ring.consumer)), 4096);
+ _ = linux.munmap(@ptrCast(@alignCast(t.ring.producer)), 4096 + 2 * t.ring.data.len);
+ _ = linux.close(t.maps.ring);
+ _ = linux.close(t.maps.drops);
+ t.gpa.free(t.log);
+ }
+
+ fn load(t: *Tracer, kind: BPF.ProgType, program: Program) !linux.fd_t {
+ // GPL: bpf_probe_read_kernel is for programs that say so.
+ const fd = BPF.prog_load(kind, program.code(), null, "GPL", 0, 0) catch |err| {
+ // Again for the verifier's reasons, which a good program need not pay for.
+ var log: BPF.Log = .{ .level = 1, .buf = t.log };
+ if (BPF.prog_load(kind, program.code(), &log, "GPL", 0, 0)) |again| _ = linux.close(again) else |_| {}
+ return err;
+ };
+ try t.fds.append(t.gpa, fd);
+ return fd;
+ }
+
+ fn rawTracepoint(t: *Tracer, name: [*:0]const u8, program: Program) !void {
+ var attr = std.mem.zeroes(BPF.Attr);
+ attr.raw_tracepoint = .{ .name = @intFromPtr(name), .prog_fd = try t.load(.raw_tracepoint, program) };
+ const rc = linux.bpf(.raw_tracepoint_open, &attr, @sizeOf(BPF.RawTracepointAttr));
+ if (linux.errno(rc) != .SUCCESS) return error.BpfAttach;
+ try t.fds.append(t.gpa, @intCast(rc));
+ }
+
+ /// Every syscall of the process.
+ pub fn syscalls(t: *Tracer) !void {
+ try t.rawTracepoint("sys_enter", sysEnter(t.maps, @intCast(t.pid)));
+ try t.rawTracepoint("sys_exit", sysExit(t.maps, @intCast(t.pid)));
+ }
+
+ /// Every time one of its threads leaves or gets a CPU. `btf` is the
+ /// contents of /sys/kernel/btf/vmlinux.
+ pub fn sched(t: *Tracer, btf: []const u8) !void {
+ const pid_at = fieldOffset(btf, "task_struct", "pid") orelse return error.NoBtf;
+ const tgid_at = fieldOffset(btf, "task_struct", "tgid") orelse return error.NoBtf;
+ try t.rawTracepoint("sched_switch", schedSwitch(t.maps, @intCast(t.pid), pid_at, tgid_at));
+ }
+
+ /// A probe through the perf event of the kprobe or uprobe PMU: `target`
+ /// is a kernel function or a file, `offset` is into it.
+ fn attach(t: *Tracer, pmu: []const u8, target: [*:0]const u8, offset: u64, semaphore: u64, program: Program) !void {
+ var path_buf: [64]u8 = undefined;
+ var number: [16]u8 = undefined;
+ const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/sys/bus/event_source/devices/{s}/type", .{pmu}, 0), .{}, 0);
+ if (linux.errno(fd) != .SUCCESS) return error.NoProbeSupport;
+ defer _ = linux.close(@intCast(fd));
+ const n = linux.read(@intCast(fd), &number, number.len);
+ if (linux.errno(n) != .SUCCESS) return error.NoProbeSupport;
+ const kind = std.fmt.parseInt(u32, std.mem.trim(u8, number[0..n], "\n "), 10) catch return error.NoProbeSupport;
+
+ // A uprobe's semaphore (the offset in the file of its counter) rides in the upper half of config.
+ // On the first thread alone, and not inherited: a program on a probe
+ // runs for whoever hits it (hence the process id in ours), and the
+ // kernel reads `target` again, from the target's memory, for an
+ // event a clone would inherit, which fails the clone.
+ var attr: linux.perf_event_attr = .{ .type = @fromBackingInt(@intCast(kind)), .config = semaphore << 32, .config1 = @intFromPtr(target), .config2 = offset };
+ const event_fd = linux.perf_event_open(&attr, t.pid, -1, -1, linux.PERF.FLAG.FD_CLOEXEC);
+ if (linux.errno(event_fd) != .SUCCESS) return if (linux.errno(event_fd) == .NOENT) error.NoSuchProbe else error.BpfAttach;
+ try t.fds.append(t.gpa, @intCast(event_fd));
+ const loaded = try t.load(.kprobe, program);
+ if (linux.errno(linux.ioctl(@intCast(event_fd), linux.PERF.EVENT_IOC.SET_BPF, @intCast(loaded))) != .SUCCESS) return error.BpfAttach;
+ }
+
+ /// Every call of a kernel function by the process: a `probe` named
+ /// `function` (define the string yourself: its id is `id`).
+ pub fn kprobe(t: *Tracer, function: [*:0]const u8, id: u64) !void {
+ try t.attach("kprobe", function, 0, 0, probe(t.maps, @intCast(t.pid), id, null));
+ }
+
+ /// Every time the process runs the instruction at `offset` of the
+ /// file `path`.
+ pub fn uprobe(t: *Tracer, path: [*:0]const u8, offset: u64, id: u64) !void {
+ try t.attach("uprobe", path, offset, 0, probe(t.maps, @intCast(t.pid), id, null));
+ }
+
+ /// A static probe of the file `path` (`p` from `usdt.probes` with no
+ /// bias). The kernel raises its semaphore while we are attached.
+ pub fn usdtProbe(t: *Tracer, elf: usdt.Elf, path: [*:0]const u8, p: usdt.Probe, id: u64) !void {
+ const semaphore = if (p.semaphore == 0) 0 else elf.offset(p.semaphore) orelse return error.NoSuchProbe;
+ try t.attach("uprobe", path, elf.offset(p.pc) orelse return error.NoSuchProbe, semaphore, try usdtProgram(t.maps, @intCast(t.pid), id, p.args));
+ }
+
+ /// Waits up to the timeout for records and decodes them into the sink.
+ /// Returns how many.
+ pub fn poll(t: *Tracer, timeout_ms: i32) !usize {
+ var fds = [1]linux.pollfd{.{ .fd = t.maps.ring, .events = linux.POLL.IN, .revents = 0 }};
+ _ = linux.poll(&fds, 1, timeout_ms);
+ var n: usize = 0;
+ while (t.ring.next()) |record| : (n += 1) try decode(record, t.sink);
+ t.ring.commit();
+ return n;
+ }
+
+ /// Records that found the ring full.
+ pub fn lost(t: *Tracer) u64 {
+ var count: u64 = 0;
+ BPF.map_lookup_elem(t.maps.drops, &std.mem.toBytes(@as(u32, 0)), std.mem.asBytes(&count)) catch return 0;
+ return count;
+ }
+};
+
+/// A record of the ring into the sink: the event, and the registers when
+/// they follow it.
+fn decode(record: []const u8, sink: event.Sink) !void {
+ if (record.len < @sizeOf(Event)) return;
+ const ev = std.mem.bytesToValue(Event, record[0..@sizeOf(Event)]);
+ if (record.len < @sizeOf(Event) + pt_regs_bytes) return sink.emit(ev, "");
+ var regs: Regs = .{};
+ @memcpy(std.mem.asBytes(&regs)[0..pt_regs_bytes], record[@sizeOf(Event)..][0..pt_regs_bytes]);
+ try sink.emit(ev, std.mem.asBytes(&regs));
+}
+
+// What follows runs the programs without a kernel: enough of a BPF machine
+// for what they use, on this process's own memory.
+const Machine = struct {
+ pid_tgid: u64 = 0,
+ now: u64 = 0,
+ cpu: u64 = 0,
+ /// What bpf_ringbuf_output returns; not 0 is a full ring.
+ ring_full: bool = false,
+ sent: [512]u8 = undefined,
+ sent_len: ?usize = null,
+ drops: u64 = 0,
+
+ fn run(m: *Machine, code: []const Insn, context: *const anyopaque) !void {
+ var stack: [512]u8 align(8) = undefined;
+ var r: [11]u64 = @splat(0);
+ r[1] = @intFromPtr(context);
+ r[10] = @intFromPtr(&stack) + stack.len;
+ var pc: usize = 0;
+ var steps: usize = 0;
+ while (true) : (steps += 1) {
+ if (steps > 10_000 or pc >= code.len) return error.RanAway;
+ const insn = code[pc];
+ pc += 1;
+ const imm: u64 = @bitCast(@as(i64, insn.imm));
+ const source = if (insn.code & BPF.X != 0) r[insn.src] else imm;
+ const address = (if (insn.code & 7 == BPF.LDX) r[insn.src] else r[insn.dst]) +% @as(u64, @bitCast(@as(i64, insn.off)));
+ switch (insn.code) {
+ BPF.ALU64 | BPF.MOV | BPF.K, BPF.ALU64 | BPF.MOV | BPF.X => r[insn.dst] = source,
+ BPF.ALU64 | BPF.ADD | BPF.K, BPF.ALU64 | BPF.ADD | BPF.X => r[insn.dst] +%= source,
+ BPF.ALU64 | BPF.RSH | BPF.K => r[insn.dst] >>= @intCast(insn.imm),
+ BPF.ALU64 | BPF.LSH | BPF.K => r[insn.dst] <<= @intCast(insn.imm),
+ BPF.ALU64 | BPF.ARSH | BPF.K => r[insn.dst] = @bitCast(@as(i64, @bitCast(r[insn.dst])) >> @intCast(insn.imm)),
+ BPF.JMP | BPF.JEQ | BPF.K => if (r[insn.dst] == imm) {
+ pc = @intCast(@as(i64, @intCast(pc)) + insn.off);
+ },
+ BPF.JMP | BPF.JNE | BPF.K => if (r[insn.dst] != imm) {
+ pc = @intCast(@as(i64, @intCast(pc)) + insn.off);
+ },
+ BPF.LD | BPF.DW | BPF.IMM => {
+ r[insn.dst] = @as(u32, @bitCast(insn.imm)) | @as(u64, @as(u32, @bitCast(code[pc].imm))) << 32;
+ pc += 1;
+ },
+ inline BPF.LDX | BPF.MEM | BPF.B, BPF.LDX | BPF.MEM | BPF.H, BPF.LDX | BPF.MEM | BPF.W, BPF.LDX | BPF.MEM | BPF.DW => |code_now| {
+ r[insn.dst] = @as(*align(1) const Width(code_now), @ptrFromInt(address)).*;
+ },
+ inline BPF.STX | BPF.MEM | BPF.H, BPF.STX | BPF.MEM | BPF.W, BPF.STX | BPF.MEM | BPF.DW => |code_now| {
+ @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(r[insn.src]);
+ },
+ inline BPF.ST | BPF.MEM | BPF.H, BPF.ST | BPF.MEM | BPF.W, BPF.ST | BPF.MEM | BPF.DW => |code_now| {
+ @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(imm);
+ },
+ BPF.STX | BPF.XADD | BPF.DW => @as(*u64, @ptrFromInt(address)).* += r[insn.src],
+ BPF.JMP | BPF.CALL => r[0] = switch (@as(BPF.Helper, @fromBackingInt(@intCast(insn.imm)))) {
+ .get_current_pid_tgid => m.pid_tgid,
+ .ktime_get_ns => m.now,
+ .get_smp_processor_id => m.cpu,
+ .probe_read_kernel, .probe_read_user => read: {
+ @memcpy(@as([*]u8, @ptrFromInt(r[1]))[0..r[2]], @as([*]const u8, @ptrFromInt(r[3]))[0..r[2]]);
+ break :read 0;
+ },
+ .ringbuf_output => out: {
+ if (r[1] != 3 or m.ring_full) break :out @as(u64, @bitCast(@as(i64, -11)));
+ m.sent_len = r[3];
+ @memcpy(m.sent[0..r[3]], @as([*]const u8, @ptrFromInt(r[2]))[0..r[3]]);
+ break :out 0;
+ },
+ .map_lookup_elem => if (r[1] == 4 and @as(*const u32, @ptrFromInt(r[2])).* == 0) @intFromPtr(&m.drops) else 0,
+ else => return error.UnknownHelper,
+ },
+ BPF.JMP | BPF.EXIT => return,
+ else => return error.UnknownInstruction,
+ }
+ }
+ }
+
+ fn Width(comptime code: u8) type {
+ return switch (code & 0x18) {
+ BPF.B => u8,
+ BPF.H => u16,
+ BPF.W => u32,
+ else => u64,
+ };
+ }
+
+ fn sentEvent(m: *const Machine) Event {
+ return std.mem.bytesToValue(Event, m.sent[0..@sizeOf(Event)]);
+ }
+};
+
+const test_maps: Maps = .{ .ring = 3, .drops = 4 };
+
+test "the syscall programs: the right process only, the event the tracer expects" {
+ var regs: Regs = .{ .rdi = 1, .rsi = 2, .rdx = 3, .r10 = 4, .r8 = 5, .r9 = 6, .orig_rax = 257, .rip = 0x401000 };
+ const enter_context = [2]u64{ @intFromPtr(&regs), 257 };
+ var m: Machine = .{ .pid_tgid = 1000 << 32 | 1001, .now = 77 };
+ try m.run(sysEnter(test_maps, 1000).code(), &enter_context);
+ try std.testing.expectEqual(@as(?usize, @sizeOf(Event) + pt_regs_bytes), m.sent_len);
+ try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_enter, .aux = 257, .a = .{ 1, 2, 3, 4, 5, 6 } }, m.sentEvent());
+ try std.testing.expectEqualSlices(u8, std.mem.asBytes(&regs)[0..pt_regs_bytes], m.sent[@sizeOf(Event)..][0..pt_regs_bytes]);
+
+ const exit_context = [2]u64{ @intFromPtr(&regs), @bitCast(@as(i64, -2)) };
+ try m.run(sysExit(test_maps, 1000).code(), &exit_context);
+ try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len);
+ try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_exit, .aux = 257, .a = .{ @bitCast(@as(i64, -2)), 0, 0, 0, 0, 0 } }, m.sentEvent());
+
+ // Another process: nothing. A full ring: counted.
+ m.sent_len = null;
+ try m.run(sysEnter(test_maps, 999).code(), &enter_context);
+ try std.testing.expectEqual(@as(?usize, null), m.sent_len);
+ m.ring_full = true;
+ try m.run(sysExit(test_maps, 1000).code(), &exit_context);
+ try m.run(sysEnter(test_maps, 1000).code(), &enter_context);
+ try std.testing.expectEqual(@as(u64, 2), m.drops);
+}
+
+test "the sched_switch program: either side of the switch, ids from where BTF says" {
+ // Two task_structs with the ids at offsets of our choosing.
+ var prev: [64]u32 = @splat(0);
+ var next: [64]u32 = @splat(0);
+ prev[10] = 1001;
+ prev[11] = 1000;
+ next[10] = 55;
+ next[11] = 50;
+ const context = [3]u64{ 1, @intFromPtr(&prev), @intFromPtr(&next) };
+ var m: Machine = .{ .now = 9, .cpu = 3 };
+ try m.run(schedSwitch(test_maps, 1000, 40, 44).code(), &context);
+ try std.testing.expectEqual(Event{ .ts = 9, .tid = 1001, .kind = .sched, .aux = 1, .a = .{ 1001, 55, 3, 0, 0, 0 } }, m.sentEvent());
+ m.sent_len = null;
+ try m.run(schedSwitch(test_maps, 50, 40, 44).code(), &context);
+ try std.testing.expectEqual(@as(u64, 55), m.sentEvent().a[1]);
+ m.sent_len = null;
+ try m.run(schedSwitch(test_maps, 7, 40, 44).code(), &context);
+ try std.testing.expectEqual(@as(?usize, null), m.sent_len);
+}
+
+test "the probe program and its record decoded; a USDT probe's arguments as usdt.Arg.fetch has them" {
+ // The probed thread's memory is ours: rbp and rdx point into this.
+ var memory: [64]u8 = undefined;
+ for (&memory, 0..) |*b, i| b.* = @truncate(0xf0 + i);
+ const regs: Regs = .{ .rdi = 11, .rsi = 0xffff_fffe, .rdx = @intFromPtr(&memory), .rcx = 3, .r8 = 15, .rbp = @intFromPtr(&memory) + 32, .rip = 0x401234, .rsp = 0x7000, .r10 = 0x1_0000_00ff };
+ var m: Machine = .{ .pid_tgid = 1000 << 32 | 1000, .now = 5 };
+ try m.run(probe(test_maps, 1000, 0xfeed, null).code(), &regs);
+ try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent());
+
+ // A sink that keeps the last event.
+ const Last = struct {
+ ev: Event = undefined,
+ regs: Regs = undefined,
+ fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void {
+ const l: *@This() = @ptrCast(@alignCast(ctx));
+ l.ev = ev.*;
+ l.regs = std.mem.bytesToValue(Regs, payload);
+ }
+ };
+ var last: Last = .{};
+ try decode(m.sent[0..m.sent_len.?], .{ .ctx = &last, .emitFn = Last.emit });
+ try std.testing.expectEqual(@as(u64, 11), last.ev.a[1]);
+ try std.testing.expectEqual(regs.rip, last.regs.rip);
+ try std.testing.expectEqual(@as(u64, 0), last.regs.fs_base);
+
+ const here = struct {
+ pub fn read(_: @This(), addr: u64, buf: []u8) usize {
+ @memcpy(buf, @as([*]const u8, @ptrFromInt(addr)));
+ return buf.len;
+ }
+ }{};
+ const specs = [_][5][]const u8{
+ .{ "-4@%esi", "4@%esi", "1@-16(%rbp)", "8@$9", "-8@$-9" },
+ .{ "-1@8(%rdx,%rcx,8)", "2@(%rdx)", "-2@-32(%rbp)", "1@%r10b", "8@%r10" },
+ };
+ for (specs) |five| {
+ var args: [5]usdt.Arg = undefined;
+ for (&args, five) |*a, spec| a.* = try .parse(spec);
+ try m.run(probe(test_maps, 1000, 0xfeed, &args).code(), &regs);
+ for (args, m.sentEvent().a[1..]) |a, got| try std.testing.expectEqual(a.fetch(&regs, here).?, got);
+ }
+ // Fewer than five: the rest is 0.
+ try m.run(probe(test_maps, 1000, 0xfeed, &.{try .parse("4@%edi")}).code(), &regs);
+ try std.testing.expectEqual([6]u64{ 0xfeed, 11, 0, 0, 0, 0 }, m.sentEvent().a);
+}
+
+test "a ring buffer as the kernel lays it out: finished records only, discarded ones skipped" {
+ var consumer: u64 = 0;
+ var producer: u64 = 0;
+ var data: [64]u8 align(8) = @splat(0);
+ var ring: RingBuf = .{ .consumer = &consumer, .producer = &producer, .data = &data };
+ const records = [_]struct { u32, []const u8 }{ .{ 5, "hello" }, .{ 3 | RingBuf.discarded, "no!" }, .{ 2, "ok" }, .{ 4 | RingBuf.busy, "wait" } };
+ for (records) |record| {
+ std.mem.writeInt(u32, data[producer..][0..4], record[0], .little);
+ @memcpy(data[producer + 8 ..][0..record[1].len], record[1]);
+ producer += std.mem.alignForward(u64, record[1].len + 8, 8);
+ }
+ try std.testing.expectEqualStrings("hello", ring.next().?);
+ try std.testing.expectEqualStrings("ok", ring.next().?);
+ try std.testing.expect(ring.next() == null);
+ ring.commit();
+ try std.testing.expectEqual(@as(u64, 48), consumer);
+ // The producer finishes it.
+ std.mem.writeInt(u32, data[48..52], 4, .little);
+ try std.testing.expectEqualStrings("wait", ring.next().?);
+}
+
+test "task_struct's pid and tgid in this kernel's BTF; privilege is told apart" {
+ var buf: [512]u8 = undefined;
+ if (missing(&buf)) |why| try std.testing.expect(std.mem.indexOf(u8, why, "CAP_") != null);
+
+ const btf = std.Io.Dir.cwd().readFileAlloc(std.testing.io, "/sys/kernel/btf/vmlinux", std.testing.allocator, .unlimited) catch return error.SkipZigTest;
+ defer std.testing.allocator.free(btf);
+ const pid_at = fieldOffset(btf, "task_struct", "pid").?;
+ // Next to each other, some way into a struct of a few kilobytes.
+ try std.testing.expectEqual(pid_at + 4, fieldOffset(btf, "task_struct", "tgid").?);
+ try std.testing.expect(pid_at > 256 and pid_at < 16384);
+ try std.testing.expect(fieldOffset(btf, "task_struct", "no_such_field") == null);
+ try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "rdi")), fieldOffset(btf, "pt_regs", "di"));
+ try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "orig_rax")), fieldOffset(btf, "pt_regs", "orig_ax"));
+}