diff options
| author | Gabriel Schneider <[email protected]> | 2026-11-10 11:49:22 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-11-10 12:19:06 -0300 |
| commit | 297943b01e18bb3d1dd41a8d3e220811de25228b (patch) | |
| tree | 4938e643abada811d3ab28b14c0e31dc94ebda62 /src/trace/bpf.zig | |
| parent | db923fa823354270217362db06356f2b1a41315e (diff) | |
| download | computeranim-297943b01e18bb3d1dd41a8d3e220811de25228b.tar.gz computeranim-297943b01e18bb3d1dd41a8d3e220811de25228b.zip | |
ztrace probe: BPF on syscalls, scheduling, kprobes, uprobes and USDT
Programs built as instruction arrays for one process (no compiler, no
libbpf), writing Events into a BPF ring buffer; a drop counter in a map;
task_struct offsets from the kernel's BTF; a USDT probe's arguments read
in the kernel at the probe. Without CAP_BPF and CAP_PERFMON it says so;
--dry-run builds and lists everything. Tested offline by running the
programs in a small BPF machine, and for real by test-kernel-vm, which
boots this kernel in QEMU where we are root.
ztrace perf: moments are no longer cut per event unless --cut;
--max-size, and a full recording stops the target and says so.
perf.Sorted merges two backends' events by time.
Co-Authored-By: Claude Fable 5.2 <[email protected]>
Diffstat (limited to 'src/trace/bpf.zig')
| -rw-r--r-- | src/trace/bpf.zig | 733 |
1 files changed, 733 insertions, 0 deletions
diff --git a/src/trace/bpf.zig b/src/trace/bpf.zig new file mode 100644 index 0000000..4069b01 --- /dev/null +++ b/src/trace/bpf.zig @@ -0,0 +1,733 @@ +//! The kernel's view with privilege: BPF programs on the raw tracepoints +//! sys_enter, sys_exit and sched_switch, and on any kprobe, uprobe or USDT +//! probe. Needs CAP_BPF and CAP_PERFMON (root). Each program writes an +//! `Event`, and for a syscall's entry and a probe the registers after it, +//! into one BPF ring buffer; `Tracer.poll` moves them into a `Sink`. +//! +//! The programs are instruction arrays built here, for one process (its id +//! is a constant in them): no compiler, no object file, no libbpf. What the +//! kernel does not tell (where `pid` is in `task_struct`) comes from its BTF. +const std = @import("std"); +const linux = std.os.linux; +const BPF = linux.BPF; +const Insn = BPF.Insn; +const event = @import("event.zig"); +const usdt = @import("usdt.zig"); +const Event = event.Event; +const Regs = event.Regs; +const Kind = event.Kind; + +/// The kernel's `struct pt_regs` is `Regs` up to here. +const pt_regs_bytes = @offsetOf(Regs, "fs_base"); +/// On a program's stack, from the frame pointer down: the key of the drop +/// counter, then the record. An event alone, or an event and registers. +const key_at = -8; +const small_at = key_at - @sizeOf(Event); +const large_at = small_at - pt_regs_bytes; + +fn field(comptime name: []const u8) i16 { + return @offsetOf(Event, name); +} + +fn arg(i: usize) i16 { + return @intCast(@offsetOf(Event, "a") + 8 * i); +} + +fn reg(comptime name: []const u8) i16 { + return @offsetOf(Regs, name); +} + +pub const Program = struct { + insns: [192]Insn = undefined, + len: usize = 0, + /// The jump that leaves early, to be pointed at the end. + leave: ?usize = null, + + pub fn code(p: *const Program) []const Insn { + return p.insns[0..p.len]; + } + + fn add(p: *Program, insn: Insn) void { + p.insns[p.len] = insn; + p.len += 1; + } + + /// r6 = the context, r7 = thread and process id; goes on only in a + /// thread of process `tgid`. + fn enter(p: *Program, tgid: u32) void { + p.add(Insn.mov(.r6, .r1)); + p.add(Insn.call(.get_current_pid_tgid)); + p.add(Insn.mov(.r7, .r0)); + p.add(Insn.rsh(.r0, 32)); + p.leave = p.len; + p.add(Insn.jne(.r0, @as(i32, @intCast(tgid)), 0)); + } + + /// The time, the thread and the kind of the event at `at`. + fn stamp(p: *Program, at: i16, kind: Kind) void { + p.add(Insn.call(.ktime_get_ns)); + p.add(Insn.stx(.double_word, .r10, at + field("ts"), .r0)); + p.add(Insn.stx(.word, .r10, at + field("tid"), .r7)); + p.add(Insn.st(.half_word, .r10, at + field("kind"), @backingInt(kind))); + } + + /// bpf_probe_read_kernel(frame + `to`, `size`, r3) + fn read(p: *Program, to: i16, size: i32) void { + p.add(Insn.mov(.r1, .r10)); + p.add(Insn.add(.r1, to)); + p.add(Insn.mov(.r2, size)); + p.add(Insn.call(.probe_read_kernel)); + } + + /// The low `bytes` of a register, extended to all of it. + fn extend(p: *Program, r: Insn.Reg, bytes: u8, signed: bool) void { + if (bytes >= 8) return; + const shift: i32 = 64 - 8 * @as(i32, bytes); + p.add(Insn.lsh(r, shift)); + p.add(if (signed) Insn.arsh(r, shift) else Insn.rsh(r, shift)); + } + + /// A register of the probed thread, from the context in r6. + fn probed(p: *Program, to: Insn.Reg, r: anytype) void { + p.add(Insn.ldx(.double_word, to, .r6, @as(i16, r.slot) * 8)); + p.extend(to, r.bytes, false); + } + + /// One argument of a USDT probe into the u64 at `to`: what + /// `usdt.Arg.fetch` computes, in the kernel, at the probe. Memory that + /// cannot be read gives 0. + fn usdtArg(p: *Program, a: usdt.Arg, to: i16) void { + switch (a.at) { + .constant => |c| { + p.add(Insn.ld_dw1(.r1, @bitCast(c))); + p.add(Insn.ld_dw2(@bitCast(c))); + }, + .register => |r| p.probed(.r1, r), + .memory => |m| { + p.add(Insn.st(.double_word, .r10, to, 0)); + p.probed(.r3, m.base); + if (m.index) |index| { + p.probed(.r1, index); + p.add(Insn.lsh(.r1, @as(i32, @ctz(m.scale)))); + p.add(Insn.add(.r3, .r1)); + } + p.add(Insn.ld_dw1(.r1, @bitCast(m.disp))); + p.add(Insn.ld_dw2(@bitCast(m.disp))); + p.add(Insn.add(.r3, .r1)); + p.add(Insn.mov(.r1, .r10)); + p.add(Insn.add(.r1, to)); + p.add(Insn.mov(.r2, @as(i32, a.size))); + p.add(Insn.call(.probe_read_user)); + p.add(Insn.ldx(.double_word, .r1, .r10, to)); + }, + } + p.extend(.r1, a.size, a.signed); + p.add(Insn.stx(.double_word, .r10, to, .r1)); + } + + /// Sends the `size` bytes at `at`, counts the record in `drops` when the + /// ring is full, and ends the program. + fn send(p: *Program, maps: Maps, at: i16, size: i32) void { + p.add(Insn.ld_map_fd1(.r1, maps.ring)); + p.add(Insn.ld_map_fd2(maps.ring)); + p.add(Insn.mov(.r2, .r10)); + p.add(Insn.add(.r2, at)); + p.add(Insn.mov(.r3, size)); + p.add(Insn.mov(.r4, 0)); + p.add(Insn.call(.ringbuf_output)); + p.add(Insn.jeq(.r0, 0, 9)); + p.add(Insn.st(.word, .r10, key_at, 0)); + p.add(Insn.ld_map_fd1(.r1, maps.drops)); + p.add(Insn.ld_map_fd2(maps.drops)); + p.add(Insn.mov(.r2, .r10)); + p.add(Insn.add(.r2, key_at)); + p.add(Insn.call(.map_lookup_elem)); + p.add(Insn.jeq(.r0, 0, 2)); + p.add(Insn.mov(.r1, 1)); + p.add(Insn.xadd(.r0, .r1)); + if (p.leave) |at_jump| p.insns[at_jump].off = @intCast(p.len - at_jump - 1); + p.add(Insn.mov(.r0, 0)); + p.add(Insn.exit()); + } +}; + +/// The maps every program writes to: the ring of records, and an array of +/// one u64 counting the records that found it full. +pub const Maps = struct { ring: linux.fd_t, drops: linux.fd_t }; + +/// raw tracepoint sys_enter(struct pt_regs *regs, long id): a +/// `syscall_enter` and the registers. +pub fn sysEnter(maps: Maps, tgid: u32) Program { + var p: Program = .{}; + p.enter(tgid); + p.add(Insn.ldx(.double_word, .r3, .r6, 0)); + p.read(large_at + @sizeOf(Event), pt_regs_bytes); + p.stamp(large_at, .syscall_enter); + p.add(Insn.ldx(.double_word, .r1, .r6, 8)); + p.add(Insn.stx(.half_word, .r10, large_at + field("aux"), .r1)); + inline for (.{ "rdi", "rsi", "rdx", "r10", "r8", "r9" }, 0..) |name, i| { + p.add(Insn.ldx(.double_word, .r1, .r10, large_at + @sizeOf(Event) + reg(name))); + p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1)); + } + p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes); + return p; +} + +/// raw tracepoint sys_exit(struct pt_regs *regs, long ret): a `syscall_exit`. +pub fn sysExit(maps: Maps, tgid: u32) Program { + var p: Program = .{}; + const number_at = small_at - 8; + p.enter(tgid); + p.add(Insn.ldx(.double_word, .r3, .r6, 0)); + p.add(Insn.add(.r3, reg("orig_rax"))); + p.read(number_at, 8); + p.stamp(small_at, .syscall_exit); + p.add(Insn.ldx(.double_word, .r1, .r10, number_at)); + p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); + p.add(Insn.ldx(.double_word, .r1, .r6, 8)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r1)); + for (1..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0)); + p.send(maps, small_at, @sizeOf(Event)); + return p; +} + +/// raw tracepoint sched_switch(bool preempt, struct task_struct *prev, +/// *next): a `sched` when either is a thread of the process. `pid_at` and +/// `tgid_at` are where those fields are in `task_struct`. +pub fn schedSwitch(maps: Maps, tgid: u32, pid_at: u32, tgid_at: u32) Program { + var p: Program = .{}; + // prev's pid and tgid, next's pid and tgid, as u32 below the event. + const ids_at = small_at - 16; + p.add(Insn.mov(.r6, .r1)); + for ([_]i16{ 8, 16 }, 0..) |task, t| for ([_]u32{ pid_at, tgid_at }, 0..) |at, i| { + p.add(Insn.ldx(.double_word, .r3, .r6, task)); + p.add(Insn.add(.r3, @as(i32, @intCast(at)))); + p.read(ids_at + @as(i16, @intCast(8 * t + 4 * i)), 4); + }; + p.add(Insn.ldx(.word, .r1, .r10, ids_at + 4)); + p.add(Insn.jeq(.r1, @as(i32, @intCast(tgid)), 2)); + p.add(Insn.ldx(.word, .r1, .r10, ids_at + 12)); + p.leave = p.len; + p.add(Insn.jne(.r1, @as(i32, @intCast(tgid)), 0)); + p.add(Insn.ldx(.word, .r7, .r10, ids_at)); + p.stamp(small_at, .sched); + p.add(Insn.ldx(.double_word, .r1, .r6, 0)); + p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r7)); + p.add(Insn.ldx(.word, .r1, .r10, ids_at + 8)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(1), .r1)); + p.add(Insn.call(.get_smp_processor_id)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(2), .r0)); + for (3..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0)); + p.send(maps, small_at, @sizeOf(Event)); + return p; +} + +/// A kprobe or uprobe (the context is the registers there): a `probe` +/// named `id`, and the registers. Its a1..a5 are the first five arguments +/// of a function call, or with `args` those of a USDT probe, read there +/// and then. +pub fn probe(maps: Maps, tgid: u32, id: u64, args: ?[]const usdt.Arg) Program { + var p: Program = .{}; + p.enter(tgid); + p.stamp(large_at, .probe); + p.add(Insn.st(.half_word, .r10, large_at + field("aux"), 0)); + p.add(Insn.ld_dw1(.r1, id)); + p.add(Insn.ld_dw2(id)); + p.add(Insn.stx(.double_word, .r10, large_at + arg(0), .r1)); + inline for (.{ "rdi", "rsi", "rdx", "rcx", "r8" }, 1..) |name, i| { + if (args == null) { + p.add(Insn.ldx(.double_word, .r1, .r6, reg(name))); + p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1)); + } else if (i <= args.?.len) { + p.usdtArg(args.?[i - 1], large_at + arg(i)); + } else p.add(Insn.st(.double_word, .r10, large_at + arg(i), 0)); + } + for (0..pt_regs_bytes / 8) |i| { + p.add(Insn.ldx(.double_word, .r1, .r6, @intCast(8 * i))); + p.add(Insn.stx(.double_word, .r10, large_at + @sizeOf(Event) + @as(i16, @intCast(8 * i)), .r1)); + } + p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes); + return p; +} + +/// `probe` for a USDT probe with the arguments `specs` ("-4@%edi +/// 8@-24(%rbp)"): the first five become a1..a5. +pub fn usdtProgram(maps: Maps, tgid: u32, id: u64, specs: []const u8) !Program { + var args: [5]usdt.Arg = undefined; + var count: usize = 0; + var each = std.mem.tokenizeScalar(u8, specs, ' '); + while (each.next()) |spec| { + if (count == args.len) break; + args[count] = try .parse(spec); + count += 1; + } + return probe(maps, tgid, id, args[0..count]); +} + +/// Where `member` is in the struct `name`, in bytes, from the kernel's +/// type information (the contents of /sys/kernel/btf/vmlinux). +pub fn fieldOffset(btf: []const u8, name: []const u8, member: []const u8) ?u32 { + const T = BPF.btf; + if (btf.len < @sizeOf(T.Header)) return null; + const header = std.mem.bytesToValue(T.Header, btf[0..@sizeOf(T.Header)]); + if (header.magic != T.magic or header.hdr_len + @as(u64, header.str_off) + header.str_len > btf.len) return null; + const types = btf[header.hdr_len + header.type_off ..][0..header.type_len]; + const strings = btf[header.hdr_len + header.str_off ..][0..header.str_len]; + var at: usize = 0; + while (at + @sizeOf(T.Type) <= types.len) { + const t = std.mem.bytesToValue(T.Type, types[at..][0..@sizeOf(T.Type)]); + at += @sizeOf(T.Type); + const count: usize = t.info.vlen; + // What follows the type, by kind. + const extra: usize = switch (t.info.kind) { + .int, .@"var", .decl_tag => 4, + .array => 12, + .@"struct", .@"union", .datasec, .enum64 => 12 * count, + .@"enum", .func_proto => 8 * count, + else => 0, + }; + if (at + extra > types.len) return null; + defer at += extra; + if (t.info.kind != .@"struct" or !std.mem.eql(u8, std.mem.sliceTo(strings[@min(t.name_off, strings.len)..], 0), name)) continue; + for (0..count) |i| { + const m = std.mem.bytesToValue(T.Member, types[at + 12 * i ..][0..12]); + if (std.mem.eql(u8, std.mem.sliceTo(strings[@min(m.name_off, strings.len)..], 0), member)) return m.offset.bit / 8; + } + } + return null; +} + +/// The reading end of a BPF ring buffer: a page with our position, a page +/// with the kernel's, and the data, which the kernel maps twice in a row so +/// that no record wraps. A record is a u32 length with two flag bits, four +/// bytes more, the bytes, padded to 8. +pub const RingBuf = struct { + consumer: *u64, + producer: *const u64, + data: []const u8, + /// Read so far; the kernel is told by `commit`, after the records were used. + at: u64 = 0, + + const busy: u32 = 1 << 31; + const discarded: u32 = 1 << 30; + + pub fn map(fd: linux.fd_t, size: usize) !RingBuf { + const ours = linux.mmap(null, 4096, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0); + if (linux.errno(ours) != .SUCCESS) return error.BpfRingMap; + const theirs = linux.mmap(null, 4096 + 2 * size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 4096); + if (linux.errno(theirs) != .SUCCESS) return error.BpfRingMap; + return .{ .consumer = @ptrFromInt(ours), .producer = @ptrFromInt(theirs), .data = @as([*]const u8, @ptrFromInt(theirs + 4096))[0..size] }; + } + + /// The next finished record, valid until `commit`. + pub fn next(r: *RingBuf) ?[]const u8 { + while (r.at < @atomicLoad(u64, r.producer, .acquire)) { + const start: usize = @intCast(r.at & (r.data.len - 1)); + const header = @atomicLoad(u32, @as(*const u32, @ptrCast(@alignCast(r.data.ptr + start))), .acquire); + if (header & busy != 0) return null; + const len = header & ~(busy | discarded); + r.at += std.mem.alignForward(u64, len + 8, 8); + if (header & discarded == 0) return (r.data.ptr + start + 8)[0..len]; + } + return null; + } + + pub fn commit(r: *RingBuf) void { + @atomicStore(u64, r.consumer, r.at, .release); + } +}; + +/// The capability bits of this process (CapEff of /proc/self/status). +fn capabilities() u64 { + var buf: [4096]u8 = undefined; + const fd = linux.open("/proc/self/status", .{}, 0); + if (linux.errno(fd) != .SUCCESS) return 0; + defer _ = linux.close(@intCast(fd)); + const n = linux.read(@intCast(fd), &buf, buf.len); + if (linux.errno(n) != .SUCCESS) return 0; + const label = "CapEff:\t"; + const at = std.mem.indexOf(u8, buf[0..n], label) orelse return 0; + return std.fmt.parseInt(u64, buf[at + label.len ..][0..16], 16) catch 0; +} + +/// What this process lacks to trace with BPF, said into `buf`; null when +/// it has the capabilities (which inside a user namespace still is not +/// enough: the kernel wants them in the initial one). +pub fn missing(buf: []u8) ?[]const u8 { + const caps = capabilities(); + const has = struct { + fn has(all: u64, cap: u6) bool { + return all >> cap & 1 != 0; + } + }.has; + if (has(caps, linux.CAP.SYS_ADMIN) or (has(caps, linux.CAP.BPF) and has(caps, linux.CAP.PERFMON))) return null; + return std.fmt.bufPrint(buf, "missing {s}{s}{s} (effective capabilities {x:0>16}, uid {d}). " ++ + "CAP_BPF is what bpf() wants to create maps and load programs, CAP_PERFMON to load tracing programs and open kprobes and uprobes; CAP_SYS_ADMIN stands for both. " ++ + "Run as root, or give the binary the two: setcap cap_bpf,cap_perfmon+ep ztrace", .{ + if (has(caps, linux.CAP.BPF)) "" else "CAP_BPF", + if (has(caps, linux.CAP.BPF) or has(caps, linux.CAP.PERFMON)) "" else " and ", + if (has(caps, linux.CAP.PERFMON)) "" else "CAP_PERFMON", + caps, + linux.getuid(), + }) catch null; +} + +pub const Tracer = struct { + gpa: std.mem.Allocator, + /// The process traced: its thread group id, in the initial pid namespace. + /// shortcut: one process, and a tracer in a pid namespace of its own sees nothing; keep the ids in a BPF map to follow children. + pid: linux.pid_t, + sink: event.Sink, + maps: Maps, + ring: RingBuf, + /// Programs and what attaches them. + fds: std.ArrayList(linux.fd_t) = .empty, + /// What the verifier said about the last program, when it refused one. + log: []u8, + + pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, sink: event.Sink, ring_bytes: u32) !Tracer { + const ring = try BPF.map_create(.ringbuf, 0, 0, ring_bytes); + errdefer _ = linux.close(ring); + const drops = try BPF.map_create(.array, 4, 8, 1); + errdefer _ = linux.close(drops); + const ring_map = try RingBuf.map(ring, ring_bytes); + const log = try gpa.alloc(u8, 1 << 16); + log[0] = 0; + return .{ .gpa = gpa, .pid = pid, .sink = sink, .maps = .{ .ring = ring, .drops = drops }, .ring = ring_map, .log = log }; + } + + pub fn close(t: *Tracer) void { + for (t.fds.items) |fd| _ = linux.close(fd); + t.fds.deinit(t.gpa); + _ = linux.munmap(@ptrCast(@alignCast(t.ring.consumer)), 4096); + _ = linux.munmap(@ptrCast(@alignCast(t.ring.producer)), 4096 + 2 * t.ring.data.len); + _ = linux.close(t.maps.ring); + _ = linux.close(t.maps.drops); + t.gpa.free(t.log); + } + + fn load(t: *Tracer, kind: BPF.ProgType, program: Program) !linux.fd_t { + // GPL: bpf_probe_read_kernel is for programs that say so. + const fd = BPF.prog_load(kind, program.code(), null, "GPL", 0, 0) catch |err| { + // Again for the verifier's reasons, which a good program need not pay for. + var log: BPF.Log = .{ .level = 1, .buf = t.log }; + if (BPF.prog_load(kind, program.code(), &log, "GPL", 0, 0)) |again| _ = linux.close(again) else |_| {} + return err; + }; + try t.fds.append(t.gpa, fd); + return fd; + } + + fn rawTracepoint(t: *Tracer, name: [*:0]const u8, program: Program) !void { + var attr = std.mem.zeroes(BPF.Attr); + attr.raw_tracepoint = .{ .name = @intFromPtr(name), .prog_fd = try t.load(.raw_tracepoint, program) }; + const rc = linux.bpf(.raw_tracepoint_open, &attr, @sizeOf(BPF.RawTracepointAttr)); + if (linux.errno(rc) != .SUCCESS) return error.BpfAttach; + try t.fds.append(t.gpa, @intCast(rc)); + } + + /// Every syscall of the process. + pub fn syscalls(t: *Tracer) !void { + try t.rawTracepoint("sys_enter", sysEnter(t.maps, @intCast(t.pid))); + try t.rawTracepoint("sys_exit", sysExit(t.maps, @intCast(t.pid))); + } + + /// Every time one of its threads leaves or gets a CPU. `btf` is the + /// contents of /sys/kernel/btf/vmlinux. + pub fn sched(t: *Tracer, btf: []const u8) !void { + const pid_at = fieldOffset(btf, "task_struct", "pid") orelse return error.NoBtf; + const tgid_at = fieldOffset(btf, "task_struct", "tgid") orelse return error.NoBtf; + try t.rawTracepoint("sched_switch", schedSwitch(t.maps, @intCast(t.pid), pid_at, tgid_at)); + } + + /// A probe through the perf event of the kprobe or uprobe PMU: `target` + /// is a kernel function or a file, `offset` is into it. + fn attach(t: *Tracer, pmu: []const u8, target: [*:0]const u8, offset: u64, semaphore: u64, program: Program) !void { + var path_buf: [64]u8 = undefined; + var number: [16]u8 = undefined; + const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/sys/bus/event_source/devices/{s}/type", .{pmu}, 0), .{}, 0); + if (linux.errno(fd) != .SUCCESS) return error.NoProbeSupport; + defer _ = linux.close(@intCast(fd)); + const n = linux.read(@intCast(fd), &number, number.len); + if (linux.errno(n) != .SUCCESS) return error.NoProbeSupport; + const kind = std.fmt.parseInt(u32, std.mem.trim(u8, number[0..n], "\n "), 10) catch return error.NoProbeSupport; + + // A uprobe's semaphore (the offset in the file of its counter) rides in the upper half of config. + // On the first thread alone, and not inherited: a program on a probe + // runs for whoever hits it (hence the process id in ours), and the + // kernel reads `target` again, from the target's memory, for an + // event a clone would inherit, which fails the clone. + var attr: linux.perf_event_attr = .{ .type = @fromBackingInt(@intCast(kind)), .config = semaphore << 32, .config1 = @intFromPtr(target), .config2 = offset }; + const event_fd = linux.perf_event_open(&attr, t.pid, -1, -1, linux.PERF.FLAG.FD_CLOEXEC); + if (linux.errno(event_fd) != .SUCCESS) return if (linux.errno(event_fd) == .NOENT) error.NoSuchProbe else error.BpfAttach; + try t.fds.append(t.gpa, @intCast(event_fd)); + const loaded = try t.load(.kprobe, program); + if (linux.errno(linux.ioctl(@intCast(event_fd), linux.PERF.EVENT_IOC.SET_BPF, @intCast(loaded))) != .SUCCESS) return error.BpfAttach; + } + + /// Every call of a kernel function by the process: a `probe` named + /// `function` (define the string yourself: its id is `id`). + pub fn kprobe(t: *Tracer, function: [*:0]const u8, id: u64) !void { + try t.attach("kprobe", function, 0, 0, probe(t.maps, @intCast(t.pid), id, null)); + } + + /// Every time the process runs the instruction at `offset` of the + /// file `path`. + pub fn uprobe(t: *Tracer, path: [*:0]const u8, offset: u64, id: u64) !void { + try t.attach("uprobe", path, offset, 0, probe(t.maps, @intCast(t.pid), id, null)); + } + + /// A static probe of the file `path` (`p` from `usdt.probes` with no + /// bias). The kernel raises its semaphore while we are attached. + pub fn usdtProbe(t: *Tracer, elf: usdt.Elf, path: [*:0]const u8, p: usdt.Probe, id: u64) !void { + const semaphore = if (p.semaphore == 0) 0 else elf.offset(p.semaphore) orelse return error.NoSuchProbe; + try t.attach("uprobe", path, elf.offset(p.pc) orelse return error.NoSuchProbe, semaphore, try usdtProgram(t.maps, @intCast(t.pid), id, p.args)); + } + + /// Waits up to the timeout for records and decodes them into the sink. + /// Returns how many. + pub fn poll(t: *Tracer, timeout_ms: i32) !usize { + var fds = [1]linux.pollfd{.{ .fd = t.maps.ring, .events = linux.POLL.IN, .revents = 0 }}; + _ = linux.poll(&fds, 1, timeout_ms); + var n: usize = 0; + while (t.ring.next()) |record| : (n += 1) try decode(record, t.sink); + t.ring.commit(); + return n; + } + + /// Records that found the ring full. + pub fn lost(t: *Tracer) u64 { + var count: u64 = 0; + BPF.map_lookup_elem(t.maps.drops, &std.mem.toBytes(@as(u32, 0)), std.mem.asBytes(&count)) catch return 0; + return count; + } +}; + +/// A record of the ring into the sink: the event, and the registers when +/// they follow it. +fn decode(record: []const u8, sink: event.Sink) !void { + if (record.len < @sizeOf(Event)) return; + const ev = std.mem.bytesToValue(Event, record[0..@sizeOf(Event)]); + if (record.len < @sizeOf(Event) + pt_regs_bytes) return sink.emit(ev, ""); + var regs: Regs = .{}; + @memcpy(std.mem.asBytes(®s)[0..pt_regs_bytes], record[@sizeOf(Event)..][0..pt_regs_bytes]); + try sink.emit(ev, std.mem.asBytes(®s)); +} + +// What follows runs the programs without a kernel: enough of a BPF machine +// for what they use, on this process's own memory. +const Machine = struct { + pid_tgid: u64 = 0, + now: u64 = 0, + cpu: u64 = 0, + /// What bpf_ringbuf_output returns; not 0 is a full ring. + ring_full: bool = false, + sent: [512]u8 = undefined, + sent_len: ?usize = null, + drops: u64 = 0, + + fn run(m: *Machine, code: []const Insn, context: *const anyopaque) !void { + var stack: [512]u8 align(8) = undefined; + var r: [11]u64 = @splat(0); + r[1] = @intFromPtr(context); + r[10] = @intFromPtr(&stack) + stack.len; + var pc: usize = 0; + var steps: usize = 0; + while (true) : (steps += 1) { + if (steps > 10_000 or pc >= code.len) return error.RanAway; + const insn = code[pc]; + pc += 1; + const imm: u64 = @bitCast(@as(i64, insn.imm)); + const source = if (insn.code & BPF.X != 0) r[insn.src] else imm; + const address = (if (insn.code & 7 == BPF.LDX) r[insn.src] else r[insn.dst]) +% @as(u64, @bitCast(@as(i64, insn.off))); + switch (insn.code) { + BPF.ALU64 | BPF.MOV | BPF.K, BPF.ALU64 | BPF.MOV | BPF.X => r[insn.dst] = source, + BPF.ALU64 | BPF.ADD | BPF.K, BPF.ALU64 | BPF.ADD | BPF.X => r[insn.dst] +%= source, + BPF.ALU64 | BPF.RSH | BPF.K => r[insn.dst] >>= @intCast(insn.imm), + BPF.ALU64 | BPF.LSH | BPF.K => r[insn.dst] <<= @intCast(insn.imm), + BPF.ALU64 | BPF.ARSH | BPF.K => r[insn.dst] = @bitCast(@as(i64, @bitCast(r[insn.dst])) >> @intCast(insn.imm)), + BPF.JMP | BPF.JEQ | BPF.K => if (r[insn.dst] == imm) { + pc = @intCast(@as(i64, @intCast(pc)) + insn.off); + }, + BPF.JMP | BPF.JNE | BPF.K => if (r[insn.dst] != imm) { + pc = @intCast(@as(i64, @intCast(pc)) + insn.off); + }, + BPF.LD | BPF.DW | BPF.IMM => { + r[insn.dst] = @as(u32, @bitCast(insn.imm)) | @as(u64, @as(u32, @bitCast(code[pc].imm))) << 32; + pc += 1; + }, + inline BPF.LDX | BPF.MEM | BPF.B, BPF.LDX | BPF.MEM | BPF.H, BPF.LDX | BPF.MEM | BPF.W, BPF.LDX | BPF.MEM | BPF.DW => |code_now| { + r[insn.dst] = @as(*align(1) const Width(code_now), @ptrFromInt(address)).*; + }, + inline BPF.STX | BPF.MEM | BPF.H, BPF.STX | BPF.MEM | BPF.W, BPF.STX | BPF.MEM | BPF.DW => |code_now| { + @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(r[insn.src]); + }, + inline BPF.ST | BPF.MEM | BPF.H, BPF.ST | BPF.MEM | BPF.W, BPF.ST | BPF.MEM | BPF.DW => |code_now| { + @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(imm); + }, + BPF.STX | BPF.XADD | BPF.DW => @as(*u64, @ptrFromInt(address)).* += r[insn.src], + BPF.JMP | BPF.CALL => r[0] = switch (@as(BPF.Helper, @fromBackingInt(@intCast(insn.imm)))) { + .get_current_pid_tgid => m.pid_tgid, + .ktime_get_ns => m.now, + .get_smp_processor_id => m.cpu, + .probe_read_kernel, .probe_read_user => read: { + @memcpy(@as([*]u8, @ptrFromInt(r[1]))[0..r[2]], @as([*]const u8, @ptrFromInt(r[3]))[0..r[2]]); + break :read 0; + }, + .ringbuf_output => out: { + if (r[1] != 3 or m.ring_full) break :out @as(u64, @bitCast(@as(i64, -11))); + m.sent_len = r[3]; + @memcpy(m.sent[0..r[3]], @as([*]const u8, @ptrFromInt(r[2]))[0..r[3]]); + break :out 0; + }, + .map_lookup_elem => if (r[1] == 4 and @as(*const u32, @ptrFromInt(r[2])).* == 0) @intFromPtr(&m.drops) else 0, + else => return error.UnknownHelper, + }, + BPF.JMP | BPF.EXIT => return, + else => return error.UnknownInstruction, + } + } + } + + fn Width(comptime code: u8) type { + return switch (code & 0x18) { + BPF.B => u8, + BPF.H => u16, + BPF.W => u32, + else => u64, + }; + } + + fn sentEvent(m: *const Machine) Event { + return std.mem.bytesToValue(Event, m.sent[0..@sizeOf(Event)]); + } +}; + +const test_maps: Maps = .{ .ring = 3, .drops = 4 }; + +test "the syscall programs: the right process only, the event the tracer expects" { + var regs: Regs = .{ .rdi = 1, .rsi = 2, .rdx = 3, .r10 = 4, .r8 = 5, .r9 = 6, .orig_rax = 257, .rip = 0x401000 }; + const enter_context = [2]u64{ @intFromPtr(®s), 257 }; + var m: Machine = .{ .pid_tgid = 1000 << 32 | 1001, .now = 77 }; + try m.run(sysEnter(test_maps, 1000).code(), &enter_context); + try std.testing.expectEqual(@as(?usize, @sizeOf(Event) + pt_regs_bytes), m.sent_len); + try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_enter, .aux = 257, .a = .{ 1, 2, 3, 4, 5, 6 } }, m.sentEvent()); + try std.testing.expectEqualSlices(u8, std.mem.asBytes(®s)[0..pt_regs_bytes], m.sent[@sizeOf(Event)..][0..pt_regs_bytes]); + + const exit_context = [2]u64{ @intFromPtr(®s), @bitCast(@as(i64, -2)) }; + try m.run(sysExit(test_maps, 1000).code(), &exit_context); + try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len); + try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_exit, .aux = 257, .a = .{ @bitCast(@as(i64, -2)), 0, 0, 0, 0, 0 } }, m.sentEvent()); + + // Another process: nothing. A full ring: counted. + m.sent_len = null; + try m.run(sysEnter(test_maps, 999).code(), &enter_context); + try std.testing.expectEqual(@as(?usize, null), m.sent_len); + m.ring_full = true; + try m.run(sysExit(test_maps, 1000).code(), &exit_context); + try m.run(sysEnter(test_maps, 1000).code(), &enter_context); + try std.testing.expectEqual(@as(u64, 2), m.drops); +} + +test "the sched_switch program: either side of the switch, ids from where BTF says" { + // Two task_structs with the ids at offsets of our choosing. + var prev: [64]u32 = @splat(0); + var next: [64]u32 = @splat(0); + prev[10] = 1001; + prev[11] = 1000; + next[10] = 55; + next[11] = 50; + const context = [3]u64{ 1, @intFromPtr(&prev), @intFromPtr(&next) }; + var m: Machine = .{ .now = 9, .cpu = 3 }; + try m.run(schedSwitch(test_maps, 1000, 40, 44).code(), &context); + try std.testing.expectEqual(Event{ .ts = 9, .tid = 1001, .kind = .sched, .aux = 1, .a = .{ 1001, 55, 3, 0, 0, 0 } }, m.sentEvent()); + m.sent_len = null; + try m.run(schedSwitch(test_maps, 50, 40, 44).code(), &context); + try std.testing.expectEqual(@as(u64, 55), m.sentEvent().a[1]); + m.sent_len = null; + try m.run(schedSwitch(test_maps, 7, 40, 44).code(), &context); + try std.testing.expectEqual(@as(?usize, null), m.sent_len); +} + +test "the probe program and its record decoded; a USDT probe's arguments as usdt.Arg.fetch has them" { + // The probed thread's memory is ours: rbp and rdx point into this. + var memory: [64]u8 = undefined; + for (&memory, 0..) |*b, i| b.* = @truncate(0xf0 + i); + const regs: Regs = .{ .rdi = 11, .rsi = 0xffff_fffe, .rdx = @intFromPtr(&memory), .rcx = 3, .r8 = 15, .rbp = @intFromPtr(&memory) + 32, .rip = 0x401234, .rsp = 0x7000, .r10 = 0x1_0000_00ff }; + var m: Machine = .{ .pid_tgid = 1000 << 32 | 1000, .now = 5 }; + try m.run(probe(test_maps, 1000, 0xfeed, null).code(), ®s); + try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent()); + + // A sink that keeps the last event. + const Last = struct { + ev: Event = undefined, + regs: Regs = undefined, + fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void { + const l: *@This() = @ptrCast(@alignCast(ctx)); + l.ev = ev.*; + l.regs = std.mem.bytesToValue(Regs, payload); + } + }; + var last: Last = .{}; + try decode(m.sent[0..m.sent_len.?], .{ .ctx = &last, .emitFn = Last.emit }); + try std.testing.expectEqual(@as(u64, 11), last.ev.a[1]); + try std.testing.expectEqual(regs.rip, last.regs.rip); + try std.testing.expectEqual(@as(u64, 0), last.regs.fs_base); + + const here = struct { + pub fn read(_: @This(), addr: u64, buf: []u8) usize { + @memcpy(buf, @as([*]const u8, @ptrFromInt(addr))); + return buf.len; + } + }{}; + const specs = [_][5][]const u8{ + .{ "-4@%esi", "4@%esi", "1@-16(%rbp)", "8@$9", "-8@$-9" }, + .{ "-1@8(%rdx,%rcx,8)", "2@(%rdx)", "-2@-32(%rbp)", "1@%r10b", "8@%r10" }, + }; + for (specs) |five| { + var args: [5]usdt.Arg = undefined; + for (&args, five) |*a, spec| a.* = try .parse(spec); + try m.run(probe(test_maps, 1000, 0xfeed, &args).code(), ®s); + for (args, m.sentEvent().a[1..]) |a, got| try std.testing.expectEqual(a.fetch(®s, here).?, got); + } + // Fewer than five: the rest is 0. + try m.run(probe(test_maps, 1000, 0xfeed, &.{try .parse("4@%edi")}).code(), ®s); + try std.testing.expectEqual([6]u64{ 0xfeed, 11, 0, 0, 0, 0 }, m.sentEvent().a); +} + +test "a ring buffer as the kernel lays it out: finished records only, discarded ones skipped" { + var consumer: u64 = 0; + var producer: u64 = 0; + var data: [64]u8 align(8) = @splat(0); + var ring: RingBuf = .{ .consumer = &consumer, .producer = &producer, .data = &data }; + const records = [_]struct { u32, []const u8 }{ .{ 5, "hello" }, .{ 3 | RingBuf.discarded, "no!" }, .{ 2, "ok" }, .{ 4 | RingBuf.busy, "wait" } }; + for (records) |record| { + std.mem.writeInt(u32, data[producer..][0..4], record[0], .little); + @memcpy(data[producer + 8 ..][0..record[1].len], record[1]); + producer += std.mem.alignForward(u64, record[1].len + 8, 8); + } + try std.testing.expectEqualStrings("hello", ring.next().?); + try std.testing.expectEqualStrings("ok", ring.next().?); + try std.testing.expect(ring.next() == null); + ring.commit(); + try std.testing.expectEqual(@as(u64, 48), consumer); + // The producer finishes it. + std.mem.writeInt(u32, data[48..52], 4, .little); + try std.testing.expectEqualStrings("wait", ring.next().?); +} + +test "task_struct's pid and tgid in this kernel's BTF; privilege is told apart" { + var buf: [512]u8 = undefined; + if (missing(&buf)) |why| try std.testing.expect(std.mem.indexOf(u8, why, "CAP_") != null); + + const btf = std.Io.Dir.cwd().readFileAlloc(std.testing.io, "/sys/kernel/btf/vmlinux", std.testing.allocator, .unlimited) catch return error.SkipZigTest; + defer std.testing.allocator.free(btf); + const pid_at = fieldOffset(btf, "task_struct", "pid").?; + // Next to each other, some way into a struct of a few kilobytes. + try std.testing.expectEqual(pid_at + 4, fieldOffset(btf, "task_struct", "tgid").?); + try std.testing.expect(pid_at > 256 and pid_at < 16384); + try std.testing.expect(fieldOffset(btf, "task_struct", "no_such_field") == null); + try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "rdi")), fieldOffset(btf, "pt_regs", "di")); + try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "orig_rax")), fieldOffset(btf, "pt_regs", "orig_ax")); +} |
