//! The kernel's view with privilege: BPF programs on the raw tracepoints //! sys_enter, sys_exit and sched_switch, and on any kprobe, uprobe or USDT //! probe. Needs CAP_BPF and CAP_PERFMON (root). Each program writes an //! `Event`, and for a probe in user space the thread's registers after it, //! into one BPF ring buffer; `Tracer.poll` moves them into a `Sink`. //! //! The programs are instruction arrays built here, for one process (its id //! is a constant in them): no compiler, no object file, no libbpf. What the //! kernel does not tell (where `pid` is in `task_struct`) comes from its BTF. const std = @import("std"); const linux = std.os.linux; const BPF = linux.BPF; const Insn = BPF.Insn; const event = @import("event.zig"); const usdt = @import("usdt.zig"); const Event = event.Event; const Regs = event.Regs; const Kind = event.Kind; /// The kernel's `struct pt_regs` is `Regs` up to here. const pt_regs_bytes = @offsetOf(Regs, "fs_base"); /// On a program's stack, from the frame pointer down: the key of the drop /// counter, then the record. An event alone, or an event and registers. const key_at = -8; const small_at = key_at - @sizeOf(Event); const large_at = small_at - pt_regs_bytes; fn field(comptime name: []const u8) i16 { return @offsetOf(Event, name); } fn arg(i: usize) i16 { return @intCast(@offsetOf(Event, "a") + 8 * i); } fn reg(comptime name: []const u8) i16 { return @offsetOf(Regs, name); } pub const Program = struct { insns: [192]Insn = undefined, len: usize = 0, /// The jump that leaves early, to be pointed at the end. leave: ?usize = null, pub fn code(p: *const Program) []const Insn { return p.insns[0..p.len]; } fn add(p: *Program, insn: Insn) void { p.insns[p.len] = insn; p.len += 1; } /// r6 = the context, r7 = thread and process id; goes on only in a /// thread of process `tgid`. fn enter(p: *Program, tgid: u32) void { p.add(Insn.mov(.r6, .r1)); p.add(Insn.call(.get_current_pid_tgid)); p.add(Insn.mov(.r7, .r0)); p.add(Insn.rsh(.r0, 32)); p.leave = p.len; p.add(Insn.jne(.r0, @as(i32, @intCast(tgid)), 0)); } /// The time, the thread and the kind of the event at `at`. fn stamp(p: *Program, at: i16, kind: Kind) void { p.add(Insn.call(.ktime_get_ns)); p.add(Insn.stx(.double_word, .r10, at + field("ts"), .r0)); p.add(Insn.stx(.word, .r10, at + field("tid"), .r7)); p.add(Insn.st(.half_word, .r10, at + field("kind"), @backingInt(kind))); } /// bpf_probe_read_kernel(frame + `to`, `size`, r3) fn read(p: *Program, to: i16, size: i32) void { p.add(Insn.mov(.r1, .r10)); p.add(Insn.add(.r1, to)); p.add(Insn.mov(.r2, size)); p.add(Insn.call(.probe_read_kernel)); } /// The low `bytes` of a register, extended to all of it. fn extend(p: *Program, r: Insn.Reg, bytes: u8, signed: bool) void { if (bytes >= 8) return; const shift: i32 = 64 - 8 * @as(i32, bytes); p.add(Insn.lsh(r, shift)); p.add(if (signed) Insn.arsh(r, shift) else Insn.rsh(r, shift)); } /// A register of the probed thread, from the context in r6. fn probed(p: *Program, to: Insn.Reg, r: anytype) void { p.add(Insn.ldx(.double_word, to, .r6, @as(i16, r.slot) * 8)); p.extend(to, r.bytes, false); } /// One argument of a USDT probe into the u64 at `to`: what /// `usdt.Arg.fetch` computes, in the kernel, at the probe. Memory that /// cannot be read gives 0. fn usdtArg(p: *Program, a: usdt.Arg, to: i16) void { switch (a.at) { .constant => |c| { p.add(Insn.ld_dw1(.r1, @bitCast(c))); p.add(Insn.ld_dw2(@bitCast(c))); }, .register => |r| p.probed(.r1, r), .memory => |m| { p.add(Insn.st(.double_word, .r10, to, 0)); p.probed(.r3, m.base); if (m.index) |index| { p.probed(.r1, index); p.add(Insn.lsh(.r1, @as(i32, @ctz(m.scale)))); p.add(Insn.add(.r3, .r1)); } p.add(Insn.ld_dw1(.r1, @bitCast(m.disp))); p.add(Insn.ld_dw2(@bitCast(m.disp))); p.add(Insn.add(.r3, .r1)); p.add(Insn.mov(.r1, .r10)); p.add(Insn.add(.r1, to)); p.add(Insn.mov(.r2, @as(i32, a.size))); p.add(Insn.call(.probe_read_user)); p.add(Insn.ldx(.double_word, .r1, .r10, to)); }, } p.extend(.r1, a.size, a.signed); p.add(Insn.stx(.double_word, .r10, to, .r1)); } /// Sends the `size` bytes at `at`, counts the record in `drops` when the /// ring is full, and ends the program. fn send(p: *Program, maps: Maps, at: i16, size: i32) void { p.add(Insn.ld_map_fd1(.r1, maps.ring)); p.add(Insn.ld_map_fd2(maps.ring)); p.add(Insn.mov(.r2, .r10)); p.add(Insn.add(.r2, at)); p.add(Insn.mov(.r3, size)); p.add(Insn.mov(.r4, 0)); p.add(Insn.call(.ringbuf_output)); p.add(Insn.jeq(.r0, 0, 9)); p.add(Insn.st(.word, .r10, key_at, 0)); p.add(Insn.ld_map_fd1(.r1, maps.drops)); p.add(Insn.ld_map_fd2(maps.drops)); p.add(Insn.mov(.r2, .r10)); p.add(Insn.add(.r2, key_at)); p.add(Insn.call(.map_lookup_elem)); p.add(Insn.jeq(.r0, 0, 2)); p.add(Insn.mov(.r1, 1)); p.add(Insn.xadd(.r0, .r1)); if (p.leave) |at_jump| p.insns[at_jump].off = @intCast(p.len - at_jump - 1); p.add(Insn.mov(.r0, 0)); p.add(Insn.exit()); } }; /// The maps every program writes to: the ring of records, and an array of /// one u64 counting the records that found it full. pub const Maps = struct { ring: linux.fd_t, drops: linux.fd_t }; /// raw tracepoint sys_enter(struct pt_regs *regs, long id): a /// `syscall_enter`. pub fn sysEnter(maps: Maps, tgid: u32) Program { var p: Program = .{}; // The registers are read to below the event, for the six that are arguments. p.enter(tgid); p.add(Insn.ldx(.double_word, .r3, .r6, 0)); p.read(large_at, pt_regs_bytes); p.stamp(small_at, .syscall_enter); p.add(Insn.ldx(.double_word, .r1, .r6, 8)); p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); inline for (.{ "rdi", "rsi", "rdx", "r10", "r8", "r9" }, 0..) |name, i| { p.add(Insn.ldx(.double_word, .r1, .r10, large_at + reg(name))); p.add(Insn.stx(.double_word, .r10, small_at + arg(i), .r1)); } p.send(maps, small_at, @sizeOf(Event)); return p; } /// raw tracepoint sys_exit(struct pt_regs *regs, long ret): a `syscall_exit`. pub fn sysExit(maps: Maps, tgid: u32) Program { var p: Program = .{}; const number_at = small_at - 8; p.enter(tgid); p.add(Insn.ldx(.double_word, .r3, .r6, 0)); p.add(Insn.add(.r3, reg("orig_rax"))); p.read(number_at, 8); p.stamp(small_at, .syscall_exit); p.add(Insn.ldx(.double_word, .r1, .r10, number_at)); p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); p.add(Insn.ldx(.double_word, .r1, .r6, 8)); p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r1)); for (1..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0)); p.send(maps, small_at, @sizeOf(Event)); return p; } /// raw tracepoint sched_switch(bool preempt, struct task_struct *prev, /// *next): a `sched` when either is a thread of the process. `pid_at` and /// `tgid_at` are where those fields are in `task_struct`. pub fn schedSwitch(maps: Maps, tgid: u32, pid_at: u32, tgid_at: u32) Program { var p: Program = .{}; // prev's pid and tgid, next's pid and tgid, as u32 below the event. const ids_at = small_at - 16; p.add(Insn.mov(.r6, .r1)); for ([_]i16{ 8, 16 }, 0..) |task, t| for ([_]u32{ pid_at, tgid_at }, 0..) |at, i| { p.add(Insn.ldx(.double_word, .r3, .r6, task)); p.add(Insn.add(.r3, @as(i32, @intCast(at)))); p.read(ids_at + @as(i16, @intCast(8 * t + 4 * i)), 4); }; p.add(Insn.ldx(.word, .r1, .r10, ids_at + 4)); p.add(Insn.jeq(.r1, @as(i32, @intCast(tgid)), 2)); p.add(Insn.ldx(.word, .r1, .r10, ids_at + 12)); p.leave = p.len; p.add(Insn.jne(.r1, @as(i32, @intCast(tgid)), 0)); p.add(Insn.ldx(.word, .r7, .r10, ids_at)); p.stamp(small_at, .sched); p.add(Insn.ldx(.double_word, .r1, .r6, 0)); p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r7)); p.add(Insn.ldx(.word, .r1, .r10, ids_at + 8)); p.add(Insn.stx(.double_word, .r10, small_at + arg(1), .r1)); p.add(Insn.call(.get_smp_processor_id)); p.add(Insn.stx(.double_word, .r10, small_at + arg(2), .r0)); for (3..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0)); p.send(maps, small_at, @sizeOf(Event)); return p; } /// A kprobe or uprobe (the context is the registers there): a `probe` /// named `id`. Its a1..a5 are the first five arguments of a function call, /// or with `args` those of a USDT probe, read there and then. In `user` /// space the registers follow the event: they are the thread's, which a /// kernel function's are not. pub fn probe(maps: Maps, tgid: u32, id: u64, args: ?[]const usdt.Arg, user: bool) Program { var p: Program = .{}; const at: i16 = if (user) large_at else small_at; p.enter(tgid); p.stamp(at, .probe); p.add(Insn.st(.half_word, .r10, at + field("aux"), 0)); p.add(Insn.ld_dw1(.r1, id)); p.add(Insn.ld_dw2(id)); p.add(Insn.stx(.double_word, .r10, at + arg(0), .r1)); inline for (.{ "rdi", "rsi", "rdx", "rcx", "r8" }, 1..) |name, i| { if (args == null) { p.add(Insn.ldx(.double_word, .r1, .r6, reg(name))); p.add(Insn.stx(.double_word, .r10, at + arg(i), .r1)); } else if (i <= args.?.len) { p.usdtArg(args.?[i - 1], at + arg(i)); } else p.add(Insn.st(.double_word, .r10, at + arg(i), 0)); } if (user) for (0..pt_regs_bytes / 8) |i| { p.add(Insn.ldx(.double_word, .r1, .r6, @intCast(8 * i))); p.add(Insn.stx(.double_word, .r10, at + @sizeOf(Event) + @as(i16, @intCast(8 * i)), .r1)); }; p.send(maps, at, if (user) @sizeOf(Event) + pt_regs_bytes else @sizeOf(Event)); return p; } /// `probe` for a USDT probe with the arguments `specs` ("-4@%edi /// 8@-24(%rbp)"): the first five become a1..a5. pub fn usdtProgram(maps: Maps, tgid: u32, id: u64, specs: []const u8) !Program { var args: [5]usdt.Arg = undefined; var count: usize = 0; var each = std.mem.tokenizeScalar(u8, specs, ' '); while (each.next()) |spec| { if (count == args.len) break; args[count] = try .parse(spec); count += 1; } return probe(maps, tgid, id, args[0..count], true); } /// Where `member` is in the struct `name`, in bytes, from the kernel's /// type information (the contents of /sys/kernel/btf/vmlinux). pub fn fieldOffset(btf: []const u8, name: []const u8, member: []const u8) ?u32 { const T = BPF.btf; if (btf.len < @sizeOf(T.Header)) return null; const header = std.mem.bytesToValue(T.Header, btf[0..@sizeOf(T.Header)]); if (header.magic != T.magic or header.hdr_len + @as(u64, header.str_off) + header.str_len > btf.len) return null; const types = btf[header.hdr_len + header.type_off ..][0..header.type_len]; const strings = btf[header.hdr_len + header.str_off ..][0..header.str_len]; var at: usize = 0; while (at + @sizeOf(T.Type) <= types.len) { const t = std.mem.bytesToValue(T.Type, types[at..][0..@sizeOf(T.Type)]); at += @sizeOf(T.Type); const count: usize = t.info.vlen; // What follows the type, by kind. const extra: usize = switch (t.info.kind) { .int, .@"var", .decl_tag => 4, .array => 12, .@"struct", .@"union", .datasec, .enum64 => 12 * count, .@"enum", .func_proto => 8 * count, else => 0, }; if (at + extra > types.len) return null; defer at += extra; if (t.info.kind != .@"struct" or !std.mem.eql(u8, std.mem.sliceTo(strings[@min(t.name_off, strings.len)..], 0), name)) continue; for (0..count) |i| { const m = std.mem.bytesToValue(T.Member, types[at + 12 * i ..][0..12]); if (std.mem.eql(u8, std.mem.sliceTo(strings[@min(m.name_off, strings.len)..], 0), member)) return m.offset.bit / 8; } } return null; } /// The reading end of a BPF ring buffer: a page with our position, a page /// with the kernel's, and the data, which the kernel maps twice in a row so /// that no record wraps. A record is a u32 length with two flag bits, four /// bytes more, the bytes, padded to 8. pub const RingBuf = struct { consumer: *u64, producer: *const u64, data: []const u8, /// Read so far; the kernel is told by `commit`, after the records were used. at: u64 = 0, const busy: u32 = 1 << 31; const discarded: u32 = 1 << 30; pub fn map(fd: linux.fd_t, size: usize) !RingBuf { const ours = linux.mmap(null, 4096, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0); if (linux.errno(ours) != .SUCCESS) return error.BpfRingMap; const theirs = linux.mmap(null, 4096 + 2 * size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 4096); if (linux.errno(theirs) != .SUCCESS) return error.BpfRingMap; return .{ .consumer = @ptrFromInt(ours), .producer = @ptrFromInt(theirs), .data = @as([*]const u8, @ptrFromInt(theirs + 4096))[0..size] }; } /// The next finished record, valid until `commit`. pub fn next(r: *RingBuf) ?[]const u8 { while (r.at < @atomicLoad(u64, r.producer, .acquire)) { const start: usize = @intCast(r.at & (r.data.len - 1)); const header = @atomicLoad(u32, @as(*const u32, @ptrCast(@alignCast(r.data.ptr + start))), .acquire); if (header & busy != 0) return null; const len = header & ~(busy | discarded); r.at += std.mem.alignForward(u64, len + 8, 8); if (header & discarded == 0) return (r.data.ptr + start + 8)[0..len]; } return null; } pub fn commit(r: *RingBuf) void { @atomicStore(u64, r.consumer, r.at, .release); } }; /// The capability bits of this process (CapEff of /proc/self/status). fn capabilities() u64 { var buf: [4096]u8 = undefined; const fd = linux.open("/proc/self/status", .{}, 0); if (linux.errno(fd) != .SUCCESS) return 0; defer _ = linux.close(@intCast(fd)); const n = linux.read(@intCast(fd), &buf, buf.len); if (linux.errno(n) != .SUCCESS) return 0; const label = "CapEff:\t"; const at = std.mem.indexOf(u8, buf[0..n], label) orelse return 0; return std.fmt.parseInt(u64, buf[at + label.len ..][0..16], 16) catch 0; } /// What this process lacks to trace with BPF, said into `buf`; null when /// it has the capabilities (which inside a user namespace still is not /// enough: the kernel wants them in the initial one). pub fn missing(buf: []u8) ?[]const u8 { const caps = capabilities(); const has = struct { fn has(all: u64, cap: u6) bool { return all >> cap & 1 != 0; } }.has; if (has(caps, linux.CAP.SYS_ADMIN) or (has(caps, linux.CAP.BPF) and has(caps, linux.CAP.PERFMON))) return null; return std.fmt.bufPrint(buf, "missing {s}{s}{s} (effective capabilities {x:0>16}, uid {d}). " ++ "CAP_BPF is what bpf() wants to create maps and load programs, CAP_PERFMON to load tracing programs and open kprobes and uprobes; CAP_SYS_ADMIN stands for both. " ++ "Run as root, or give the binary the two: setcap cap_bpf,cap_perfmon+ep ztrace", .{ if (has(caps, linux.CAP.BPF)) "" else "CAP_BPF", if (has(caps, linux.CAP.BPF) or has(caps, linux.CAP.PERFMON)) "" else " and ", if (has(caps, linux.CAP.PERFMON)) "" else "CAP_PERFMON", caps, linux.getuid(), }) catch null; } pub const Tracer = struct { gpa: std.mem.Allocator, /// The process traced: its thread group id, in the initial pid namespace. /// shortcut: one process, and a tracer in a pid namespace of its own sees nothing; keep the ids in a BPF map to follow children. pid: linux.pid_t, sink: event.Sink, maps: Maps, ring: RingBuf, /// Programs and what attaches them. fds: std.ArrayList(linux.fd_t) = .empty, /// What the verifier said about the last program, when it refused one. log: []u8, pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, sink: event.Sink, ring_bytes: u32) !Tracer { const ring = try BPF.map_create(.ringbuf, 0, 0, ring_bytes); errdefer _ = linux.close(ring); const drops = try BPF.map_create(.array, 4, 8, 1); errdefer _ = linux.close(drops); const ring_map = try RingBuf.map(ring, ring_bytes); const log = try gpa.alloc(u8, 1 << 16); log[0] = 0; return .{ .gpa = gpa, .pid = pid, .sink = sink, .maps = .{ .ring = ring, .drops = drops }, .ring = ring_map, .log = log }; } pub fn close(t: *Tracer) void { for (t.fds.items) |fd| _ = linux.close(fd); t.fds.deinit(t.gpa); _ = linux.munmap(@ptrCast(@alignCast(t.ring.consumer)), 4096); _ = linux.munmap(@ptrCast(@alignCast(t.ring.producer)), 4096 + 2 * t.ring.data.len); _ = linux.close(t.maps.ring); _ = linux.close(t.maps.drops); t.gpa.free(t.log); } fn load(t: *Tracer, kind: BPF.ProgType, program: Program) !linux.fd_t { // GPL: bpf_probe_read_kernel is for programs that say so. const fd = BPF.prog_load(kind, program.code(), null, "GPL", 0, 0) catch |err| { // Again for the verifier's reasons, which a good program need not pay for. var log: BPF.Log = .{ .level = 1, .buf = t.log }; if (BPF.prog_load(kind, program.code(), &log, "GPL", 0, 0)) |again| _ = linux.close(again) else |_| {} return err; }; try t.fds.append(t.gpa, fd); return fd; } fn rawTracepoint(t: *Tracer, name: [*:0]const u8, program: Program) !void { var attr = std.mem.zeroes(BPF.Attr); attr.raw_tracepoint = .{ .name = @intFromPtr(name), .prog_fd = try t.load(.raw_tracepoint, program) }; const rc = linux.bpf(.raw_tracepoint_open, &attr, @sizeOf(BPF.RawTracepointAttr)); if (linux.errno(rc) != .SUCCESS) return error.BpfAttach; try t.fds.append(t.gpa, @intCast(rc)); } /// Every syscall of the process. pub fn syscalls(t: *Tracer) !void { try t.rawTracepoint("sys_enter", sysEnter(t.maps, @intCast(t.pid))); try t.rawTracepoint("sys_exit", sysExit(t.maps, @intCast(t.pid))); } /// Every time one of its threads leaves or gets a CPU. `btf` is the /// contents of /sys/kernel/btf/vmlinux. pub fn sched(t: *Tracer, btf: []const u8) !void { const pid_at = fieldOffset(btf, "task_struct", "pid") orelse return error.NoBtf; const tgid_at = fieldOffset(btf, "task_struct", "tgid") orelse return error.NoBtf; try t.rawTracepoint("sched_switch", schedSwitch(t.maps, @intCast(t.pid), pid_at, tgid_at)); } /// A probe through the perf event of the kprobe or uprobe PMU: `target` /// is a kernel function or a file, `offset` is into it. fn attach(t: *Tracer, pmu: []const u8, target: [*:0]const u8, offset: u64, semaphore: u64, program: Program) !void { var path_buf: [64]u8 = undefined; var number: [16]u8 = undefined; const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/sys/bus/event_source/devices/{s}/type", .{pmu}, 0), .{}, 0); if (linux.errno(fd) != .SUCCESS) return error.NoProbeSupport; defer _ = linux.close(@intCast(fd)); const n = linux.read(@intCast(fd), &number, number.len); if (linux.errno(n) != .SUCCESS) return error.NoProbeSupport; const kind = std.fmt.parseInt(u32, std.mem.trim(u8, number[0..n], "\n "), 10) catch return error.NoProbeSupport; // A uprobe's semaphore (the offset in the file of its counter) rides in the upper half of config. // On the first thread alone, and not inherited: a program on a probe // runs for whoever hits it (hence the process id in ours), and the // kernel reads `target` again, from the target's memory, for an // event a clone would inherit, which fails the clone. var attr: linux.perf_event_attr = .{ .type = @fromBackingInt(@intCast(kind)), .config = semaphore << 32, .config1 = @intFromPtr(target), .config2 = offset }; const event_fd = linux.perf_event_open(&attr, t.pid, -1, -1, linux.PERF.FLAG.FD_CLOEXEC); if (linux.errno(event_fd) != .SUCCESS) return if (linux.errno(event_fd) == .NOENT) error.NoSuchProbe else error.BpfAttach; try t.fds.append(t.gpa, @intCast(event_fd)); const loaded = try t.load(.kprobe, program); if (linux.errno(linux.ioctl(@intCast(event_fd), linux.PERF.EVENT_IOC.SET_BPF, @intCast(loaded))) != .SUCCESS) return error.BpfAttach; } /// Every call of a kernel function by the process: a `probe` named /// `function` (define the string yourself: its id is `id`). pub fn kprobe(t: *Tracer, function: [*:0]const u8, id: u64) !void { try t.attach("kprobe", function, 0, 0, probe(t.maps, @intCast(t.pid), id, null, false)); } /// Every time the process runs the instruction at `offset` of the /// file `path`. pub fn uprobe(t: *Tracer, path: [*:0]const u8, offset: u64, id: u64) !void { try t.attach("uprobe", path, offset, 0, probe(t.maps, @intCast(t.pid), id, null, true)); } /// A static probe of the file `path` (`p` from `usdt.probes` with no /// bias). The kernel raises its semaphore in every process that maps /// the file, while we are attached; where the same page of the file is /// mapped writable twice it raises the wrong word, so for a target at /// hand write the semaphore yourself and pass the probe without one. pub fn usdtProbe(t: *Tracer, elf: usdt.Elf, path: [*:0]const u8, p: usdt.Probe, id: u64) !void { const semaphore = if (p.semaphore == 0) 0 else elf.offset(p.semaphore) orelse return error.NoSuchProbe; try t.attach("uprobe", path, elf.offset(p.pc) orelse return error.NoSuchProbe, semaphore, try usdtProgram(t.maps, @intCast(t.pid), id, p.args)); } /// Waits up to the timeout for records and decodes them into the sink. /// Returns how many. pub fn poll(t: *Tracer, timeout_ms: i32) !usize { var fds = [1]linux.pollfd{.{ .fd = t.maps.ring, .events = linux.POLL.IN, .revents = 0 }}; _ = linux.poll(&fds, 1, timeout_ms); var n: usize = 0; while (t.ring.next()) |record| : (n += 1) try decode(record, t.sink); t.ring.commit(); return n; } /// Records that found the ring full. pub fn lost(t: *Tracer) u64 { var count: u64 = 0; BPF.map_lookup_elem(t.maps.drops, &std.mem.toBytes(@as(u32, 0)), std.mem.asBytes(&count)) catch return 0; return count; } }; /// A record of the ring into the sink: the event, and the registers when /// they follow it. fn decode(record: []const u8, sink: event.Sink) !void { if (record.len < @sizeOf(Event)) return; const ev = std.mem.bytesToValue(Event, record[0..@sizeOf(Event)]); if (record.len < @sizeOf(Event) + pt_regs_bytes) return sink.emit(ev, ""); var regs: Regs = .{}; @memcpy(std.mem.asBytes(®s)[0..pt_regs_bytes], record[@sizeOf(Event)..][0..pt_regs_bytes]); try sink.emit(ev, std.mem.asBytes(®s)); } // What follows runs the programs without a kernel: enough of a BPF machine // for what they use, on this process's own memory. const Machine = struct { pid_tgid: u64 = 0, now: u64 = 0, cpu: u64 = 0, /// What bpf_ringbuf_output returns; not 0 is a full ring. ring_full: bool = false, sent: [512]u8 = undefined, sent_len: ?usize = null, drops: u64 = 0, fn run(m: *Machine, code: []const Insn, context: *const anyopaque) !void { var stack: [512]u8 align(8) = undefined; var r: [11]u64 = @splat(0); r[1] = @intFromPtr(context); r[10] = @intFromPtr(&stack) + stack.len; var pc: usize = 0; var steps: usize = 0; while (true) : (steps += 1) { if (steps > 10_000 or pc >= code.len) return error.RanAway; const insn = code[pc]; pc += 1; const imm: u64 = @bitCast(@as(i64, insn.imm)); const source = if (insn.code & BPF.X != 0) r[insn.src] else imm; const address = (if (insn.code & 7 == BPF.LDX) r[insn.src] else r[insn.dst]) +% @as(u64, @bitCast(@as(i64, insn.off))); switch (insn.code) { BPF.ALU64 | BPF.MOV | BPF.K, BPF.ALU64 | BPF.MOV | BPF.X => r[insn.dst] = source, BPF.ALU64 | BPF.ADD | BPF.K, BPF.ALU64 | BPF.ADD | BPF.X => r[insn.dst] +%= source, BPF.ALU64 | BPF.RSH | BPF.K => r[insn.dst] >>= @intCast(insn.imm), BPF.ALU64 | BPF.LSH | BPF.K => r[insn.dst] <<= @intCast(insn.imm), BPF.ALU64 | BPF.ARSH | BPF.K => r[insn.dst] = @bitCast(@as(i64, @bitCast(r[insn.dst])) >> @intCast(insn.imm)), BPF.JMP | BPF.JEQ | BPF.K => if (r[insn.dst] == imm) { pc = @intCast(@as(i64, @intCast(pc)) + insn.off); }, BPF.JMP | BPF.JNE | BPF.K => if (r[insn.dst] != imm) { pc = @intCast(@as(i64, @intCast(pc)) + insn.off); }, BPF.LD | BPF.DW | BPF.IMM => { r[insn.dst] = @as(u32, @bitCast(insn.imm)) | @as(u64, @as(u32, @bitCast(code[pc].imm))) << 32; pc += 1; }, inline BPF.LDX | BPF.MEM | BPF.B, BPF.LDX | BPF.MEM | BPF.H, BPF.LDX | BPF.MEM | BPF.W, BPF.LDX | BPF.MEM | BPF.DW => |code_now| { r[insn.dst] = @as(*align(1) const Width(code_now), @ptrFromInt(address)).*; }, inline BPF.STX | BPF.MEM | BPF.H, BPF.STX | BPF.MEM | BPF.W, BPF.STX | BPF.MEM | BPF.DW => |code_now| { @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(r[insn.src]); }, inline BPF.ST | BPF.MEM | BPF.H, BPF.ST | BPF.MEM | BPF.W, BPF.ST | BPF.MEM | BPF.DW => |code_now| { @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(imm); }, BPF.STX | BPF.XADD | BPF.DW => @as(*u64, @ptrFromInt(address)).* += r[insn.src], BPF.JMP | BPF.CALL => r[0] = switch (@as(BPF.Helper, @fromBackingInt(@intCast(insn.imm)))) { .get_current_pid_tgid => m.pid_tgid, .ktime_get_ns => m.now, .get_smp_processor_id => m.cpu, .probe_read_kernel, .probe_read_user => read: { @memcpy(@as([*]u8, @ptrFromInt(r[1]))[0..r[2]], @as([*]const u8, @ptrFromInt(r[3]))[0..r[2]]); break :read 0; }, .ringbuf_output => out: { if (r[1] != 3 or m.ring_full) break :out @as(u64, @bitCast(@as(i64, -11))); m.sent_len = r[3]; @memcpy(m.sent[0..r[3]], @as([*]const u8, @ptrFromInt(r[2]))[0..r[3]]); break :out 0; }, .map_lookup_elem => if (r[1] == 4 and @as(*const u32, @ptrFromInt(r[2])).* == 0) @intFromPtr(&m.drops) else 0, else => return error.UnknownHelper, }, BPF.JMP | BPF.EXIT => return, else => return error.UnknownInstruction, } } } fn Width(comptime code: u8) type { return switch (code & 0x18) { BPF.B => u8, BPF.H => u16, BPF.W => u32, else => u64, }; } fn sentEvent(m: *const Machine) Event { return std.mem.bytesToValue(Event, m.sent[0..@sizeOf(Event)]); } }; const test_maps: Maps = .{ .ring = 3, .drops = 4 }; test "the syscall programs: the right process only, the event the tracer expects" { var regs: Regs = .{ .rdi = 1, .rsi = 2, .rdx = 3, .r10 = 4, .r8 = 5, .r9 = 6, .orig_rax = 257, .rip = 0x401000 }; const enter_context = [2]u64{ @intFromPtr(®s), 257 }; var m: Machine = .{ .pid_tgid = 1000 << 32 | 1001, .now = 77 }; try m.run(sysEnter(test_maps, 1000).code(), &enter_context); try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len); try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_enter, .aux = 257, .a = .{ 1, 2, 3, 4, 5, 6 } }, m.sentEvent()); const exit_context = [2]u64{ @intFromPtr(®s), @bitCast(@as(i64, -2)) }; try m.run(sysExit(test_maps, 1000).code(), &exit_context); try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len); try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_exit, .aux = 257, .a = .{ @bitCast(@as(i64, -2)), 0, 0, 0, 0, 0 } }, m.sentEvent()); // Another process: nothing. A full ring: counted. m.sent_len = null; try m.run(sysEnter(test_maps, 999).code(), &enter_context); try std.testing.expectEqual(@as(?usize, null), m.sent_len); m.ring_full = true; try m.run(sysExit(test_maps, 1000).code(), &exit_context); try m.run(sysEnter(test_maps, 1000).code(), &enter_context); try std.testing.expectEqual(@as(u64, 2), m.drops); } test "the sched_switch program: either side of the switch, ids from where BTF says" { // Two task_structs with the ids at offsets of our choosing. var prev: [64]u32 = @splat(0); var next: [64]u32 = @splat(0); prev[10] = 1001; prev[11] = 1000; next[10] = 55; next[11] = 50; const context = [3]u64{ 1, @intFromPtr(&prev), @intFromPtr(&next) }; var m: Machine = .{ .now = 9, .cpu = 3 }; try m.run(schedSwitch(test_maps, 1000, 40, 44).code(), &context); try std.testing.expectEqual(Event{ .ts = 9, .tid = 1001, .kind = .sched, .aux = 1, .a = .{ 1001, 55, 3, 0, 0, 0 } }, m.sentEvent()); m.sent_len = null; try m.run(schedSwitch(test_maps, 50, 40, 44).code(), &context); try std.testing.expectEqual(@as(u64, 55), m.sentEvent().a[1]); m.sent_len = null; try m.run(schedSwitch(test_maps, 7, 40, 44).code(), &context); try std.testing.expectEqual(@as(?usize, null), m.sent_len); } test "the probe program and its record decoded; a USDT probe's arguments as usdt.Arg.fetch has them" { // The probed thread's memory is ours: rbp and rdx point into this. var memory: [64]u8 = undefined; for (&memory, 0..) |*b, i| b.* = @truncate(0xf0 + i); const regs: Regs = .{ .rdi = 11, .rsi = 0xffff_fffe, .rdx = @intFromPtr(&memory), .rcx = 3, .r8 = 15, .rbp = @intFromPtr(&memory) + 32, .rip = 0x401234, .rsp = 0x7000, .r10 = 0x1_0000_00ff }; var m: Machine = .{ .pid_tgid = 1000 << 32 | 1000, .now = 5 }; // In the kernel: the event alone. In user space: the registers too. try m.run(probe(test_maps, 1000, 0xfeed, null, false).code(), ®s); try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len); try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent()); try m.run(probe(test_maps, 1000, 0xfeed, null, true).code(), ®s); try std.testing.expectEqual(@as(?usize, @sizeOf(Event) + pt_regs_bytes), m.sent_len); try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent()); // A sink that keeps the last event. const Last = struct { ev: Event = undefined, regs: Regs = undefined, fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void { const l: *@This() = @ptrCast(@alignCast(ctx)); l.ev = ev.*; l.regs = std.mem.bytesToValue(Regs, payload); } }; var last: Last = .{}; try decode(m.sent[0..m.sent_len.?], .{ .ctx = &last, .emitFn = Last.emit }); try std.testing.expectEqual(@as(u64, 11), last.ev.a[1]); try std.testing.expectEqual(regs.rip, last.regs.rip); try std.testing.expectEqual(@as(u64, 0), last.regs.fs_base); const here = struct { pub fn read(_: @This(), addr: u64, buf: []u8) usize { @memcpy(buf, @as([*]const u8, @ptrFromInt(addr))); return buf.len; } }{}; const specs = [_][5][]const u8{ .{ "-4@%esi", "4@%esi", "1@-16(%rbp)", "8@$9", "-8@$-9" }, .{ "-1@8(%rdx,%rcx,8)", "2@(%rdx)", "-2@-32(%rbp)", "1@%r10b", "8@%r10" }, }; for (specs) |five| { var args: [5]usdt.Arg = undefined; for (&args, five) |*a, spec| a.* = try .parse(spec); try m.run(probe(test_maps, 1000, 0xfeed, &args, true).code(), ®s); for (args, m.sentEvent().a[1..]) |a, got| try std.testing.expectEqual(a.fetch(®s, here).?, got); } // Fewer than five: the rest is 0. try m.run(probe(test_maps, 1000, 0xfeed, &.{try .parse("4@%edi")}, true).code(), ®s); try std.testing.expectEqual([6]u64{ 0xfeed, 11, 0, 0, 0, 0 }, m.sentEvent().a); } test "a ring buffer as the kernel lays it out: finished records only, discarded ones skipped" { var consumer: u64 = 0; var producer: u64 = 0; var data: [64]u8 align(8) = @splat(0); var ring: RingBuf = .{ .consumer = &consumer, .producer = &producer, .data = &data }; const records = [_]struct { u32, []const u8 }{ .{ 5, "hello" }, .{ 3 | RingBuf.discarded, "no!" }, .{ 2, "ok" }, .{ 4 | RingBuf.busy, "wait" } }; for (records) |record| { std.mem.writeInt(u32, data[producer..][0..4], record[0], .little); @memcpy(data[producer + 8 ..][0..record[1].len], record[1]); producer += std.mem.alignForward(u64, record[1].len + 8, 8); } try std.testing.expectEqualStrings("hello", ring.next().?); try std.testing.expectEqualStrings("ok", ring.next().?); try std.testing.expect(ring.next() == null); ring.commit(); try std.testing.expectEqual(@as(u64, 48), consumer); // The producer finishes it. std.mem.writeInt(u32, data[48..52], 4, .little); try std.testing.expectEqualStrings("wait", ring.next().?); } test "task_struct's pid and tgid in this kernel's BTF; privilege is told apart" { var buf: [512]u8 = undefined; if (missing(&buf)) |why| try std.testing.expect(std.mem.indexOf(u8, why, "CAP_") != null); const btf = std.Io.Dir.cwd().readFileAlloc(std.testing.io, "/sys/kernel/btf/vmlinux", std.testing.allocator, .unlimited) catch return error.SkipZigTest; defer std.testing.allocator.free(btf); const pid_at = fieldOffset(btf, "task_struct", "pid").?; // Next to each other, some way into a struct of a few kilobytes. try std.testing.expectEqual(pid_at + 4, fieldOffset(btf, "task_struct", "tgid").?); try std.testing.expect(pid_at > 256 and pid_at < 16384); try std.testing.expect(fieldOffset(btf, "task_struct", "no_such_field") == null); try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "rdi")), fieldOffset(btf, "pt_regs", "di")); try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "orig_rax")), fieldOffset(btf, "pt_regs", "orig_ax")); }