diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/cli/main.zig | 2 | ||||
| -rw-r--r-- | src/cli/perf.zig | 60 | ||||
| -rw-r--r-- | src/cli/probe.zig | 215 | ||||
| -rw-r--r-- | src/trace/bpf.zig | 733 | ||||
| -rw-r--r-- | src/trace/perf.zig | 77 | ||||
| -rw-r--r-- | src/trace/root.zig | 3 | ||||
| -rw-r--r-- | src/trace/usdt.zig | 3 |
7 files changed, 1080 insertions, 13 deletions
diff --git a/src/cli/main.zig b/src/cli/main.zig index 9e6b0e5..554efde 100644 --- a/src/cli/main.zig +++ b/src/cli/main.zig @@ -8,6 +8,7 @@ const commands = .{ .dbg = @import("dbg.zig"), .run = @import("run.zig"), .perf = @import("perf.zig"), + .probe = @import("probe.zig"), .dump = @import("dump.zig"), .show = @import("show.zig"), .cc = @import("cc.zig"), @@ -20,6 +21,7 @@ const usage = \\ dbg debug a program, recording it: step, break, look, go back \\ run record a program that carries a ring producer (agent, instrumentation) \\ perf record what the kernel sees of a program at full speed (perf events) + \\ probe ... with BPF programs on syscalls, scheduling, kprobes, uprobes, USDT (root) \\ dump print a recording's events \\ show print a recording's summary, or the program's state at a moment \\ cc compile a program with instrumentation that records it diff --git a/src/cli/perf.zig b/src/cli/perf.zig index 57ad11a..58f7d77 100644 --- a/src/cli/perf.zig +++ b/src/cli/perf.zig @@ -30,10 +30,15 @@ pub const usage = \\ size) with the value found there; with :rw every access \\ --break WHAT ... on code: every time it runs, with the registers \\ --threads also the threads and children it creates + \\ --cut end a moment at every fault and every hit of --watch and + \\ --break: the state at each can be gone back to, for more bytes + \\ --max-size BYTES stop when the recording is this large (default 1 GiB) \\ --ring BYTES the kernel's buffer, per CPU (a power of two, default 512 KiB) \\ \\There are four debug registers. Only user space is seen: a page the \\kernel touches first (a read() into fresh memory) faults unseen. + \\Moments end where the recorder ends them anyway (a syscall's exit, a + \\megabyte of events), so that a profile or a storm of faults stays small. \\ ; @@ -44,6 +49,8 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { var out: []const u8 = "out.ztrace"; var options: zt.perf.Options = .{ .counters = &.{ .instructions, .cycles } }; var interval_ms: u32 = 0; + var cut = false; + var max_bytes: ?u64 = null; var specs: std.ArrayList(Spec) = .empty; var i: usize = 0; while (i < args.len) : (i += 1) { @@ -62,6 +69,8 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { options.syscalls = true; } else if (std.mem.eql(u8, arg, "--threads")) { options.follow = true; + } else if (std.mem.eql(u8, arg, "--cut")) { + cut = true; } else if (value == null) { fail(usage, .{}); } else if (std.mem.eql(u8, arg, "-o")) { @@ -79,6 +88,9 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { } else if (std.mem.eql(u8, arg, "--sample")) { options.sample_hz = std.fmt.parseInt(u32, value.?, 0) catch fail("--sample needs a rate\n", .{}); i += 1; + } else if (std.mem.eql(u8, arg, "--max-size")) { + max_bytes = std.fmt.parseInt(u64, value.?, 0) catch fail("--max-size needs bytes\n", .{}); + i += 1; } else if (std.mem.eql(u8, arg, "--ring")) { options.ring_bytes = std.fmt.parseInt(usize, value.?, 0) catch 0; if (!std.math.isPowerOfTwo(options.ring_bytes) or options.ring_bytes < 4096) fail("--ring needs a power of two, 4096 or more\n", .{}); @@ -105,8 +117,9 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { var proc = try zt.ptrace.Process.spawn(path, argv, init.minimal.environ.block.slice.ptr); errdefer proc.kill(); var rec = try zt.Recorder.create(init.gpa, init.io, out); + if (max_bytes) |bytes| rec.max_bytes = bytes; // What changes the state a moment holds (registers, memory) ends one. - rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit(); + if (cut) rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit(); var session = zt.perf.Session.open(init.gpa, proc.pid, options, rec.sink()) catch |err| switch (err) { error.PerfAccess => fail("perf_event_open was refused: kernel.perf_event_paranoid is above 2, or a security module forbids it\n", .{}), error.PerfNoSuchEvent => fail("perf_event_open: this CPU or kernel lacks one of the events asked for (in a VM without a PMU, try -e task-clock; --sample takes at most kernel.perf_event_max_sample_rate)\n", .{}), @@ -133,7 +146,19 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { zt.perf.release(proc); const start = zt.event.now(); - var next_read = start; + const exit_code: ?u64 = pump(&session, &rec, interval_ms) catch |err| switch (err) { + error.RecordingFull => null, + else => |e| return e, + }; + if (exit_code == null) proc.kill(); + const ns = zt.event.now() - start; + try finish(init, &rec, out, exit_code, session.lost, ns, options.counters, session.counts); +} + +/// Decodes until the target ends; its exit code. `error.RecordingFull` +/// when the recording reaches its limit first. +fn pump(session: *zt.perf.Session, rec: *zt.Recorder, interval_ms: u32) !u64 { + var next_read = zt.event.now(); while (true) { var wait_ms: i32 = 100; if (interval_ms != 0) { @@ -147,22 +172,33 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { if (!try session.poll(wait_ms)) break; } try session.readCounters(); + return exited(session.pid, rec.sink()); +} + +/// Waits for the target's end and records it. The exit code. +pub fn exited(pid: linux.pid_t, sink: zt.Sink) !u64 { var status: u32 = 0; - _ = linux.waitpid(proc.pid, @ptrCast(&status), 0); - const exit_code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status))); - const ns = zt.event.now() - start; - try rec.emit(.{ .ts = zt.event.now(), .tid = @intCast(proc.pid), .kind = .exit, .a = .{ exit_code, 0, 0, 0, 0, 0 } }, ""); + _ = linux.waitpid(pid, @ptrCast(&status), 0); + const code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status))); + try sink.emit(.{ .ts = zt.event.now(), .tid = @intCast(pid), .kind = .exit, .a = .{ code, 0, 0, 0, 0, 0 } }, ""); + return code; +} +/// Closes the recording and says what it holds; `exit_code` is null when +/// it filled up and the target was stopped for it. +pub fn finish(init: std.process.Init, rec: *zt.Recorder, out: []const u8, exit_code: ?u64, lost: u64, ns: u64, counters: []const zt.perf.Counter, counts: []const u64) !void { const events = rec.count; const moments = rec.moments + @intFromBool(rec.delta.items.len > 0); try rec.close(); const size = (try std.Io.Dir.cwd().statFile(init.io, out, .{})).size; - for (options.counters, session.counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c }); - std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes, {d:.1} ms, {d:.0} events/s, exit {d}\n", .{ - out, moments, - events, session.lost, - size, @as(f64, @floatFromInt(ns)) / 1e6, - @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9), exit_code, + const code = exit_code orelse fail("{s} is full at {d} bytes ({d} events) and the program was stopped: record fewer kinds of event, or allow more with --max-size BYTES\n", .{ out, size, events }); + for (counters, counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c }); + std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes ({d:.1} an event), {d:.1} ms, {d:.0} events/s, exit {d}\n", .{ + out, moments, + events, lost, + size, @as(f64, @floatFromInt(size)) / @as(f64, @floatFromInt(@max(events, 1))), + @as(f64, @floatFromInt(ns)) / 1e6, @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9), + code, }); } diff --git a/src/cli/probe.zig b/src/cli/probe.zig new file mode 100644 index 0000000..f0e2d12 --- /dev/null +++ b/src/cli/probe.zig @@ -0,0 +1,215 @@ +//! ztrace probe: BPF programs in the kernel watching a program: its +//! syscalls, its scheduling, any kernel or user function, static probes. +//! For root; --dry-run shows what it would do without being it. +const std = @import("std"); +const linux = std.os.linux; +const zt = @import("ztrace"); +const util = @import("util.zig"); +const perf = @import("perf.zig"); +const fail = util.fail; +const bpf = zt.bpf; + +pub const usage = + \\usage: ztrace probe [-o FILE] [OPTIONS] -- PROGRAM [ARGS] + \\ + \\Runs PROGRAM at full speed with BPF programs attached for it in the + \\kernel, into a recording (default out.ztrace). Needs CAP_BPF and + \\CAP_PERFMON, which root has. + \\ + \\ --syscalls every syscall, with the registers at its entry + \\ --sched every switch of a CPU to or from one of its threads + \\ --kprobe FUNCTION + \\ every call of a kernel function it makes: five arguments + \\ --uprobe [FILE:]SYMBOL + \\ every call of a function of its own, or of a library + \\ given by its file: five arguments, the registers + \\ --usdt [FILE:]PROVIDER:NAME + \\ a static probe (<sys/sdt.h>), with its own arguments + \\ --dry-run build the programs, find the places, print them; attach nothing + \\ --max-size BYTES stop when the recording is this large (default 1 GiB) + \\ --ring BYTES the ring the programs write to (a power of two, default 4 MiB) + \\ +; + +const Probe = struct { kind: enum { kprobe, uprobe, usdt }, what: [:0]const u8 }; + +pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { + const arena = init.arena.allocator(); + var out: []const u8 = "out.ztrace"; + var syscalls = false; + var sched = false; + var dry = false; + var max_bytes: ?u64 = null; + var ring_bytes: u32 = 1 << 22; + var probes: std.ArrayList(Probe) = .empty; + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + const value: ?[:0]const u8 = if (i + 1 < args.len) args[i + 1] else null; + if (std.mem.eql(u8, arg, "--")) { + i += 1; + break; + } else if (std.mem.eql(u8, arg, "--syscalls")) { + syscalls = true; + } else if (std.mem.eql(u8, arg, "--sched")) { + sched = true; + } else if (std.mem.eql(u8, arg, "--dry-run")) { + dry = true; + } else if (value == null) { + fail(usage, .{}); + } else if (std.mem.eql(u8, arg, "-o")) { + out = value.?; + i += 1; + } else if (std.mem.eql(u8, arg, "--max-size")) { + max_bytes = std.fmt.parseInt(u64, value.?, 0) catch fail("--max-size needs bytes\n", .{}); + i += 1; + } else if (std.mem.eql(u8, arg, "--ring")) { + ring_bytes = std.fmt.parseInt(u32, value.?, 0) catch 0; + if (!std.math.isPowerOfTwo(ring_bytes) or ring_bytes < 4096) fail("--ring needs a power of two, 4096 or more\n", .{}); + i += 1; + } else if (std.meta.stringToEnum(@FieldType(Probe, "kind"), arg[@min(arg.len, 2)..])) |kind| { + try probes.append(arena, .{ .kind = kind, .what = value.? }); + i += 1; + } else fail(usage, .{}); + } + if (i >= args.len) fail(usage, .{}); + const path = std.mem.span(try util.resolve(init, args[i])); + + // Everything that can be wrong without the kernel's say is found first. + var places: std.ArrayList(Place) = .empty; + for (probes.items) |p| try places.append(arena, try place(arena, p, path)); + var offsets: [2]u32 = .{ 0, 0 }; + var btf: []const u8 = ""; + if (sched) { + btf = std.Io.Dir.cwd().readFileAlloc(init.io, "/sys/kernel/btf/vmlinux", arena, .unlimited) catch fail("--sched needs the kernel's type information, and /sys/kernel/btf/vmlinux cannot be read\n", .{}); + offsets = .{ + bpf.fieldOffset(btf, "task_struct", "pid") orelse fail("no task_struct.pid in the kernel's BTF\n", .{}), + bpf.fieldOffset(btf, "task_struct", "tgid") orelse fail("no task_struct.tgid in the kernel's BTF\n", .{}), + }; + } + var why: [512]u8 = undefined; + const missing = bpf.missing(&why); + + if (dry) { + // The process id and the maps are not there yet: 0 stands for them. + const maps: bpf.Maps = .{ .ring = 0, .drops = 0 }; + std.debug.print("for the process started from {s}:\n", .{path}); + if (syscalls) { + list("raw tracepoint sys_enter: a syscall_enter and the registers", bpf.sysEnter(maps, 0)); + list("raw tracepoint sys_exit: a syscall_exit", bpf.sysExit(maps, 0)); + } + if (sched) { + std.debug.print("task_struct.pid is at {d}, .tgid at {d} (BTF)\n", .{ offsets[0], offsets[1] }); + list("raw tracepoint sched_switch: a sched", bpf.schedSwitch(maps, 0, offsets[0], offsets[1])); + } + for (places.items) |p| { + std.debug.print("{t} {s}: {s} +{x}", .{ p.kind, p.name, p.target, p.offset }); + if (p.usdt) |u| std.debug.print(", semaphore at +{x}, arguments {s}", .{ if (u.semaphore == 0) 0 else p.elf.?.offset(u.semaphore) orelse 0, u.args }); + std.debug.print("\n", .{}); + const id = zt.perf.stringId(p.name); + list("its program: a probe and the registers", if (p.usdt) |u| bpf.usdtProgram(maps, 0, id, u.args) catch fail("{s}: an argument of this probe ({s}) is in a form not understood yet\n", .{ p.name, u.args }) else bpf.probe(maps, 0, id, null)); + } + std.debug.print("{s}\n", .{if (missing) |m| m else "nothing is missing to attach them"}); + return; + } + if (missing) |m| fail("ztrace probe: {s}\n", .{m}); + + const argv = try util.targetArgv(arena, args[i..]); + var proc = try zt.ptrace.Process.spawn(path, argv, init.minimal.environ.block.slice.ptr); + errdefer proc.kill(); + var rec = try zt.Recorder.create(init.gpa, init.io, out); + if (max_bytes) |bytes| rec.max_bytes = bytes; + // Two sources, the programs' ring and perf's: one sequence in the recording. + var sorted: zt.perf.Sorted = .{ .gpa = init.gpa, .to = rec.sink() }; + defer sorted.deinit(); + var tracer = bpf.Tracer.open(init.gpa, proc.pid, sorted.sink(), ring_bytes) catch |err| refused(err, null); + defer tracer.close(); + if (syscalls) tracer.syscalls() catch |err| refused(err, &tracer); + if (sched) tracer.sched(btf) catch |err| refused(err, &tracer); + for (places.items) |p| { + const id = zt.perf.stringId(p.name); + try sorted.sink().emit(.{ .ts = zt.event.now(), .kind = .string, .a = .{ id, 0, 0, 0, 0, 0 } }, p.name); + (if (p.usdt) |u| tracer.usdtProbe(p.elf.?, p.target, u, id) else switch (p.kind) { + .kprobe => tracer.kprobe(p.target, id), + else => tracer.uprobe(p.target, p.offset, id), + }) catch |err| switch (err) { + error.NoSuchProbe => fail("{s}: the kernel has no such place to probe\n", .{p.name}), + error.BadUsdtArg => fail("{s}: an argument of this probe ({s}) is in a form not understood yet\n", .{ p.name, p.usdt.?.args }), + else => |e| refused(e, &tracer), + }; + } + // The maps and the threads, which no program here reports. + var session = try zt.perf.Session.open(init.gpa, proc.pid, .{}, sorted.sink()); + defer session.close(); + + zt.perf.release(proc); + const start = zt.event.now(); + const exit_code: ?u64 = pump(&tracer, &session, &sorted) catch |err| switch (err) { + error.RecordingFull => null, + else => |e| return e, + }; + if (exit_code == null) proc.kill(); + try perf.finish(init, &rec, out, exit_code, tracer.lost() + session.lost, zt.event.now() - start, &.{}, &.{}); +} + +fn pump(tracer: *bpf.Tracer, session: *zt.perf.Session, sorted: *zt.perf.Sorted) !u64 { + // What both sources have reported past: the start of the round before. + var past: u64 = 0; + while (true) { + const now = zt.event.now(); + const alive = try session.poll(0); + const records = try tracer.poll(if (alive) 20 else 0); + try sorted.flush(past); + past = now; + if (records == 0 and !alive) break; + } + const code = try perf.exited(tracer.pid, sorted.sink()); + try sorted.flush(std.math.maxInt(u64)); + return code; +} + +fn refused(err: anyerror, tracer: ?*bpf.Tracer) noreturn { + const log = if (tracer) |t| std.mem.sliceTo(t.log, 0) else ""; + switch (err) { + error.PermissionDenied => fail("ztrace probe: bpf() refused though the capabilities are there: inside a user namespace or a container they do not count, the kernel wants them in the initial namespace\n", .{}), + error.UnsafeProgram, error.InvalidProgram => fail("ztrace probe: the kernel's verifier refused a program ({t}):\n{s}\n", .{ err, log }), + else => fail("ztrace probe: {t}\n{s}", .{ err, log }), + } +} + +/// Where a probe goes: a kernel function, or an offset into a file. +const Place = struct { + kind: @FieldType(Probe, "kind"), + name: []const u8, + target: [:0]const u8, + offset: u64 = 0, + elf: ?zt.usdt.Elf = null, + usdt: ?zt.usdt.Probe = null, +}; + +fn place(arena: std.mem.Allocator, p: Probe, program: [:0]const u8) !Place { + if (p.kind == .kprobe) return .{ .kind = .kprobe, .name = p.what, .target = p.what }; + // FILE: in front, or the program itself. + const colons = std.mem.count(u8, p.what, ":"); + const named = colons == @as(usize, if (p.kind == .usdt) 2 else 1); + const split = if (named) std.mem.indexOfScalar(u8, p.what, ':').? else 0; + const file = if (named) try arena.dupeSentinel(u8, p.what[0..split], 0) else program; + const name = p.what[if (named) split + 1 else 0..]; + const elf = zt.usdt.Elf.open(file) catch fail("{s}: not an ELF file that can be read\n", .{file}); + if (p.kind == .uprobe) { + const sym = elf.symbol(name) orelse fail("{s}: no symbol {s}\n", .{ file, name }); + return .{ .kind = .uprobe, .name = name, .target = file, .offset = elf.offset(sym.value) orelse fail("{s}: {s} is not in the file\n", .{ file, name }), .elf = elf }; + } + for (try zt.usdt.probes(arena, elf, 0)) |found| { + const colon = std.mem.indexOfScalar(u8, name, ':') orelse fail("--usdt wants [FILE:]PROVIDER:NAME\n", .{}); + if (!std.mem.eql(u8, found.provider, name[0..colon]) or !std.mem.eql(u8, found.name, name[colon + 1 ..])) continue; + return .{ .kind = .usdt, .name = name, .target = file, .offset = elf.offset(found.pc) orelse continue, .elf = elf, .usdt = found }; + } + fail("{s}: no static probe {s} (readelf -n lists them)\n", .{ file, name }); +} + +/// A program as the verifier numbers it: opcode, registers, offset, immediate. +fn list(what: []const u8, program: bpf.Program) void { + std.debug.print("{s}, {d} instructions\n", .{ what, program.len }); + for (program.code(), 0..) |insn, at| std.debug.print(" {d:>3}: {x:0>2} r{d} r{d} {d:>5} {d}\n", .{ at, insn.code, insn.dst, insn.src, insn.off, insn.imm }); +} diff --git a/src/trace/bpf.zig b/src/trace/bpf.zig new file mode 100644 index 0000000..4069b01 --- /dev/null +++ b/src/trace/bpf.zig @@ -0,0 +1,733 @@ +//! The kernel's view with privilege: BPF programs on the raw tracepoints +//! sys_enter, sys_exit and sched_switch, and on any kprobe, uprobe or USDT +//! probe. Needs CAP_BPF and CAP_PERFMON (root). Each program writes an +//! `Event`, and for a syscall's entry and a probe the registers after it, +//! into one BPF ring buffer; `Tracer.poll` moves them into a `Sink`. +//! +//! The programs are instruction arrays built here, for one process (its id +//! is a constant in them): no compiler, no object file, no libbpf. What the +//! kernel does not tell (where `pid` is in `task_struct`) comes from its BTF. +const std = @import("std"); +const linux = std.os.linux; +const BPF = linux.BPF; +const Insn = BPF.Insn; +const event = @import("event.zig"); +const usdt = @import("usdt.zig"); +const Event = event.Event; +const Regs = event.Regs; +const Kind = event.Kind; + +/// The kernel's `struct pt_regs` is `Regs` up to here. +const pt_regs_bytes = @offsetOf(Regs, "fs_base"); +/// On a program's stack, from the frame pointer down: the key of the drop +/// counter, then the record. An event alone, or an event and registers. +const key_at = -8; +const small_at = key_at - @sizeOf(Event); +const large_at = small_at - pt_regs_bytes; + +fn field(comptime name: []const u8) i16 { + return @offsetOf(Event, name); +} + +fn arg(i: usize) i16 { + return @intCast(@offsetOf(Event, "a") + 8 * i); +} + +fn reg(comptime name: []const u8) i16 { + return @offsetOf(Regs, name); +} + +pub const Program = struct { + insns: [192]Insn = undefined, + len: usize = 0, + /// The jump that leaves early, to be pointed at the end. + leave: ?usize = null, + + pub fn code(p: *const Program) []const Insn { + return p.insns[0..p.len]; + } + + fn add(p: *Program, insn: Insn) void { + p.insns[p.len] = insn; + p.len += 1; + } + + /// r6 = the context, r7 = thread and process id; goes on only in a + /// thread of process `tgid`. + fn enter(p: *Program, tgid: u32) void { + p.add(Insn.mov(.r6, .r1)); + p.add(Insn.call(.get_current_pid_tgid)); + p.add(Insn.mov(.r7, .r0)); + p.add(Insn.rsh(.r0, 32)); + p.leave = p.len; + p.add(Insn.jne(.r0, @as(i32, @intCast(tgid)), 0)); + } + + /// The time, the thread and the kind of the event at `at`. + fn stamp(p: *Program, at: i16, kind: Kind) void { + p.add(Insn.call(.ktime_get_ns)); + p.add(Insn.stx(.double_word, .r10, at + field("ts"), .r0)); + p.add(Insn.stx(.word, .r10, at + field("tid"), .r7)); + p.add(Insn.st(.half_word, .r10, at + field("kind"), @backingInt(kind))); + } + + /// bpf_probe_read_kernel(frame + `to`, `size`, r3) + fn read(p: *Program, to: i16, size: i32) void { + p.add(Insn.mov(.r1, .r10)); + p.add(Insn.add(.r1, to)); + p.add(Insn.mov(.r2, size)); + p.add(Insn.call(.probe_read_kernel)); + } + + /// The low `bytes` of a register, extended to all of it. + fn extend(p: *Program, r: Insn.Reg, bytes: u8, signed: bool) void { + if (bytes >= 8) return; + const shift: i32 = 64 - 8 * @as(i32, bytes); + p.add(Insn.lsh(r, shift)); + p.add(if (signed) Insn.arsh(r, shift) else Insn.rsh(r, shift)); + } + + /// A register of the probed thread, from the context in r6. + fn probed(p: *Program, to: Insn.Reg, r: anytype) void { + p.add(Insn.ldx(.double_word, to, .r6, @as(i16, r.slot) * 8)); + p.extend(to, r.bytes, false); + } + + /// One argument of a USDT probe into the u64 at `to`: what + /// `usdt.Arg.fetch` computes, in the kernel, at the probe. Memory that + /// cannot be read gives 0. + fn usdtArg(p: *Program, a: usdt.Arg, to: i16) void { + switch (a.at) { + .constant => |c| { + p.add(Insn.ld_dw1(.r1, @bitCast(c))); + p.add(Insn.ld_dw2(@bitCast(c))); + }, + .register => |r| p.probed(.r1, r), + .memory => |m| { + p.add(Insn.st(.double_word, .r10, to, 0)); + p.probed(.r3, m.base); + if (m.index) |index| { + p.probed(.r1, index); + p.add(Insn.lsh(.r1, @as(i32, @ctz(m.scale)))); + p.add(Insn.add(.r3, .r1)); + } + p.add(Insn.ld_dw1(.r1, @bitCast(m.disp))); + p.add(Insn.ld_dw2(@bitCast(m.disp))); + p.add(Insn.add(.r3, .r1)); + p.add(Insn.mov(.r1, .r10)); + p.add(Insn.add(.r1, to)); + p.add(Insn.mov(.r2, @as(i32, a.size))); + p.add(Insn.call(.probe_read_user)); + p.add(Insn.ldx(.double_word, .r1, .r10, to)); + }, + } + p.extend(.r1, a.size, a.signed); + p.add(Insn.stx(.double_word, .r10, to, .r1)); + } + + /// Sends the `size` bytes at `at`, counts the record in `drops` when the + /// ring is full, and ends the program. + fn send(p: *Program, maps: Maps, at: i16, size: i32) void { + p.add(Insn.ld_map_fd1(.r1, maps.ring)); + p.add(Insn.ld_map_fd2(maps.ring)); + p.add(Insn.mov(.r2, .r10)); + p.add(Insn.add(.r2, at)); + p.add(Insn.mov(.r3, size)); + p.add(Insn.mov(.r4, 0)); + p.add(Insn.call(.ringbuf_output)); + p.add(Insn.jeq(.r0, 0, 9)); + p.add(Insn.st(.word, .r10, key_at, 0)); + p.add(Insn.ld_map_fd1(.r1, maps.drops)); + p.add(Insn.ld_map_fd2(maps.drops)); + p.add(Insn.mov(.r2, .r10)); + p.add(Insn.add(.r2, key_at)); + p.add(Insn.call(.map_lookup_elem)); + p.add(Insn.jeq(.r0, 0, 2)); + p.add(Insn.mov(.r1, 1)); + p.add(Insn.xadd(.r0, .r1)); + if (p.leave) |at_jump| p.insns[at_jump].off = @intCast(p.len - at_jump - 1); + p.add(Insn.mov(.r0, 0)); + p.add(Insn.exit()); + } +}; + +/// The maps every program writes to: the ring of records, and an array of +/// one u64 counting the records that found it full. +pub const Maps = struct { ring: linux.fd_t, drops: linux.fd_t }; + +/// raw tracepoint sys_enter(struct pt_regs *regs, long id): a +/// `syscall_enter` and the registers. +pub fn sysEnter(maps: Maps, tgid: u32) Program { + var p: Program = .{}; + p.enter(tgid); + p.add(Insn.ldx(.double_word, .r3, .r6, 0)); + p.read(large_at + @sizeOf(Event), pt_regs_bytes); + p.stamp(large_at, .syscall_enter); + p.add(Insn.ldx(.double_word, .r1, .r6, 8)); + p.add(Insn.stx(.half_word, .r10, large_at + field("aux"), .r1)); + inline for (.{ "rdi", "rsi", "rdx", "r10", "r8", "r9" }, 0..) |name, i| { + p.add(Insn.ldx(.double_word, .r1, .r10, large_at + @sizeOf(Event) + reg(name))); + p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1)); + } + p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes); + return p; +} + +/// raw tracepoint sys_exit(struct pt_regs *regs, long ret): a `syscall_exit`. +pub fn sysExit(maps: Maps, tgid: u32) Program { + var p: Program = .{}; + const number_at = small_at - 8; + p.enter(tgid); + p.add(Insn.ldx(.double_word, .r3, .r6, 0)); + p.add(Insn.add(.r3, reg("orig_rax"))); + p.read(number_at, 8); + p.stamp(small_at, .syscall_exit); + p.add(Insn.ldx(.double_word, .r1, .r10, number_at)); + p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); + p.add(Insn.ldx(.double_word, .r1, .r6, 8)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r1)); + for (1..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0)); + p.send(maps, small_at, @sizeOf(Event)); + return p; +} + +/// raw tracepoint sched_switch(bool preempt, struct task_struct *prev, +/// *next): a `sched` when either is a thread of the process. `pid_at` and +/// `tgid_at` are where those fields are in `task_struct`. +pub fn schedSwitch(maps: Maps, tgid: u32, pid_at: u32, tgid_at: u32) Program { + var p: Program = .{}; + // prev's pid and tgid, next's pid and tgid, as u32 below the event. + const ids_at = small_at - 16; + p.add(Insn.mov(.r6, .r1)); + for ([_]i16{ 8, 16 }, 0..) |task, t| for ([_]u32{ pid_at, tgid_at }, 0..) |at, i| { + p.add(Insn.ldx(.double_word, .r3, .r6, task)); + p.add(Insn.add(.r3, @as(i32, @intCast(at)))); + p.read(ids_at + @as(i16, @intCast(8 * t + 4 * i)), 4); + }; + p.add(Insn.ldx(.word, .r1, .r10, ids_at + 4)); + p.add(Insn.jeq(.r1, @as(i32, @intCast(tgid)), 2)); + p.add(Insn.ldx(.word, .r1, .r10, ids_at + 12)); + p.leave = p.len; + p.add(Insn.jne(.r1, @as(i32, @intCast(tgid)), 0)); + p.add(Insn.ldx(.word, .r7, .r10, ids_at)); + p.stamp(small_at, .sched); + p.add(Insn.ldx(.double_word, .r1, .r6, 0)); + p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r7)); + p.add(Insn.ldx(.word, .r1, .r10, ids_at + 8)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(1), .r1)); + p.add(Insn.call(.get_smp_processor_id)); + p.add(Insn.stx(.double_word, .r10, small_at + arg(2), .r0)); + for (3..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0)); + p.send(maps, small_at, @sizeOf(Event)); + return p; +} + +/// A kprobe or uprobe (the context is the registers there): a `probe` +/// named `id`, and the registers. Its a1..a5 are the first five arguments +/// of a function call, or with `args` those of a USDT probe, read there +/// and then. +pub fn probe(maps: Maps, tgid: u32, id: u64, args: ?[]const usdt.Arg) Program { + var p: Program = .{}; + p.enter(tgid); + p.stamp(large_at, .probe); + p.add(Insn.st(.half_word, .r10, large_at + field("aux"), 0)); + p.add(Insn.ld_dw1(.r1, id)); + p.add(Insn.ld_dw2(id)); + p.add(Insn.stx(.double_word, .r10, large_at + arg(0), .r1)); + inline for (.{ "rdi", "rsi", "rdx", "rcx", "r8" }, 1..) |name, i| { + if (args == null) { + p.add(Insn.ldx(.double_word, .r1, .r6, reg(name))); + p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1)); + } else if (i <= args.?.len) { + p.usdtArg(args.?[i - 1], large_at + arg(i)); + } else p.add(Insn.st(.double_word, .r10, large_at + arg(i), 0)); + } + for (0..pt_regs_bytes / 8) |i| { + p.add(Insn.ldx(.double_word, .r1, .r6, @intCast(8 * i))); + p.add(Insn.stx(.double_word, .r10, large_at + @sizeOf(Event) + @as(i16, @intCast(8 * i)), .r1)); + } + p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes); + return p; +} + +/// `probe` for a USDT probe with the arguments `specs` ("-4@%edi +/// 8@-24(%rbp)"): the first five become a1..a5. +pub fn usdtProgram(maps: Maps, tgid: u32, id: u64, specs: []const u8) !Program { + var args: [5]usdt.Arg = undefined; + var count: usize = 0; + var each = std.mem.tokenizeScalar(u8, specs, ' '); + while (each.next()) |spec| { + if (count == args.len) break; + args[count] = try .parse(spec); + count += 1; + } + return probe(maps, tgid, id, args[0..count]); +} + +/// Where `member` is in the struct `name`, in bytes, from the kernel's +/// type information (the contents of /sys/kernel/btf/vmlinux). +pub fn fieldOffset(btf: []const u8, name: []const u8, member: []const u8) ?u32 { + const T = BPF.btf; + if (btf.len < @sizeOf(T.Header)) return null; + const header = std.mem.bytesToValue(T.Header, btf[0..@sizeOf(T.Header)]); + if (header.magic != T.magic or header.hdr_len + @as(u64, header.str_off) + header.str_len > btf.len) return null; + const types = btf[header.hdr_len + header.type_off ..][0..header.type_len]; + const strings = btf[header.hdr_len + header.str_off ..][0..header.str_len]; + var at: usize = 0; + while (at + @sizeOf(T.Type) <= types.len) { + const t = std.mem.bytesToValue(T.Type, types[at..][0..@sizeOf(T.Type)]); + at += @sizeOf(T.Type); + const count: usize = t.info.vlen; + // What follows the type, by kind. + const extra: usize = switch (t.info.kind) { + .int, .@"var", .decl_tag => 4, + .array => 12, + .@"struct", .@"union", .datasec, .enum64 => 12 * count, + .@"enum", .func_proto => 8 * count, + else => 0, + }; + if (at + extra > types.len) return null; + defer at += extra; + if (t.info.kind != .@"struct" or !std.mem.eql(u8, std.mem.sliceTo(strings[@min(t.name_off, strings.len)..], 0), name)) continue; + for (0..count) |i| { + const m = std.mem.bytesToValue(T.Member, types[at + 12 * i ..][0..12]); + if (std.mem.eql(u8, std.mem.sliceTo(strings[@min(m.name_off, strings.len)..], 0), member)) return m.offset.bit / 8; + } + } + return null; +} + +/// The reading end of a BPF ring buffer: a page with our position, a page +/// with the kernel's, and the data, which the kernel maps twice in a row so +/// that no record wraps. A record is a u32 length with two flag bits, four +/// bytes more, the bytes, padded to 8. +pub const RingBuf = struct { + consumer: *u64, + producer: *const u64, + data: []const u8, + /// Read so far; the kernel is told by `commit`, after the records were used. + at: u64 = 0, + + const busy: u32 = 1 << 31; + const discarded: u32 = 1 << 30; + + pub fn map(fd: linux.fd_t, size: usize) !RingBuf { + const ours = linux.mmap(null, 4096, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0); + if (linux.errno(ours) != .SUCCESS) return error.BpfRingMap; + const theirs = linux.mmap(null, 4096 + 2 * size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 4096); + if (linux.errno(theirs) != .SUCCESS) return error.BpfRingMap; + return .{ .consumer = @ptrFromInt(ours), .producer = @ptrFromInt(theirs), .data = @as([*]const u8, @ptrFromInt(theirs + 4096))[0..size] }; + } + + /// The next finished record, valid until `commit`. + pub fn next(r: *RingBuf) ?[]const u8 { + while (r.at < @atomicLoad(u64, r.producer, .acquire)) { + const start: usize = @intCast(r.at & (r.data.len - 1)); + const header = @atomicLoad(u32, @as(*const u32, @ptrCast(@alignCast(r.data.ptr + start))), .acquire); + if (header & busy != 0) return null; + const len = header & ~(busy | discarded); + r.at += std.mem.alignForward(u64, len + 8, 8); + if (header & discarded == 0) return (r.data.ptr + start + 8)[0..len]; + } + return null; + } + + pub fn commit(r: *RingBuf) void { + @atomicStore(u64, r.consumer, r.at, .release); + } +}; + +/// The capability bits of this process (CapEff of /proc/self/status). +fn capabilities() u64 { + var buf: [4096]u8 = undefined; + const fd = linux.open("/proc/self/status", .{}, 0); + if (linux.errno(fd) != .SUCCESS) return 0; + defer _ = linux.close(@intCast(fd)); + const n = linux.read(@intCast(fd), &buf, buf.len); + if (linux.errno(n) != .SUCCESS) return 0; + const label = "CapEff:\t"; + const at = std.mem.indexOf(u8, buf[0..n], label) orelse return 0; + return std.fmt.parseInt(u64, buf[at + label.len ..][0..16], 16) catch 0; +} + +/// What this process lacks to trace with BPF, said into `buf`; null when +/// it has the capabilities (which inside a user namespace still is not +/// enough: the kernel wants them in the initial one). +pub fn missing(buf: []u8) ?[]const u8 { + const caps = capabilities(); + const has = struct { + fn has(all: u64, cap: u6) bool { + return all >> cap & 1 != 0; + } + }.has; + if (has(caps, linux.CAP.SYS_ADMIN) or (has(caps, linux.CAP.BPF) and has(caps, linux.CAP.PERFMON))) return null; + return std.fmt.bufPrint(buf, "missing {s}{s}{s} (effective capabilities {x:0>16}, uid {d}). " ++ + "CAP_BPF is what bpf() wants to create maps and load programs, CAP_PERFMON to load tracing programs and open kprobes and uprobes; CAP_SYS_ADMIN stands for both. " ++ + "Run as root, or give the binary the two: setcap cap_bpf,cap_perfmon+ep ztrace", .{ + if (has(caps, linux.CAP.BPF)) "" else "CAP_BPF", + if (has(caps, linux.CAP.BPF) or has(caps, linux.CAP.PERFMON)) "" else " and ", + if (has(caps, linux.CAP.PERFMON)) "" else "CAP_PERFMON", + caps, + linux.getuid(), + }) catch null; +} + +pub const Tracer = struct { + gpa: std.mem.Allocator, + /// The process traced: its thread group id, in the initial pid namespace. + /// shortcut: one process, and a tracer in a pid namespace of its own sees nothing; keep the ids in a BPF map to follow children. + pid: linux.pid_t, + sink: event.Sink, + maps: Maps, + ring: RingBuf, + /// Programs and what attaches them. + fds: std.ArrayList(linux.fd_t) = .empty, + /// What the verifier said about the last program, when it refused one. + log: []u8, + + pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, sink: event.Sink, ring_bytes: u32) !Tracer { + const ring = try BPF.map_create(.ringbuf, 0, 0, ring_bytes); + errdefer _ = linux.close(ring); + const drops = try BPF.map_create(.array, 4, 8, 1); + errdefer _ = linux.close(drops); + const ring_map = try RingBuf.map(ring, ring_bytes); + const log = try gpa.alloc(u8, 1 << 16); + log[0] = 0; + return .{ .gpa = gpa, .pid = pid, .sink = sink, .maps = .{ .ring = ring, .drops = drops }, .ring = ring_map, .log = log }; + } + + pub fn close(t: *Tracer) void { + for (t.fds.items) |fd| _ = linux.close(fd); + t.fds.deinit(t.gpa); + _ = linux.munmap(@ptrCast(@alignCast(t.ring.consumer)), 4096); + _ = linux.munmap(@ptrCast(@alignCast(t.ring.producer)), 4096 + 2 * t.ring.data.len); + _ = linux.close(t.maps.ring); + _ = linux.close(t.maps.drops); + t.gpa.free(t.log); + } + + fn load(t: *Tracer, kind: BPF.ProgType, program: Program) !linux.fd_t { + // GPL: bpf_probe_read_kernel is for programs that say so. + const fd = BPF.prog_load(kind, program.code(), null, "GPL", 0, 0) catch |err| { + // Again for the verifier's reasons, which a good program need not pay for. + var log: BPF.Log = .{ .level = 1, .buf = t.log }; + if (BPF.prog_load(kind, program.code(), &log, "GPL", 0, 0)) |again| _ = linux.close(again) else |_| {} + return err; + }; + try t.fds.append(t.gpa, fd); + return fd; + } + + fn rawTracepoint(t: *Tracer, name: [*:0]const u8, program: Program) !void { + var attr = std.mem.zeroes(BPF.Attr); + attr.raw_tracepoint = .{ .name = @intFromPtr(name), .prog_fd = try t.load(.raw_tracepoint, program) }; + const rc = linux.bpf(.raw_tracepoint_open, &attr, @sizeOf(BPF.RawTracepointAttr)); + if (linux.errno(rc) != .SUCCESS) return error.BpfAttach; + try t.fds.append(t.gpa, @intCast(rc)); + } + + /// Every syscall of the process. + pub fn syscalls(t: *Tracer) !void { + try t.rawTracepoint("sys_enter", sysEnter(t.maps, @intCast(t.pid))); + try t.rawTracepoint("sys_exit", sysExit(t.maps, @intCast(t.pid))); + } + + /// Every time one of its threads leaves or gets a CPU. `btf` is the + /// contents of /sys/kernel/btf/vmlinux. + pub fn sched(t: *Tracer, btf: []const u8) !void { + const pid_at = fieldOffset(btf, "task_struct", "pid") orelse return error.NoBtf; + const tgid_at = fieldOffset(btf, "task_struct", "tgid") orelse return error.NoBtf; + try t.rawTracepoint("sched_switch", schedSwitch(t.maps, @intCast(t.pid), pid_at, tgid_at)); + } + + /// A probe through the perf event of the kprobe or uprobe PMU: `target` + /// is a kernel function or a file, `offset` is into it. + fn attach(t: *Tracer, pmu: []const u8, target: [*:0]const u8, offset: u64, semaphore: u64, program: Program) !void { + var path_buf: [64]u8 = undefined; + var number: [16]u8 = undefined; + const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/sys/bus/event_source/devices/{s}/type", .{pmu}, 0), .{}, 0); + if (linux.errno(fd) != .SUCCESS) return error.NoProbeSupport; + defer _ = linux.close(@intCast(fd)); + const n = linux.read(@intCast(fd), &number, number.len); + if (linux.errno(n) != .SUCCESS) return error.NoProbeSupport; + const kind = std.fmt.parseInt(u32, std.mem.trim(u8, number[0..n], "\n "), 10) catch return error.NoProbeSupport; + + // A uprobe's semaphore (the offset in the file of its counter) rides in the upper half of config. + // On the first thread alone, and not inherited: a program on a probe + // runs for whoever hits it (hence the process id in ours), and the + // kernel reads `target` again, from the target's memory, for an + // event a clone would inherit, which fails the clone. + var attr: linux.perf_event_attr = .{ .type = @fromBackingInt(@intCast(kind)), .config = semaphore << 32, .config1 = @intFromPtr(target), .config2 = offset }; + const event_fd = linux.perf_event_open(&attr, t.pid, -1, -1, linux.PERF.FLAG.FD_CLOEXEC); + if (linux.errno(event_fd) != .SUCCESS) return if (linux.errno(event_fd) == .NOENT) error.NoSuchProbe else error.BpfAttach; + try t.fds.append(t.gpa, @intCast(event_fd)); + const loaded = try t.load(.kprobe, program); + if (linux.errno(linux.ioctl(@intCast(event_fd), linux.PERF.EVENT_IOC.SET_BPF, @intCast(loaded))) != .SUCCESS) return error.BpfAttach; + } + + /// Every call of a kernel function by the process: a `probe` named + /// `function` (define the string yourself: its id is `id`). + pub fn kprobe(t: *Tracer, function: [*:0]const u8, id: u64) !void { + try t.attach("kprobe", function, 0, 0, probe(t.maps, @intCast(t.pid), id, null)); + } + + /// Every time the process runs the instruction at `offset` of the + /// file `path`. + pub fn uprobe(t: *Tracer, path: [*:0]const u8, offset: u64, id: u64) !void { + try t.attach("uprobe", path, offset, 0, probe(t.maps, @intCast(t.pid), id, null)); + } + + /// A static probe of the file `path` (`p` from `usdt.probes` with no + /// bias). The kernel raises its semaphore while we are attached. + pub fn usdtProbe(t: *Tracer, elf: usdt.Elf, path: [*:0]const u8, p: usdt.Probe, id: u64) !void { + const semaphore = if (p.semaphore == 0) 0 else elf.offset(p.semaphore) orelse return error.NoSuchProbe; + try t.attach("uprobe", path, elf.offset(p.pc) orelse return error.NoSuchProbe, semaphore, try usdtProgram(t.maps, @intCast(t.pid), id, p.args)); + } + + /// Waits up to the timeout for records and decodes them into the sink. + /// Returns how many. + pub fn poll(t: *Tracer, timeout_ms: i32) !usize { + var fds = [1]linux.pollfd{.{ .fd = t.maps.ring, .events = linux.POLL.IN, .revents = 0 }}; + _ = linux.poll(&fds, 1, timeout_ms); + var n: usize = 0; + while (t.ring.next()) |record| : (n += 1) try decode(record, t.sink); + t.ring.commit(); + return n; + } + + /// Records that found the ring full. + pub fn lost(t: *Tracer) u64 { + var count: u64 = 0; + BPF.map_lookup_elem(t.maps.drops, &std.mem.toBytes(@as(u32, 0)), std.mem.asBytes(&count)) catch return 0; + return count; + } +}; + +/// A record of the ring into the sink: the event, and the registers when +/// they follow it. +fn decode(record: []const u8, sink: event.Sink) !void { + if (record.len < @sizeOf(Event)) return; + const ev = std.mem.bytesToValue(Event, record[0..@sizeOf(Event)]); + if (record.len < @sizeOf(Event) + pt_regs_bytes) return sink.emit(ev, ""); + var regs: Regs = .{}; + @memcpy(std.mem.asBytes(®s)[0..pt_regs_bytes], record[@sizeOf(Event)..][0..pt_regs_bytes]); + try sink.emit(ev, std.mem.asBytes(®s)); +} + +// What follows runs the programs without a kernel: enough of a BPF machine +// for what they use, on this process's own memory. +const Machine = struct { + pid_tgid: u64 = 0, + now: u64 = 0, + cpu: u64 = 0, + /// What bpf_ringbuf_output returns; not 0 is a full ring. + ring_full: bool = false, + sent: [512]u8 = undefined, + sent_len: ?usize = null, + drops: u64 = 0, + + fn run(m: *Machine, code: []const Insn, context: *const anyopaque) !void { + var stack: [512]u8 align(8) = undefined; + var r: [11]u64 = @splat(0); + r[1] = @intFromPtr(context); + r[10] = @intFromPtr(&stack) + stack.len; + var pc: usize = 0; + var steps: usize = 0; + while (true) : (steps += 1) { + if (steps > 10_000 or pc >= code.len) return error.RanAway; + const insn = code[pc]; + pc += 1; + const imm: u64 = @bitCast(@as(i64, insn.imm)); + const source = if (insn.code & BPF.X != 0) r[insn.src] else imm; + const address = (if (insn.code & 7 == BPF.LDX) r[insn.src] else r[insn.dst]) +% @as(u64, @bitCast(@as(i64, insn.off))); + switch (insn.code) { + BPF.ALU64 | BPF.MOV | BPF.K, BPF.ALU64 | BPF.MOV | BPF.X => r[insn.dst] = source, + BPF.ALU64 | BPF.ADD | BPF.K, BPF.ALU64 | BPF.ADD | BPF.X => r[insn.dst] +%= source, + BPF.ALU64 | BPF.RSH | BPF.K => r[insn.dst] >>= @intCast(insn.imm), + BPF.ALU64 | BPF.LSH | BPF.K => r[insn.dst] <<= @intCast(insn.imm), + BPF.ALU64 | BPF.ARSH | BPF.K => r[insn.dst] = @bitCast(@as(i64, @bitCast(r[insn.dst])) >> @intCast(insn.imm)), + BPF.JMP | BPF.JEQ | BPF.K => if (r[insn.dst] == imm) { + pc = @intCast(@as(i64, @intCast(pc)) + insn.off); + }, + BPF.JMP | BPF.JNE | BPF.K => if (r[insn.dst] != imm) { + pc = @intCast(@as(i64, @intCast(pc)) + insn.off); + }, + BPF.LD | BPF.DW | BPF.IMM => { + r[insn.dst] = @as(u32, @bitCast(insn.imm)) | @as(u64, @as(u32, @bitCast(code[pc].imm))) << 32; + pc += 1; + }, + inline BPF.LDX | BPF.MEM | BPF.B, BPF.LDX | BPF.MEM | BPF.H, BPF.LDX | BPF.MEM | BPF.W, BPF.LDX | BPF.MEM | BPF.DW => |code_now| { + r[insn.dst] = @as(*align(1) const Width(code_now), @ptrFromInt(address)).*; + }, + inline BPF.STX | BPF.MEM | BPF.H, BPF.STX | BPF.MEM | BPF.W, BPF.STX | BPF.MEM | BPF.DW => |code_now| { + @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(r[insn.src]); + }, + inline BPF.ST | BPF.MEM | BPF.H, BPF.ST | BPF.MEM | BPF.W, BPF.ST | BPF.MEM | BPF.DW => |code_now| { + @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(imm); + }, + BPF.STX | BPF.XADD | BPF.DW => @as(*u64, @ptrFromInt(address)).* += r[insn.src], + BPF.JMP | BPF.CALL => r[0] = switch (@as(BPF.Helper, @fromBackingInt(@intCast(insn.imm)))) { + .get_current_pid_tgid => m.pid_tgid, + .ktime_get_ns => m.now, + .get_smp_processor_id => m.cpu, + .probe_read_kernel, .probe_read_user => read: { + @memcpy(@as([*]u8, @ptrFromInt(r[1]))[0..r[2]], @as([*]const u8, @ptrFromInt(r[3]))[0..r[2]]); + break :read 0; + }, + .ringbuf_output => out: { + if (r[1] != 3 or m.ring_full) break :out @as(u64, @bitCast(@as(i64, -11))); + m.sent_len = r[3]; + @memcpy(m.sent[0..r[3]], @as([*]const u8, @ptrFromInt(r[2]))[0..r[3]]); + break :out 0; + }, + .map_lookup_elem => if (r[1] == 4 and @as(*const u32, @ptrFromInt(r[2])).* == 0) @intFromPtr(&m.drops) else 0, + else => return error.UnknownHelper, + }, + BPF.JMP | BPF.EXIT => return, + else => return error.UnknownInstruction, + } + } + } + + fn Width(comptime code: u8) type { + return switch (code & 0x18) { + BPF.B => u8, + BPF.H => u16, + BPF.W => u32, + else => u64, + }; + } + + fn sentEvent(m: *const Machine) Event { + return std.mem.bytesToValue(Event, m.sent[0..@sizeOf(Event)]); + } +}; + +const test_maps: Maps = .{ .ring = 3, .drops = 4 }; + +test "the syscall programs: the right process only, the event the tracer expects" { + var regs: Regs = .{ .rdi = 1, .rsi = 2, .rdx = 3, .r10 = 4, .r8 = 5, .r9 = 6, .orig_rax = 257, .rip = 0x401000 }; + const enter_context = [2]u64{ @intFromPtr(®s), 257 }; + var m: Machine = .{ .pid_tgid = 1000 << 32 | 1001, .now = 77 }; + try m.run(sysEnter(test_maps, 1000).code(), &enter_context); + try std.testing.expectEqual(@as(?usize, @sizeOf(Event) + pt_regs_bytes), m.sent_len); + try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_enter, .aux = 257, .a = .{ 1, 2, 3, 4, 5, 6 } }, m.sentEvent()); + try std.testing.expectEqualSlices(u8, std.mem.asBytes(®s)[0..pt_regs_bytes], m.sent[@sizeOf(Event)..][0..pt_regs_bytes]); + + const exit_context = [2]u64{ @intFromPtr(®s), @bitCast(@as(i64, -2)) }; + try m.run(sysExit(test_maps, 1000).code(), &exit_context); + try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len); + try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_exit, .aux = 257, .a = .{ @bitCast(@as(i64, -2)), 0, 0, 0, 0, 0 } }, m.sentEvent()); + + // Another process: nothing. A full ring: counted. + m.sent_len = null; + try m.run(sysEnter(test_maps, 999).code(), &enter_context); + try std.testing.expectEqual(@as(?usize, null), m.sent_len); + m.ring_full = true; + try m.run(sysExit(test_maps, 1000).code(), &exit_context); + try m.run(sysEnter(test_maps, 1000).code(), &enter_context); + try std.testing.expectEqual(@as(u64, 2), m.drops); +} + +test "the sched_switch program: either side of the switch, ids from where BTF says" { + // Two task_structs with the ids at offsets of our choosing. + var prev: [64]u32 = @splat(0); + var next: [64]u32 = @splat(0); + prev[10] = 1001; + prev[11] = 1000; + next[10] = 55; + next[11] = 50; + const context = [3]u64{ 1, @intFromPtr(&prev), @intFromPtr(&next) }; + var m: Machine = .{ .now = 9, .cpu = 3 }; + try m.run(schedSwitch(test_maps, 1000, 40, 44).code(), &context); + try std.testing.expectEqual(Event{ .ts = 9, .tid = 1001, .kind = .sched, .aux = 1, .a = .{ 1001, 55, 3, 0, 0, 0 } }, m.sentEvent()); + m.sent_len = null; + try m.run(schedSwitch(test_maps, 50, 40, 44).code(), &context); + try std.testing.expectEqual(@as(u64, 55), m.sentEvent().a[1]); + m.sent_len = null; + try m.run(schedSwitch(test_maps, 7, 40, 44).code(), &context); + try std.testing.expectEqual(@as(?usize, null), m.sent_len); +} + +test "the probe program and its record decoded; a USDT probe's arguments as usdt.Arg.fetch has them" { + // The probed thread's memory is ours: rbp and rdx point into this. + var memory: [64]u8 = undefined; + for (&memory, 0..) |*b, i| b.* = @truncate(0xf0 + i); + const regs: Regs = .{ .rdi = 11, .rsi = 0xffff_fffe, .rdx = @intFromPtr(&memory), .rcx = 3, .r8 = 15, .rbp = @intFromPtr(&memory) + 32, .rip = 0x401234, .rsp = 0x7000, .r10 = 0x1_0000_00ff }; + var m: Machine = .{ .pid_tgid = 1000 << 32 | 1000, .now = 5 }; + try m.run(probe(test_maps, 1000, 0xfeed, null).code(), ®s); + try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent()); + + // A sink that keeps the last event. + const Last = struct { + ev: Event = undefined, + regs: Regs = undefined, + fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void { + const l: *@This() = @ptrCast(@alignCast(ctx)); + l.ev = ev.*; + l.regs = std.mem.bytesToValue(Regs, payload); + } + }; + var last: Last = .{}; + try decode(m.sent[0..m.sent_len.?], .{ .ctx = &last, .emitFn = Last.emit }); + try std.testing.expectEqual(@as(u64, 11), last.ev.a[1]); + try std.testing.expectEqual(regs.rip, last.regs.rip); + try std.testing.expectEqual(@as(u64, 0), last.regs.fs_base); + + const here = struct { + pub fn read(_: @This(), addr: u64, buf: []u8) usize { + @memcpy(buf, @as([*]const u8, @ptrFromInt(addr))); + return buf.len; + } + }{}; + const specs = [_][5][]const u8{ + .{ "-4@%esi", "4@%esi", "1@-16(%rbp)", "8@$9", "-8@$-9" }, + .{ "-1@8(%rdx,%rcx,8)", "2@(%rdx)", "-2@-32(%rbp)", "1@%r10b", "8@%r10" }, + }; + for (specs) |five| { + var args: [5]usdt.Arg = undefined; + for (&args, five) |*a, spec| a.* = try .parse(spec); + try m.run(probe(test_maps, 1000, 0xfeed, &args).code(), ®s); + for (args, m.sentEvent().a[1..]) |a, got| try std.testing.expectEqual(a.fetch(®s, here).?, got); + } + // Fewer than five: the rest is 0. + try m.run(probe(test_maps, 1000, 0xfeed, &.{try .parse("4@%edi")}).code(), ®s); + try std.testing.expectEqual([6]u64{ 0xfeed, 11, 0, 0, 0, 0 }, m.sentEvent().a); +} + +test "a ring buffer as the kernel lays it out: finished records only, discarded ones skipped" { + var consumer: u64 = 0; + var producer: u64 = 0; + var data: [64]u8 align(8) = @splat(0); + var ring: RingBuf = .{ .consumer = &consumer, .producer = &producer, .data = &data }; + const records = [_]struct { u32, []const u8 }{ .{ 5, "hello" }, .{ 3 | RingBuf.discarded, "no!" }, .{ 2, "ok" }, .{ 4 | RingBuf.busy, "wait" } }; + for (records) |record| { + std.mem.writeInt(u32, data[producer..][0..4], record[0], .little); + @memcpy(data[producer + 8 ..][0..record[1].len], record[1]); + producer += std.mem.alignForward(u64, record[1].len + 8, 8); + } + try std.testing.expectEqualStrings("hello", ring.next().?); + try std.testing.expectEqualStrings("ok", ring.next().?); + try std.testing.expect(ring.next() == null); + ring.commit(); + try std.testing.expectEqual(@as(u64, 48), consumer); + // The producer finishes it. + std.mem.writeInt(u32, data[48..52], 4, .little); + try std.testing.expectEqualStrings("wait", ring.next().?); +} + +test "task_struct's pid and tgid in this kernel's BTF; privilege is told apart" { + var buf: [512]u8 = undefined; + if (missing(&buf)) |why| try std.testing.expect(std.mem.indexOf(u8, why, "CAP_") != null); + + const btf = std.Io.Dir.cwd().readFileAlloc(std.testing.io, "/sys/kernel/btf/vmlinux", std.testing.allocator, .unlimited) catch return error.SkipZigTest; + defer std.testing.allocator.free(btf); + const pid_at = fieldOffset(btf, "task_struct", "pid").?; + // Next to each other, some way into a struct of a few kilobytes. + try std.testing.expectEqual(pid_at + 4, fieldOffset(btf, "task_struct", "tgid").?); + try std.testing.expect(pid_at > 256 and pid_at < 16384); + try std.testing.expect(fieldOffset(btf, "task_struct", "no_such_field") == null); + try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "rdi")), fieldOffset(btf, "pt_regs", "di")); + try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "orig_rax")), fieldOffset(btf, "pt_regs", "orig_ax")); +} diff --git a/src/trace/perf.zig b/src/trace/perf.zig index 7c9ef43..c251741 100644 --- a/src/trace/perf.zig +++ b/src/trace/perf.zig @@ -513,6 +513,58 @@ fn userData(raw: []const u8) []const u8 { return ""; } +/// A sink in front of another for when two backends record the same run +/// (this one and BPF, or the agent's ring): each emits in time order, both +/// with the same clock, and here they become one sequence. Events are held +/// until `flush` passes on, oldest first, those up to a time that every +/// backend has reported past. +pub const Sorted = struct { + gpa: std.mem.Allocator, + to: event.Sink, + held: std.ArrayList(Held) = .empty, + payloads: std.ArrayList(u8) = .empty, + + const Held = struct { ev: Event, at: usize, len: usize }; + + pub fn sink(s: *Sorted) event.Sink { + return .{ .ctx = s, .emitFn = hold }; + } + + fn hold(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void { + const s: *Sorted = @ptrCast(@alignCast(ctx)); + try s.held.append(s.gpa, .{ .ev = ev.*, .at = s.payloads.items.len, .len = payload.len }); + try s.payloads.appendSlice(s.gpa, payload); + } + + pub fn flush(s: *Sorted, until: u64) !void { + // Stable: events of one instant stay in the order their backend gave them. + std.mem.sort(Held, s.held.items, {}, struct { + fn older(_: void, a: Held, b: Held) bool { + return a.ev.ts < b.ev.ts; + } + }.older); + var done: usize = 0; + while (done < s.held.items.len and s.held.items[done].ev.ts <= until) : (done += 1) { + const h = s.held.items[done]; + try s.to.emit(h.ev, s.payloads.items[h.at..][0..h.len]); + } + // What stays moves to the front, its payloads with it. + var kept: std.ArrayList(u8) = .empty; + for (s.held.items[done..]) |*h| { + try kept.appendSlice(s.gpa, s.payloads.items[h.at..][0..h.len]); + h.at = kept.items.len - h.len; + } + s.held.replaceRangeAssumeCapacity(0, done, &.{}); + s.payloads.deinit(s.gpa); + s.payloads = kept; + } + + pub fn deinit(s: *Sorted) void { + s.held.deinit(s.gpa); + s.payloads.deinit(s.gpa); + } +}; + /// Lets a target held by `ptrace.Process.spawn` run on its own: once the /// events are attached, ptrace has done its part. pub fn release(p: ptrace.Process) void { @@ -596,6 +648,31 @@ fn perfOpen(attr: *linux.perf_event_attr, pid: linux.pid_t, cpu: i32) !linux.fd_ }; } +test "two backends' events come out as one sequence in time" { + const Seen = struct { + order: [8]u8 = undefined, + len: usize = 0, + fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void { + const seen: *@This() = @ptrCast(@alignCast(ctx)); + try std.testing.expectEqual(ev.a[0], payload.len); + seen.order[seen.len] = payload[0]; + seen.len += 1; + } + }; + var seen: Seen = .{}; + var sorted: Sorted = .{ .gpa = std.testing.allocator, .to = .{ .ctx = &seen, .emitFn = Seen.emit } }; + defer sorted.deinit(); + // One backend, then the other, each in its own order. + for ([_]struct { u64, []const u8 }{ .{ 10, "a" }, .{ 30, "ccc" }, .{ 50, "eeeee" }, .{ 20, "bb" }, .{ 30, "dddd" }, .{ 60, "ffffff" } }) |e| { + try sorted.sink().emit(.{ .ts = e[0], .kind = .mark, .a = .{ e[1].len, 0, 0, 0, 0, 0 } }, e[1]); + } + try sorted.flush(30); + try std.testing.expectEqualStrings("abcd", seen.order[0..seen.len]); + try sorted.sink().emit(.{ .ts = 55, .kind = .mark, .a = .{ 1, 0, 0, 0, 0, 0 } }, "x"); + try sorted.flush(std.math.maxInt(u64)); + try std.testing.expectEqualStrings("abcdexf", seen.order[0..seen.len]); +} + test "a maps line; the user data of a syscall record" { const m = parseMap("7f80b4224000-7f80b439f000 r-xp 00024000 08:02 1234 /usr/lib/libc.so.6").?; try std.testing.expectEqual(@as(u64, 0x7f80b4224000), m.start); diff --git a/src/trace/root.zig b/src/trace/root.zig index 80745cb..62a7a90 100644 --- a/src/trace/root.zig +++ b/src/trace/root.zig @@ -20,6 +20,8 @@ pub const host = @import("host.zig"); pub const perf = @import("perf.zig"); /// Static probes (the ELF notes of <sys/sdt.h>) and their arguments. pub const usdt = @import("usdt.zig"); +/// The kernel's view, privileged: BPF on tracepoints, kprobes, uprobes, USDT. +pub const bpf = @import("bpf.zig"); test { _ = event; @@ -33,4 +35,5 @@ test { _ = host; _ = perf; _ = usdt; + _ = bpf; } diff --git a/src/trace/usdt.zig b/src/trace/usdt.zig index ba03079..d5f1d04 100644 --- a/src/trace/usdt.zig +++ b/src/trace/usdt.zig @@ -194,6 +194,7 @@ pub const Arg = struct { .index = if (parts.next()) |r| try register(r) else null, .scale = if (parts.next()) |s| std.fmt.parseInt(u8, s, 10) catch return error.BadUsdtArg else 1, } }; + if (std.mem.indexOfScalar(u8, &.{ 1, 2, 4, 8 }, arg.at.memory.scale) == null) return error.BadUsdtArg; } return arg; } @@ -278,7 +279,7 @@ test "argument specs are parsed and fetched" { .{ "8@0x7000(%rbp)", null }, }; for (cases) |case| try std.testing.expectEqual(case[1], (try Arg.parse(case[0])).fetch(®s, memory)); - for ([_][]const u8{ "4@counter(%rip)", "3@%eax", "4@%ah", "4", "4@", "4@8(%rdx", "4@%r8x" }) |bad| { + for ([_][]const u8{ "4@counter(%rip)", "3@%eax", "4@%ah", "4", "4@", "4@8(%rdx", "4@%r8x", "4@(%rdx,%rax,3)" }) |bad| { try std.testing.expectError(error.BadUsdtArg, Arg.parse(bad)); } } |
