summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/cli/main.zig2
-rw-r--r--src/cli/perf.zig60
-rw-r--r--src/cli/probe.zig215
-rw-r--r--src/trace/bpf.zig733
-rw-r--r--src/trace/perf.zig77
-rw-r--r--src/trace/root.zig3
-rw-r--r--src/trace/usdt.zig3
7 files changed, 1080 insertions, 13 deletions
diff --git a/src/cli/main.zig b/src/cli/main.zig
index 9e6b0e5..554efde 100644
--- a/src/cli/main.zig
+++ b/src/cli/main.zig
@@ -8,6 +8,7 @@ const commands = .{
.dbg = @import("dbg.zig"),
.run = @import("run.zig"),
.perf = @import("perf.zig"),
+ .probe = @import("probe.zig"),
.dump = @import("dump.zig"),
.show = @import("show.zig"),
.cc = @import("cc.zig"),
@@ -20,6 +21,7 @@ const usage =
\\ dbg debug a program, recording it: step, break, look, go back
\\ run record a program that carries a ring producer (agent, instrumentation)
\\ perf record what the kernel sees of a program at full speed (perf events)
+ \\ probe ... with BPF programs on syscalls, scheduling, kprobes, uprobes, USDT (root)
\\ dump print a recording's events
\\ show print a recording's summary, or the program's state at a moment
\\ cc compile a program with instrumentation that records it
diff --git a/src/cli/perf.zig b/src/cli/perf.zig
index 57ad11a..58f7d77 100644
--- a/src/cli/perf.zig
+++ b/src/cli/perf.zig
@@ -30,10 +30,15 @@ pub const usage =
\\ size) with the value found there; with :rw every access
\\ --break WHAT ... on code: every time it runs, with the registers
\\ --threads also the threads and children it creates
+ \\ --cut end a moment at every fault and every hit of --watch and
+ \\ --break: the state at each can be gone back to, for more bytes
+ \\ --max-size BYTES stop when the recording is this large (default 1 GiB)
\\ --ring BYTES the kernel's buffer, per CPU (a power of two, default 512 KiB)
\\
\\There are four debug registers. Only user space is seen: a page the
\\kernel touches first (a read() into fresh memory) faults unseen.
+ \\Moments end where the recorder ends them anyway (a syscall's exit, a
+ \\megabyte of events), so that a profile or a storm of faults stays small.
\\
;
@@ -44,6 +49,8 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
var out: []const u8 = "out.ztrace";
var options: zt.perf.Options = .{ .counters = &.{ .instructions, .cycles } };
var interval_ms: u32 = 0;
+ var cut = false;
+ var max_bytes: ?u64 = null;
var specs: std.ArrayList(Spec) = .empty;
var i: usize = 0;
while (i < args.len) : (i += 1) {
@@ -62,6 +69,8 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
options.syscalls = true;
} else if (std.mem.eql(u8, arg, "--threads")) {
options.follow = true;
+ } else if (std.mem.eql(u8, arg, "--cut")) {
+ cut = true;
} else if (value == null) {
fail(usage, .{});
} else if (std.mem.eql(u8, arg, "-o")) {
@@ -79,6 +88,9 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
} else if (std.mem.eql(u8, arg, "--sample")) {
options.sample_hz = std.fmt.parseInt(u32, value.?, 0) catch fail("--sample needs a rate\n", .{});
i += 1;
+ } else if (std.mem.eql(u8, arg, "--max-size")) {
+ max_bytes = std.fmt.parseInt(u64, value.?, 0) catch fail("--max-size needs bytes\n", .{});
+ i += 1;
} else if (std.mem.eql(u8, arg, "--ring")) {
options.ring_bytes = std.fmt.parseInt(usize, value.?, 0) catch 0;
if (!std.math.isPowerOfTwo(options.ring_bytes) or options.ring_bytes < 4096) fail("--ring needs a power of two, 4096 or more\n", .{});
@@ -105,8 +117,9 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
var proc = try zt.ptrace.Process.spawn(path, argv, init.minimal.environ.block.slice.ptr);
errdefer proc.kill();
var rec = try zt.Recorder.create(init.gpa, init.io, out);
+ if (max_bytes) |bytes| rec.max_bytes = bytes;
// What changes the state a moment holds (registers, memory) ends one.
- rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit();
+ if (cut) rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit();
var session = zt.perf.Session.open(init.gpa, proc.pid, options, rec.sink()) catch |err| switch (err) {
error.PerfAccess => fail("perf_event_open was refused: kernel.perf_event_paranoid is above 2, or a security module forbids it\n", .{}),
error.PerfNoSuchEvent => fail("perf_event_open: this CPU or kernel lacks one of the events asked for (in a VM without a PMU, try -e task-clock; --sample takes at most kernel.perf_event_max_sample_rate)\n", .{}),
@@ -133,7 +146,19 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
zt.perf.release(proc);
const start = zt.event.now();
- var next_read = start;
+ const exit_code: ?u64 = pump(&session, &rec, interval_ms) catch |err| switch (err) {
+ error.RecordingFull => null,
+ else => |e| return e,
+ };
+ if (exit_code == null) proc.kill();
+ const ns = zt.event.now() - start;
+ try finish(init, &rec, out, exit_code, session.lost, ns, options.counters, session.counts);
+}
+
+/// Decodes until the target ends; its exit code. `error.RecordingFull`
+/// when the recording reaches its limit first.
+fn pump(session: *zt.perf.Session, rec: *zt.Recorder, interval_ms: u32) !u64 {
+ var next_read = zt.event.now();
while (true) {
var wait_ms: i32 = 100;
if (interval_ms != 0) {
@@ -147,22 +172,33 @@ pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
if (!try session.poll(wait_ms)) break;
}
try session.readCounters();
+ return exited(session.pid, rec.sink());
+}
+
+/// Waits for the target's end and records it. The exit code.
+pub fn exited(pid: linux.pid_t, sink: zt.Sink) !u64 {
var status: u32 = 0;
- _ = linux.waitpid(proc.pid, @ptrCast(&status), 0);
- const exit_code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status)));
- const ns = zt.event.now() - start;
- try rec.emit(.{ .ts = zt.event.now(), .tid = @intCast(proc.pid), .kind = .exit, .a = .{ exit_code, 0, 0, 0, 0, 0 } }, "");
+ _ = linux.waitpid(pid, @ptrCast(&status), 0);
+ const code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status)));
+ try sink.emit(.{ .ts = zt.event.now(), .tid = @intCast(pid), .kind = .exit, .a = .{ code, 0, 0, 0, 0, 0 } }, "");
+ return code;
+}
+/// Closes the recording and says what it holds; `exit_code` is null when
+/// it filled up and the target was stopped for it.
+pub fn finish(init: std.process.Init, rec: *zt.Recorder, out: []const u8, exit_code: ?u64, lost: u64, ns: u64, counters: []const zt.perf.Counter, counts: []const u64) !void {
const events = rec.count;
const moments = rec.moments + @intFromBool(rec.delta.items.len > 0);
try rec.close();
const size = (try std.Io.Dir.cwd().statFile(init.io, out, .{})).size;
- for (options.counters, session.counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c });
- std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes, {d:.1} ms, {d:.0} events/s, exit {d}\n", .{
- out, moments,
- events, session.lost,
- size, @as(f64, @floatFromInt(ns)) / 1e6,
- @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9), exit_code,
+ const code = exit_code orelse fail("{s} is full at {d} bytes ({d} events) and the program was stopped: record fewer kinds of event, or allow more with --max-size BYTES\n", .{ out, size, events });
+ for (counters, counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c });
+ std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes ({d:.1} an event), {d:.1} ms, {d:.0} events/s, exit {d}\n", .{
+ out, moments,
+ events, lost,
+ size, @as(f64, @floatFromInt(size)) / @as(f64, @floatFromInt(@max(events, 1))),
+ @as(f64, @floatFromInt(ns)) / 1e6, @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9),
+ code,
});
}
diff --git a/src/cli/probe.zig b/src/cli/probe.zig
new file mode 100644
index 0000000..f0e2d12
--- /dev/null
+++ b/src/cli/probe.zig
@@ -0,0 +1,215 @@
+//! ztrace probe: BPF programs in the kernel watching a program: its
+//! syscalls, its scheduling, any kernel or user function, static probes.
+//! For root; --dry-run shows what it would do without being it.
+const std = @import("std");
+const linux = std.os.linux;
+const zt = @import("ztrace");
+const util = @import("util.zig");
+const perf = @import("perf.zig");
+const fail = util.fail;
+const bpf = zt.bpf;
+
+pub const usage =
+ \\usage: ztrace probe [-o FILE] [OPTIONS] -- PROGRAM [ARGS]
+ \\
+ \\Runs PROGRAM at full speed with BPF programs attached for it in the
+ \\kernel, into a recording (default out.ztrace). Needs CAP_BPF and
+ \\CAP_PERFMON, which root has.
+ \\
+ \\ --syscalls every syscall, with the registers at its entry
+ \\ --sched every switch of a CPU to or from one of its threads
+ \\ --kprobe FUNCTION
+ \\ every call of a kernel function it makes: five arguments
+ \\ --uprobe [FILE:]SYMBOL
+ \\ every call of a function of its own, or of a library
+ \\ given by its file: five arguments, the registers
+ \\ --usdt [FILE:]PROVIDER:NAME
+ \\ a static probe (<sys/sdt.h>), with its own arguments
+ \\ --dry-run build the programs, find the places, print them; attach nothing
+ \\ --max-size BYTES stop when the recording is this large (default 1 GiB)
+ \\ --ring BYTES the ring the programs write to (a power of two, default 4 MiB)
+ \\
+;
+
+const Probe = struct { kind: enum { kprobe, uprobe, usdt }, what: [:0]const u8 };
+
+pub fn run(init: std.process.Init, args: []const [:0]const u8) !void {
+ const arena = init.arena.allocator();
+ var out: []const u8 = "out.ztrace";
+ var syscalls = false;
+ var sched = false;
+ var dry = false;
+ var max_bytes: ?u64 = null;
+ var ring_bytes: u32 = 1 << 22;
+ var probes: std.ArrayList(Probe) = .empty;
+ var i: usize = 0;
+ while (i < args.len) : (i += 1) {
+ const arg = args[i];
+ const value: ?[:0]const u8 = if (i + 1 < args.len) args[i + 1] else null;
+ if (std.mem.eql(u8, arg, "--")) {
+ i += 1;
+ break;
+ } else if (std.mem.eql(u8, arg, "--syscalls")) {
+ syscalls = true;
+ } else if (std.mem.eql(u8, arg, "--sched")) {
+ sched = true;
+ } else if (std.mem.eql(u8, arg, "--dry-run")) {
+ dry = true;
+ } else if (value == null) {
+ fail(usage, .{});
+ } else if (std.mem.eql(u8, arg, "-o")) {
+ out = value.?;
+ i += 1;
+ } else if (std.mem.eql(u8, arg, "--max-size")) {
+ max_bytes = std.fmt.parseInt(u64, value.?, 0) catch fail("--max-size needs bytes\n", .{});
+ i += 1;
+ } else if (std.mem.eql(u8, arg, "--ring")) {
+ ring_bytes = std.fmt.parseInt(u32, value.?, 0) catch 0;
+ if (!std.math.isPowerOfTwo(ring_bytes) or ring_bytes < 4096) fail("--ring needs a power of two, 4096 or more\n", .{});
+ i += 1;
+ } else if (std.meta.stringToEnum(@FieldType(Probe, "kind"), arg[@min(arg.len, 2)..])) |kind| {
+ try probes.append(arena, .{ .kind = kind, .what = value.? });
+ i += 1;
+ } else fail(usage, .{});
+ }
+ if (i >= args.len) fail(usage, .{});
+ const path = std.mem.span(try util.resolve(init, args[i]));
+
+ // Everything that can be wrong without the kernel's say is found first.
+ var places: std.ArrayList(Place) = .empty;
+ for (probes.items) |p| try places.append(arena, try place(arena, p, path));
+ var offsets: [2]u32 = .{ 0, 0 };
+ var btf: []const u8 = "";
+ if (sched) {
+ btf = std.Io.Dir.cwd().readFileAlloc(init.io, "/sys/kernel/btf/vmlinux", arena, .unlimited) catch fail("--sched needs the kernel's type information, and /sys/kernel/btf/vmlinux cannot be read\n", .{});
+ offsets = .{
+ bpf.fieldOffset(btf, "task_struct", "pid") orelse fail("no task_struct.pid in the kernel's BTF\n", .{}),
+ bpf.fieldOffset(btf, "task_struct", "tgid") orelse fail("no task_struct.tgid in the kernel's BTF\n", .{}),
+ };
+ }
+ var why: [512]u8 = undefined;
+ const missing = bpf.missing(&why);
+
+ if (dry) {
+ // The process id and the maps are not there yet: 0 stands for them.
+ const maps: bpf.Maps = .{ .ring = 0, .drops = 0 };
+ std.debug.print("for the process started from {s}:\n", .{path});
+ if (syscalls) {
+ list("raw tracepoint sys_enter: a syscall_enter and the registers", bpf.sysEnter(maps, 0));
+ list("raw tracepoint sys_exit: a syscall_exit", bpf.sysExit(maps, 0));
+ }
+ if (sched) {
+ std.debug.print("task_struct.pid is at {d}, .tgid at {d} (BTF)\n", .{ offsets[0], offsets[1] });
+ list("raw tracepoint sched_switch: a sched", bpf.schedSwitch(maps, 0, offsets[0], offsets[1]));
+ }
+ for (places.items) |p| {
+ std.debug.print("{t} {s}: {s} +{x}", .{ p.kind, p.name, p.target, p.offset });
+ if (p.usdt) |u| std.debug.print(", semaphore at +{x}, arguments {s}", .{ if (u.semaphore == 0) 0 else p.elf.?.offset(u.semaphore) orelse 0, u.args });
+ std.debug.print("\n", .{});
+ const id = zt.perf.stringId(p.name);
+ list("its program: a probe and the registers", if (p.usdt) |u| bpf.usdtProgram(maps, 0, id, u.args) catch fail("{s}: an argument of this probe ({s}) is in a form not understood yet\n", .{ p.name, u.args }) else bpf.probe(maps, 0, id, null));
+ }
+ std.debug.print("{s}\n", .{if (missing) |m| m else "nothing is missing to attach them"});
+ return;
+ }
+ if (missing) |m| fail("ztrace probe: {s}\n", .{m});
+
+ const argv = try util.targetArgv(arena, args[i..]);
+ var proc = try zt.ptrace.Process.spawn(path, argv, init.minimal.environ.block.slice.ptr);
+ errdefer proc.kill();
+ var rec = try zt.Recorder.create(init.gpa, init.io, out);
+ if (max_bytes) |bytes| rec.max_bytes = bytes;
+ // Two sources, the programs' ring and perf's: one sequence in the recording.
+ var sorted: zt.perf.Sorted = .{ .gpa = init.gpa, .to = rec.sink() };
+ defer sorted.deinit();
+ var tracer = bpf.Tracer.open(init.gpa, proc.pid, sorted.sink(), ring_bytes) catch |err| refused(err, null);
+ defer tracer.close();
+ if (syscalls) tracer.syscalls() catch |err| refused(err, &tracer);
+ if (sched) tracer.sched(btf) catch |err| refused(err, &tracer);
+ for (places.items) |p| {
+ const id = zt.perf.stringId(p.name);
+ try sorted.sink().emit(.{ .ts = zt.event.now(), .kind = .string, .a = .{ id, 0, 0, 0, 0, 0 } }, p.name);
+ (if (p.usdt) |u| tracer.usdtProbe(p.elf.?, p.target, u, id) else switch (p.kind) {
+ .kprobe => tracer.kprobe(p.target, id),
+ else => tracer.uprobe(p.target, p.offset, id),
+ }) catch |err| switch (err) {
+ error.NoSuchProbe => fail("{s}: the kernel has no such place to probe\n", .{p.name}),
+ error.BadUsdtArg => fail("{s}: an argument of this probe ({s}) is in a form not understood yet\n", .{ p.name, p.usdt.?.args }),
+ else => |e| refused(e, &tracer),
+ };
+ }
+ // The maps and the threads, which no program here reports.
+ var session = try zt.perf.Session.open(init.gpa, proc.pid, .{}, sorted.sink());
+ defer session.close();
+
+ zt.perf.release(proc);
+ const start = zt.event.now();
+ const exit_code: ?u64 = pump(&tracer, &session, &sorted) catch |err| switch (err) {
+ error.RecordingFull => null,
+ else => |e| return e,
+ };
+ if (exit_code == null) proc.kill();
+ try perf.finish(init, &rec, out, exit_code, tracer.lost() + session.lost, zt.event.now() - start, &.{}, &.{});
+}
+
+fn pump(tracer: *bpf.Tracer, session: *zt.perf.Session, sorted: *zt.perf.Sorted) !u64 {
+ // What both sources have reported past: the start of the round before.
+ var past: u64 = 0;
+ while (true) {
+ const now = zt.event.now();
+ const alive = try session.poll(0);
+ const records = try tracer.poll(if (alive) 20 else 0);
+ try sorted.flush(past);
+ past = now;
+ if (records == 0 and !alive) break;
+ }
+ const code = try perf.exited(tracer.pid, sorted.sink());
+ try sorted.flush(std.math.maxInt(u64));
+ return code;
+}
+
+fn refused(err: anyerror, tracer: ?*bpf.Tracer) noreturn {
+ const log = if (tracer) |t| std.mem.sliceTo(t.log, 0) else "";
+ switch (err) {
+ error.PermissionDenied => fail("ztrace probe: bpf() refused though the capabilities are there: inside a user namespace or a container they do not count, the kernel wants them in the initial namespace\n", .{}),
+ error.UnsafeProgram, error.InvalidProgram => fail("ztrace probe: the kernel's verifier refused a program ({t}):\n{s}\n", .{ err, log }),
+ else => fail("ztrace probe: {t}\n{s}", .{ err, log }),
+ }
+}
+
+/// Where a probe goes: a kernel function, or an offset into a file.
+const Place = struct {
+ kind: @FieldType(Probe, "kind"),
+ name: []const u8,
+ target: [:0]const u8,
+ offset: u64 = 0,
+ elf: ?zt.usdt.Elf = null,
+ usdt: ?zt.usdt.Probe = null,
+};
+
+fn place(arena: std.mem.Allocator, p: Probe, program: [:0]const u8) !Place {
+ if (p.kind == .kprobe) return .{ .kind = .kprobe, .name = p.what, .target = p.what };
+ // FILE: in front, or the program itself.
+ const colons = std.mem.count(u8, p.what, ":");
+ const named = colons == @as(usize, if (p.kind == .usdt) 2 else 1);
+ const split = if (named) std.mem.indexOfScalar(u8, p.what, ':').? else 0;
+ const file = if (named) try arena.dupeSentinel(u8, p.what[0..split], 0) else program;
+ const name = p.what[if (named) split + 1 else 0..];
+ const elf = zt.usdt.Elf.open(file) catch fail("{s}: not an ELF file that can be read\n", .{file});
+ if (p.kind == .uprobe) {
+ const sym = elf.symbol(name) orelse fail("{s}: no symbol {s}\n", .{ file, name });
+ return .{ .kind = .uprobe, .name = name, .target = file, .offset = elf.offset(sym.value) orelse fail("{s}: {s} is not in the file\n", .{ file, name }), .elf = elf };
+ }
+ for (try zt.usdt.probes(arena, elf, 0)) |found| {
+ const colon = std.mem.indexOfScalar(u8, name, ':') orelse fail("--usdt wants [FILE:]PROVIDER:NAME\n", .{});
+ if (!std.mem.eql(u8, found.provider, name[0..colon]) or !std.mem.eql(u8, found.name, name[colon + 1 ..])) continue;
+ return .{ .kind = .usdt, .name = name, .target = file, .offset = elf.offset(found.pc) orelse continue, .elf = elf, .usdt = found };
+ }
+ fail("{s}: no static probe {s} (readelf -n lists them)\n", .{ file, name });
+}
+
+/// A program as the verifier numbers it: opcode, registers, offset, immediate.
+fn list(what: []const u8, program: bpf.Program) void {
+ std.debug.print("{s}, {d} instructions\n", .{ what, program.len });
+ for (program.code(), 0..) |insn, at| std.debug.print(" {d:>3}: {x:0>2} r{d} r{d} {d:>5} {d}\n", .{ at, insn.code, insn.dst, insn.src, insn.off, insn.imm });
+}
diff --git a/src/trace/bpf.zig b/src/trace/bpf.zig
new file mode 100644
index 0000000..4069b01
--- /dev/null
+++ b/src/trace/bpf.zig
@@ -0,0 +1,733 @@
+//! The kernel's view with privilege: BPF programs on the raw tracepoints
+//! sys_enter, sys_exit and sched_switch, and on any kprobe, uprobe or USDT
+//! probe. Needs CAP_BPF and CAP_PERFMON (root). Each program writes an
+//! `Event`, and for a syscall's entry and a probe the registers after it,
+//! into one BPF ring buffer; `Tracer.poll` moves them into a `Sink`.
+//!
+//! The programs are instruction arrays built here, for one process (its id
+//! is a constant in them): no compiler, no object file, no libbpf. What the
+//! kernel does not tell (where `pid` is in `task_struct`) comes from its BTF.
+const std = @import("std");
+const linux = std.os.linux;
+const BPF = linux.BPF;
+const Insn = BPF.Insn;
+const event = @import("event.zig");
+const usdt = @import("usdt.zig");
+const Event = event.Event;
+const Regs = event.Regs;
+const Kind = event.Kind;
+
+/// The kernel's `struct pt_regs` is `Regs` up to here.
+const pt_regs_bytes = @offsetOf(Regs, "fs_base");
+/// On a program's stack, from the frame pointer down: the key of the drop
+/// counter, then the record. An event alone, or an event and registers.
+const key_at = -8;
+const small_at = key_at - @sizeOf(Event);
+const large_at = small_at - pt_regs_bytes;
+
+fn field(comptime name: []const u8) i16 {
+ return @offsetOf(Event, name);
+}
+
+fn arg(i: usize) i16 {
+ return @intCast(@offsetOf(Event, "a") + 8 * i);
+}
+
+fn reg(comptime name: []const u8) i16 {
+ return @offsetOf(Regs, name);
+}
+
+pub const Program = struct {
+ insns: [192]Insn = undefined,
+ len: usize = 0,
+ /// The jump that leaves early, to be pointed at the end.
+ leave: ?usize = null,
+
+ pub fn code(p: *const Program) []const Insn {
+ return p.insns[0..p.len];
+ }
+
+ fn add(p: *Program, insn: Insn) void {
+ p.insns[p.len] = insn;
+ p.len += 1;
+ }
+
+ /// r6 = the context, r7 = thread and process id; goes on only in a
+ /// thread of process `tgid`.
+ fn enter(p: *Program, tgid: u32) void {
+ p.add(Insn.mov(.r6, .r1));
+ p.add(Insn.call(.get_current_pid_tgid));
+ p.add(Insn.mov(.r7, .r0));
+ p.add(Insn.rsh(.r0, 32));
+ p.leave = p.len;
+ p.add(Insn.jne(.r0, @as(i32, @intCast(tgid)), 0));
+ }
+
+ /// The time, the thread and the kind of the event at `at`.
+ fn stamp(p: *Program, at: i16, kind: Kind) void {
+ p.add(Insn.call(.ktime_get_ns));
+ p.add(Insn.stx(.double_word, .r10, at + field("ts"), .r0));
+ p.add(Insn.stx(.word, .r10, at + field("tid"), .r7));
+ p.add(Insn.st(.half_word, .r10, at + field("kind"), @backingInt(kind)));
+ }
+
+ /// bpf_probe_read_kernel(frame + `to`, `size`, r3)
+ fn read(p: *Program, to: i16, size: i32) void {
+ p.add(Insn.mov(.r1, .r10));
+ p.add(Insn.add(.r1, to));
+ p.add(Insn.mov(.r2, size));
+ p.add(Insn.call(.probe_read_kernel));
+ }
+
+ /// The low `bytes` of a register, extended to all of it.
+ fn extend(p: *Program, r: Insn.Reg, bytes: u8, signed: bool) void {
+ if (bytes >= 8) return;
+ const shift: i32 = 64 - 8 * @as(i32, bytes);
+ p.add(Insn.lsh(r, shift));
+ p.add(if (signed) Insn.arsh(r, shift) else Insn.rsh(r, shift));
+ }
+
+ /// A register of the probed thread, from the context in r6.
+ fn probed(p: *Program, to: Insn.Reg, r: anytype) void {
+ p.add(Insn.ldx(.double_word, to, .r6, @as(i16, r.slot) * 8));
+ p.extend(to, r.bytes, false);
+ }
+
+ /// One argument of a USDT probe into the u64 at `to`: what
+ /// `usdt.Arg.fetch` computes, in the kernel, at the probe. Memory that
+ /// cannot be read gives 0.
+ fn usdtArg(p: *Program, a: usdt.Arg, to: i16) void {
+ switch (a.at) {
+ .constant => |c| {
+ p.add(Insn.ld_dw1(.r1, @bitCast(c)));
+ p.add(Insn.ld_dw2(@bitCast(c)));
+ },
+ .register => |r| p.probed(.r1, r),
+ .memory => |m| {
+ p.add(Insn.st(.double_word, .r10, to, 0));
+ p.probed(.r3, m.base);
+ if (m.index) |index| {
+ p.probed(.r1, index);
+ p.add(Insn.lsh(.r1, @as(i32, @ctz(m.scale))));
+ p.add(Insn.add(.r3, .r1));
+ }
+ p.add(Insn.ld_dw1(.r1, @bitCast(m.disp)));
+ p.add(Insn.ld_dw2(@bitCast(m.disp)));
+ p.add(Insn.add(.r3, .r1));
+ p.add(Insn.mov(.r1, .r10));
+ p.add(Insn.add(.r1, to));
+ p.add(Insn.mov(.r2, @as(i32, a.size)));
+ p.add(Insn.call(.probe_read_user));
+ p.add(Insn.ldx(.double_word, .r1, .r10, to));
+ },
+ }
+ p.extend(.r1, a.size, a.signed);
+ p.add(Insn.stx(.double_word, .r10, to, .r1));
+ }
+
+ /// Sends the `size` bytes at `at`, counts the record in `drops` when the
+ /// ring is full, and ends the program.
+ fn send(p: *Program, maps: Maps, at: i16, size: i32) void {
+ p.add(Insn.ld_map_fd1(.r1, maps.ring));
+ p.add(Insn.ld_map_fd2(maps.ring));
+ p.add(Insn.mov(.r2, .r10));
+ p.add(Insn.add(.r2, at));
+ p.add(Insn.mov(.r3, size));
+ p.add(Insn.mov(.r4, 0));
+ p.add(Insn.call(.ringbuf_output));
+ p.add(Insn.jeq(.r0, 0, 9));
+ p.add(Insn.st(.word, .r10, key_at, 0));
+ p.add(Insn.ld_map_fd1(.r1, maps.drops));
+ p.add(Insn.ld_map_fd2(maps.drops));
+ p.add(Insn.mov(.r2, .r10));
+ p.add(Insn.add(.r2, key_at));
+ p.add(Insn.call(.map_lookup_elem));
+ p.add(Insn.jeq(.r0, 0, 2));
+ p.add(Insn.mov(.r1, 1));
+ p.add(Insn.xadd(.r0, .r1));
+ if (p.leave) |at_jump| p.insns[at_jump].off = @intCast(p.len - at_jump - 1);
+ p.add(Insn.mov(.r0, 0));
+ p.add(Insn.exit());
+ }
+};
+
+/// The maps every program writes to: the ring of records, and an array of
+/// one u64 counting the records that found it full.
+pub const Maps = struct { ring: linux.fd_t, drops: linux.fd_t };
+
+/// raw tracepoint sys_enter(struct pt_regs *regs, long id): a
+/// `syscall_enter` and the registers.
+pub fn sysEnter(maps: Maps, tgid: u32) Program {
+ var p: Program = .{};
+ p.enter(tgid);
+ p.add(Insn.ldx(.double_word, .r3, .r6, 0));
+ p.read(large_at + @sizeOf(Event), pt_regs_bytes);
+ p.stamp(large_at, .syscall_enter);
+ p.add(Insn.ldx(.double_word, .r1, .r6, 8));
+ p.add(Insn.stx(.half_word, .r10, large_at + field("aux"), .r1));
+ inline for (.{ "rdi", "rsi", "rdx", "r10", "r8", "r9" }, 0..) |name, i| {
+ p.add(Insn.ldx(.double_word, .r1, .r10, large_at + @sizeOf(Event) + reg(name)));
+ p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1));
+ }
+ p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes);
+ return p;
+}
+
+/// raw tracepoint sys_exit(struct pt_regs *regs, long ret): a `syscall_exit`.
+pub fn sysExit(maps: Maps, tgid: u32) Program {
+ var p: Program = .{};
+ const number_at = small_at - 8;
+ p.enter(tgid);
+ p.add(Insn.ldx(.double_word, .r3, .r6, 0));
+ p.add(Insn.add(.r3, reg("orig_rax")));
+ p.read(number_at, 8);
+ p.stamp(small_at, .syscall_exit);
+ p.add(Insn.ldx(.double_word, .r1, .r10, number_at));
+ p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
+ p.add(Insn.ldx(.double_word, .r1, .r6, 8));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r1));
+ for (1..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0));
+ p.send(maps, small_at, @sizeOf(Event));
+ return p;
+}
+
+/// raw tracepoint sched_switch(bool preempt, struct task_struct *prev,
+/// *next): a `sched` when either is a thread of the process. `pid_at` and
+/// `tgid_at` are where those fields are in `task_struct`.
+pub fn schedSwitch(maps: Maps, tgid: u32, pid_at: u32, tgid_at: u32) Program {
+ var p: Program = .{};
+ // prev's pid and tgid, next's pid and tgid, as u32 below the event.
+ const ids_at = small_at - 16;
+ p.add(Insn.mov(.r6, .r1));
+ for ([_]i16{ 8, 16 }, 0..) |task, t| for ([_]u32{ pid_at, tgid_at }, 0..) |at, i| {
+ p.add(Insn.ldx(.double_word, .r3, .r6, task));
+ p.add(Insn.add(.r3, @as(i32, @intCast(at))));
+ p.read(ids_at + @as(i16, @intCast(8 * t + 4 * i)), 4);
+ };
+ p.add(Insn.ldx(.word, .r1, .r10, ids_at + 4));
+ p.add(Insn.jeq(.r1, @as(i32, @intCast(tgid)), 2));
+ p.add(Insn.ldx(.word, .r1, .r10, ids_at + 12));
+ p.leave = p.len;
+ p.add(Insn.jne(.r1, @as(i32, @intCast(tgid)), 0));
+ p.add(Insn.ldx(.word, .r7, .r10, ids_at));
+ p.stamp(small_at, .sched);
+ p.add(Insn.ldx(.double_word, .r1, .r6, 0));
+ p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r7));
+ p.add(Insn.ldx(.word, .r1, .r10, ids_at + 8));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(1), .r1));
+ p.add(Insn.call(.get_smp_processor_id));
+ p.add(Insn.stx(.double_word, .r10, small_at + arg(2), .r0));
+ for (3..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0));
+ p.send(maps, small_at, @sizeOf(Event));
+ return p;
+}
+
+/// A kprobe or uprobe (the context is the registers there): a `probe`
+/// named `id`, and the registers. Its a1..a5 are the first five arguments
+/// of a function call, or with `args` those of a USDT probe, read there
+/// and then.
+pub fn probe(maps: Maps, tgid: u32, id: u64, args: ?[]const usdt.Arg) Program {
+ var p: Program = .{};
+ p.enter(tgid);
+ p.stamp(large_at, .probe);
+ p.add(Insn.st(.half_word, .r10, large_at + field("aux"), 0));
+ p.add(Insn.ld_dw1(.r1, id));
+ p.add(Insn.ld_dw2(id));
+ p.add(Insn.stx(.double_word, .r10, large_at + arg(0), .r1));
+ inline for (.{ "rdi", "rsi", "rdx", "rcx", "r8" }, 1..) |name, i| {
+ if (args == null) {
+ p.add(Insn.ldx(.double_word, .r1, .r6, reg(name)));
+ p.add(Insn.stx(.double_word, .r10, large_at + arg(i), .r1));
+ } else if (i <= args.?.len) {
+ p.usdtArg(args.?[i - 1], large_at + arg(i));
+ } else p.add(Insn.st(.double_word, .r10, large_at + arg(i), 0));
+ }
+ for (0..pt_regs_bytes / 8) |i| {
+ p.add(Insn.ldx(.double_word, .r1, .r6, @intCast(8 * i)));
+ p.add(Insn.stx(.double_word, .r10, large_at + @sizeOf(Event) + @as(i16, @intCast(8 * i)), .r1));
+ }
+ p.send(maps, large_at, @sizeOf(Event) + pt_regs_bytes);
+ return p;
+}
+
+/// `probe` for a USDT probe with the arguments `specs` ("-4@%edi
+/// 8@-24(%rbp)"): the first five become a1..a5.
+pub fn usdtProgram(maps: Maps, tgid: u32, id: u64, specs: []const u8) !Program {
+ var args: [5]usdt.Arg = undefined;
+ var count: usize = 0;
+ var each = std.mem.tokenizeScalar(u8, specs, ' ');
+ while (each.next()) |spec| {
+ if (count == args.len) break;
+ args[count] = try .parse(spec);
+ count += 1;
+ }
+ return probe(maps, tgid, id, args[0..count]);
+}
+
+/// Where `member` is in the struct `name`, in bytes, from the kernel's
+/// type information (the contents of /sys/kernel/btf/vmlinux).
+pub fn fieldOffset(btf: []const u8, name: []const u8, member: []const u8) ?u32 {
+ const T = BPF.btf;
+ if (btf.len < @sizeOf(T.Header)) return null;
+ const header = std.mem.bytesToValue(T.Header, btf[0..@sizeOf(T.Header)]);
+ if (header.magic != T.magic or header.hdr_len + @as(u64, header.str_off) + header.str_len > btf.len) return null;
+ const types = btf[header.hdr_len + header.type_off ..][0..header.type_len];
+ const strings = btf[header.hdr_len + header.str_off ..][0..header.str_len];
+ var at: usize = 0;
+ while (at + @sizeOf(T.Type) <= types.len) {
+ const t = std.mem.bytesToValue(T.Type, types[at..][0..@sizeOf(T.Type)]);
+ at += @sizeOf(T.Type);
+ const count: usize = t.info.vlen;
+ // What follows the type, by kind.
+ const extra: usize = switch (t.info.kind) {
+ .int, .@"var", .decl_tag => 4,
+ .array => 12,
+ .@"struct", .@"union", .datasec, .enum64 => 12 * count,
+ .@"enum", .func_proto => 8 * count,
+ else => 0,
+ };
+ if (at + extra > types.len) return null;
+ defer at += extra;
+ if (t.info.kind != .@"struct" or !std.mem.eql(u8, std.mem.sliceTo(strings[@min(t.name_off, strings.len)..], 0), name)) continue;
+ for (0..count) |i| {
+ const m = std.mem.bytesToValue(T.Member, types[at + 12 * i ..][0..12]);
+ if (std.mem.eql(u8, std.mem.sliceTo(strings[@min(m.name_off, strings.len)..], 0), member)) return m.offset.bit / 8;
+ }
+ }
+ return null;
+}
+
+/// The reading end of a BPF ring buffer: a page with our position, a page
+/// with the kernel's, and the data, which the kernel maps twice in a row so
+/// that no record wraps. A record is a u32 length with two flag bits, four
+/// bytes more, the bytes, padded to 8.
+pub const RingBuf = struct {
+ consumer: *u64,
+ producer: *const u64,
+ data: []const u8,
+ /// Read so far; the kernel is told by `commit`, after the records were used.
+ at: u64 = 0,
+
+ const busy: u32 = 1 << 31;
+ const discarded: u32 = 1 << 30;
+
+ pub fn map(fd: linux.fd_t, size: usize) !RingBuf {
+ const ours = linux.mmap(null, 4096, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0);
+ if (linux.errno(ours) != .SUCCESS) return error.BpfRingMap;
+ const theirs = linux.mmap(null, 4096 + 2 * size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 4096);
+ if (linux.errno(theirs) != .SUCCESS) return error.BpfRingMap;
+ return .{ .consumer = @ptrFromInt(ours), .producer = @ptrFromInt(theirs), .data = @as([*]const u8, @ptrFromInt(theirs + 4096))[0..size] };
+ }
+
+ /// The next finished record, valid until `commit`.
+ pub fn next(r: *RingBuf) ?[]const u8 {
+ while (r.at < @atomicLoad(u64, r.producer, .acquire)) {
+ const start: usize = @intCast(r.at & (r.data.len - 1));
+ const header = @atomicLoad(u32, @as(*const u32, @ptrCast(@alignCast(r.data.ptr + start))), .acquire);
+ if (header & busy != 0) return null;
+ const len = header & ~(busy | discarded);
+ r.at += std.mem.alignForward(u64, len + 8, 8);
+ if (header & discarded == 0) return (r.data.ptr + start + 8)[0..len];
+ }
+ return null;
+ }
+
+ pub fn commit(r: *RingBuf) void {
+ @atomicStore(u64, r.consumer, r.at, .release);
+ }
+};
+
+/// The capability bits of this process (CapEff of /proc/self/status).
+fn capabilities() u64 {
+ var buf: [4096]u8 = undefined;
+ const fd = linux.open("/proc/self/status", .{}, 0);
+ if (linux.errno(fd) != .SUCCESS) return 0;
+ defer _ = linux.close(@intCast(fd));
+ const n = linux.read(@intCast(fd), &buf, buf.len);
+ if (linux.errno(n) != .SUCCESS) return 0;
+ const label = "CapEff:\t";
+ const at = std.mem.indexOf(u8, buf[0..n], label) orelse return 0;
+ return std.fmt.parseInt(u64, buf[at + label.len ..][0..16], 16) catch 0;
+}
+
+/// What this process lacks to trace with BPF, said into `buf`; null when
+/// it has the capabilities (which inside a user namespace still is not
+/// enough: the kernel wants them in the initial one).
+pub fn missing(buf: []u8) ?[]const u8 {
+ const caps = capabilities();
+ const has = struct {
+ fn has(all: u64, cap: u6) bool {
+ return all >> cap & 1 != 0;
+ }
+ }.has;
+ if (has(caps, linux.CAP.SYS_ADMIN) or (has(caps, linux.CAP.BPF) and has(caps, linux.CAP.PERFMON))) return null;
+ return std.fmt.bufPrint(buf, "missing {s}{s}{s} (effective capabilities {x:0>16}, uid {d}). " ++
+ "CAP_BPF is what bpf() wants to create maps and load programs, CAP_PERFMON to load tracing programs and open kprobes and uprobes; CAP_SYS_ADMIN stands for both. " ++
+ "Run as root, or give the binary the two: setcap cap_bpf,cap_perfmon+ep ztrace", .{
+ if (has(caps, linux.CAP.BPF)) "" else "CAP_BPF",
+ if (has(caps, linux.CAP.BPF) or has(caps, linux.CAP.PERFMON)) "" else " and ",
+ if (has(caps, linux.CAP.PERFMON)) "" else "CAP_PERFMON",
+ caps,
+ linux.getuid(),
+ }) catch null;
+}
+
+pub const Tracer = struct {
+ gpa: std.mem.Allocator,
+ /// The process traced: its thread group id, in the initial pid namespace.
+ /// shortcut: one process, and a tracer in a pid namespace of its own sees nothing; keep the ids in a BPF map to follow children.
+ pid: linux.pid_t,
+ sink: event.Sink,
+ maps: Maps,
+ ring: RingBuf,
+ /// Programs and what attaches them.
+ fds: std.ArrayList(linux.fd_t) = .empty,
+ /// What the verifier said about the last program, when it refused one.
+ log: []u8,
+
+ pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, sink: event.Sink, ring_bytes: u32) !Tracer {
+ const ring = try BPF.map_create(.ringbuf, 0, 0, ring_bytes);
+ errdefer _ = linux.close(ring);
+ const drops = try BPF.map_create(.array, 4, 8, 1);
+ errdefer _ = linux.close(drops);
+ const ring_map = try RingBuf.map(ring, ring_bytes);
+ const log = try gpa.alloc(u8, 1 << 16);
+ log[0] = 0;
+ return .{ .gpa = gpa, .pid = pid, .sink = sink, .maps = .{ .ring = ring, .drops = drops }, .ring = ring_map, .log = log };
+ }
+
+ pub fn close(t: *Tracer) void {
+ for (t.fds.items) |fd| _ = linux.close(fd);
+ t.fds.deinit(t.gpa);
+ _ = linux.munmap(@ptrCast(@alignCast(t.ring.consumer)), 4096);
+ _ = linux.munmap(@ptrCast(@alignCast(t.ring.producer)), 4096 + 2 * t.ring.data.len);
+ _ = linux.close(t.maps.ring);
+ _ = linux.close(t.maps.drops);
+ t.gpa.free(t.log);
+ }
+
+ fn load(t: *Tracer, kind: BPF.ProgType, program: Program) !linux.fd_t {
+ // GPL: bpf_probe_read_kernel is for programs that say so.
+ const fd = BPF.prog_load(kind, program.code(), null, "GPL", 0, 0) catch |err| {
+ // Again for the verifier's reasons, which a good program need not pay for.
+ var log: BPF.Log = .{ .level = 1, .buf = t.log };
+ if (BPF.prog_load(kind, program.code(), &log, "GPL", 0, 0)) |again| _ = linux.close(again) else |_| {}
+ return err;
+ };
+ try t.fds.append(t.gpa, fd);
+ return fd;
+ }
+
+ fn rawTracepoint(t: *Tracer, name: [*:0]const u8, program: Program) !void {
+ var attr = std.mem.zeroes(BPF.Attr);
+ attr.raw_tracepoint = .{ .name = @intFromPtr(name), .prog_fd = try t.load(.raw_tracepoint, program) };
+ const rc = linux.bpf(.raw_tracepoint_open, &attr, @sizeOf(BPF.RawTracepointAttr));
+ if (linux.errno(rc) != .SUCCESS) return error.BpfAttach;
+ try t.fds.append(t.gpa, @intCast(rc));
+ }
+
+ /// Every syscall of the process.
+ pub fn syscalls(t: *Tracer) !void {
+ try t.rawTracepoint("sys_enter", sysEnter(t.maps, @intCast(t.pid)));
+ try t.rawTracepoint("sys_exit", sysExit(t.maps, @intCast(t.pid)));
+ }
+
+ /// Every time one of its threads leaves or gets a CPU. `btf` is the
+ /// contents of /sys/kernel/btf/vmlinux.
+ pub fn sched(t: *Tracer, btf: []const u8) !void {
+ const pid_at = fieldOffset(btf, "task_struct", "pid") orelse return error.NoBtf;
+ const tgid_at = fieldOffset(btf, "task_struct", "tgid") orelse return error.NoBtf;
+ try t.rawTracepoint("sched_switch", schedSwitch(t.maps, @intCast(t.pid), pid_at, tgid_at));
+ }
+
+ /// A probe through the perf event of the kprobe or uprobe PMU: `target`
+ /// is a kernel function or a file, `offset` is into it.
+ fn attach(t: *Tracer, pmu: []const u8, target: [*:0]const u8, offset: u64, semaphore: u64, program: Program) !void {
+ var path_buf: [64]u8 = undefined;
+ var number: [16]u8 = undefined;
+ const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/sys/bus/event_source/devices/{s}/type", .{pmu}, 0), .{}, 0);
+ if (linux.errno(fd) != .SUCCESS) return error.NoProbeSupport;
+ defer _ = linux.close(@intCast(fd));
+ const n = linux.read(@intCast(fd), &number, number.len);
+ if (linux.errno(n) != .SUCCESS) return error.NoProbeSupport;
+ const kind = std.fmt.parseInt(u32, std.mem.trim(u8, number[0..n], "\n "), 10) catch return error.NoProbeSupport;
+
+ // A uprobe's semaphore (the offset in the file of its counter) rides in the upper half of config.
+ // On the first thread alone, and not inherited: a program on a probe
+ // runs for whoever hits it (hence the process id in ours), and the
+ // kernel reads `target` again, from the target's memory, for an
+ // event a clone would inherit, which fails the clone.
+ var attr: linux.perf_event_attr = .{ .type = @fromBackingInt(@intCast(kind)), .config = semaphore << 32, .config1 = @intFromPtr(target), .config2 = offset };
+ const event_fd = linux.perf_event_open(&attr, t.pid, -1, -1, linux.PERF.FLAG.FD_CLOEXEC);
+ if (linux.errno(event_fd) != .SUCCESS) return if (linux.errno(event_fd) == .NOENT) error.NoSuchProbe else error.BpfAttach;
+ try t.fds.append(t.gpa, @intCast(event_fd));
+ const loaded = try t.load(.kprobe, program);
+ if (linux.errno(linux.ioctl(@intCast(event_fd), linux.PERF.EVENT_IOC.SET_BPF, @intCast(loaded))) != .SUCCESS) return error.BpfAttach;
+ }
+
+ /// Every call of a kernel function by the process: a `probe` named
+ /// `function` (define the string yourself: its id is `id`).
+ pub fn kprobe(t: *Tracer, function: [*:0]const u8, id: u64) !void {
+ try t.attach("kprobe", function, 0, 0, probe(t.maps, @intCast(t.pid), id, null));
+ }
+
+ /// Every time the process runs the instruction at `offset` of the
+ /// file `path`.
+ pub fn uprobe(t: *Tracer, path: [*:0]const u8, offset: u64, id: u64) !void {
+ try t.attach("uprobe", path, offset, 0, probe(t.maps, @intCast(t.pid), id, null));
+ }
+
+ /// A static probe of the file `path` (`p` from `usdt.probes` with no
+ /// bias). The kernel raises its semaphore while we are attached.
+ pub fn usdtProbe(t: *Tracer, elf: usdt.Elf, path: [*:0]const u8, p: usdt.Probe, id: u64) !void {
+ const semaphore = if (p.semaphore == 0) 0 else elf.offset(p.semaphore) orelse return error.NoSuchProbe;
+ try t.attach("uprobe", path, elf.offset(p.pc) orelse return error.NoSuchProbe, semaphore, try usdtProgram(t.maps, @intCast(t.pid), id, p.args));
+ }
+
+ /// Waits up to the timeout for records and decodes them into the sink.
+ /// Returns how many.
+ pub fn poll(t: *Tracer, timeout_ms: i32) !usize {
+ var fds = [1]linux.pollfd{.{ .fd = t.maps.ring, .events = linux.POLL.IN, .revents = 0 }};
+ _ = linux.poll(&fds, 1, timeout_ms);
+ var n: usize = 0;
+ while (t.ring.next()) |record| : (n += 1) try decode(record, t.sink);
+ t.ring.commit();
+ return n;
+ }
+
+ /// Records that found the ring full.
+ pub fn lost(t: *Tracer) u64 {
+ var count: u64 = 0;
+ BPF.map_lookup_elem(t.maps.drops, &std.mem.toBytes(@as(u32, 0)), std.mem.asBytes(&count)) catch return 0;
+ return count;
+ }
+};
+
+/// A record of the ring into the sink: the event, and the registers when
+/// they follow it.
+fn decode(record: []const u8, sink: event.Sink) !void {
+ if (record.len < @sizeOf(Event)) return;
+ const ev = std.mem.bytesToValue(Event, record[0..@sizeOf(Event)]);
+ if (record.len < @sizeOf(Event) + pt_regs_bytes) return sink.emit(ev, "");
+ var regs: Regs = .{};
+ @memcpy(std.mem.asBytes(&regs)[0..pt_regs_bytes], record[@sizeOf(Event)..][0..pt_regs_bytes]);
+ try sink.emit(ev, std.mem.asBytes(&regs));
+}
+
+// What follows runs the programs without a kernel: enough of a BPF machine
+// for what they use, on this process's own memory.
+const Machine = struct {
+ pid_tgid: u64 = 0,
+ now: u64 = 0,
+ cpu: u64 = 0,
+ /// What bpf_ringbuf_output returns; not 0 is a full ring.
+ ring_full: bool = false,
+ sent: [512]u8 = undefined,
+ sent_len: ?usize = null,
+ drops: u64 = 0,
+
+ fn run(m: *Machine, code: []const Insn, context: *const anyopaque) !void {
+ var stack: [512]u8 align(8) = undefined;
+ var r: [11]u64 = @splat(0);
+ r[1] = @intFromPtr(context);
+ r[10] = @intFromPtr(&stack) + stack.len;
+ var pc: usize = 0;
+ var steps: usize = 0;
+ while (true) : (steps += 1) {
+ if (steps > 10_000 or pc >= code.len) return error.RanAway;
+ const insn = code[pc];
+ pc += 1;
+ const imm: u64 = @bitCast(@as(i64, insn.imm));
+ const source = if (insn.code & BPF.X != 0) r[insn.src] else imm;
+ const address = (if (insn.code & 7 == BPF.LDX) r[insn.src] else r[insn.dst]) +% @as(u64, @bitCast(@as(i64, insn.off)));
+ switch (insn.code) {
+ BPF.ALU64 | BPF.MOV | BPF.K, BPF.ALU64 | BPF.MOV | BPF.X => r[insn.dst] = source,
+ BPF.ALU64 | BPF.ADD | BPF.K, BPF.ALU64 | BPF.ADD | BPF.X => r[insn.dst] +%= source,
+ BPF.ALU64 | BPF.RSH | BPF.K => r[insn.dst] >>= @intCast(insn.imm),
+ BPF.ALU64 | BPF.LSH | BPF.K => r[insn.dst] <<= @intCast(insn.imm),
+ BPF.ALU64 | BPF.ARSH | BPF.K => r[insn.dst] = @bitCast(@as(i64, @bitCast(r[insn.dst])) >> @intCast(insn.imm)),
+ BPF.JMP | BPF.JEQ | BPF.K => if (r[insn.dst] == imm) {
+ pc = @intCast(@as(i64, @intCast(pc)) + insn.off);
+ },
+ BPF.JMP | BPF.JNE | BPF.K => if (r[insn.dst] != imm) {
+ pc = @intCast(@as(i64, @intCast(pc)) + insn.off);
+ },
+ BPF.LD | BPF.DW | BPF.IMM => {
+ r[insn.dst] = @as(u32, @bitCast(insn.imm)) | @as(u64, @as(u32, @bitCast(code[pc].imm))) << 32;
+ pc += 1;
+ },
+ inline BPF.LDX | BPF.MEM | BPF.B, BPF.LDX | BPF.MEM | BPF.H, BPF.LDX | BPF.MEM | BPF.W, BPF.LDX | BPF.MEM | BPF.DW => |code_now| {
+ r[insn.dst] = @as(*align(1) const Width(code_now), @ptrFromInt(address)).*;
+ },
+ inline BPF.STX | BPF.MEM | BPF.H, BPF.STX | BPF.MEM | BPF.W, BPF.STX | BPF.MEM | BPF.DW => |code_now| {
+ @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(r[insn.src]);
+ },
+ inline BPF.ST | BPF.MEM | BPF.H, BPF.ST | BPF.MEM | BPF.W, BPF.ST | BPF.MEM | BPF.DW => |code_now| {
+ @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(imm);
+ },
+ BPF.STX | BPF.XADD | BPF.DW => @as(*u64, @ptrFromInt(address)).* += r[insn.src],
+ BPF.JMP | BPF.CALL => r[0] = switch (@as(BPF.Helper, @fromBackingInt(@intCast(insn.imm)))) {
+ .get_current_pid_tgid => m.pid_tgid,
+ .ktime_get_ns => m.now,
+ .get_smp_processor_id => m.cpu,
+ .probe_read_kernel, .probe_read_user => read: {
+ @memcpy(@as([*]u8, @ptrFromInt(r[1]))[0..r[2]], @as([*]const u8, @ptrFromInt(r[3]))[0..r[2]]);
+ break :read 0;
+ },
+ .ringbuf_output => out: {
+ if (r[1] != 3 or m.ring_full) break :out @as(u64, @bitCast(@as(i64, -11)));
+ m.sent_len = r[3];
+ @memcpy(m.sent[0..r[3]], @as([*]const u8, @ptrFromInt(r[2]))[0..r[3]]);
+ break :out 0;
+ },
+ .map_lookup_elem => if (r[1] == 4 and @as(*const u32, @ptrFromInt(r[2])).* == 0) @intFromPtr(&m.drops) else 0,
+ else => return error.UnknownHelper,
+ },
+ BPF.JMP | BPF.EXIT => return,
+ else => return error.UnknownInstruction,
+ }
+ }
+ }
+
+ fn Width(comptime code: u8) type {
+ return switch (code & 0x18) {
+ BPF.B => u8,
+ BPF.H => u16,
+ BPF.W => u32,
+ else => u64,
+ };
+ }
+
+ fn sentEvent(m: *const Machine) Event {
+ return std.mem.bytesToValue(Event, m.sent[0..@sizeOf(Event)]);
+ }
+};
+
+const test_maps: Maps = .{ .ring = 3, .drops = 4 };
+
+test "the syscall programs: the right process only, the event the tracer expects" {
+ var regs: Regs = .{ .rdi = 1, .rsi = 2, .rdx = 3, .r10 = 4, .r8 = 5, .r9 = 6, .orig_rax = 257, .rip = 0x401000 };
+ const enter_context = [2]u64{ @intFromPtr(&regs), 257 };
+ var m: Machine = .{ .pid_tgid = 1000 << 32 | 1001, .now = 77 };
+ try m.run(sysEnter(test_maps, 1000).code(), &enter_context);
+ try std.testing.expectEqual(@as(?usize, @sizeOf(Event) + pt_regs_bytes), m.sent_len);
+ try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_enter, .aux = 257, .a = .{ 1, 2, 3, 4, 5, 6 } }, m.sentEvent());
+ try std.testing.expectEqualSlices(u8, std.mem.asBytes(&regs)[0..pt_regs_bytes], m.sent[@sizeOf(Event)..][0..pt_regs_bytes]);
+
+ const exit_context = [2]u64{ @intFromPtr(&regs), @bitCast(@as(i64, -2)) };
+ try m.run(sysExit(test_maps, 1000).code(), &exit_context);
+ try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len);
+ try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_exit, .aux = 257, .a = .{ @bitCast(@as(i64, -2)), 0, 0, 0, 0, 0 } }, m.sentEvent());
+
+ // Another process: nothing. A full ring: counted.
+ m.sent_len = null;
+ try m.run(sysEnter(test_maps, 999).code(), &enter_context);
+ try std.testing.expectEqual(@as(?usize, null), m.sent_len);
+ m.ring_full = true;
+ try m.run(sysExit(test_maps, 1000).code(), &exit_context);
+ try m.run(sysEnter(test_maps, 1000).code(), &enter_context);
+ try std.testing.expectEqual(@as(u64, 2), m.drops);
+}
+
+test "the sched_switch program: either side of the switch, ids from where BTF says" {
+ // Two task_structs with the ids at offsets of our choosing.
+ var prev: [64]u32 = @splat(0);
+ var next: [64]u32 = @splat(0);
+ prev[10] = 1001;
+ prev[11] = 1000;
+ next[10] = 55;
+ next[11] = 50;
+ const context = [3]u64{ 1, @intFromPtr(&prev), @intFromPtr(&next) };
+ var m: Machine = .{ .now = 9, .cpu = 3 };
+ try m.run(schedSwitch(test_maps, 1000, 40, 44).code(), &context);
+ try std.testing.expectEqual(Event{ .ts = 9, .tid = 1001, .kind = .sched, .aux = 1, .a = .{ 1001, 55, 3, 0, 0, 0 } }, m.sentEvent());
+ m.sent_len = null;
+ try m.run(schedSwitch(test_maps, 50, 40, 44).code(), &context);
+ try std.testing.expectEqual(@as(u64, 55), m.sentEvent().a[1]);
+ m.sent_len = null;
+ try m.run(schedSwitch(test_maps, 7, 40, 44).code(), &context);
+ try std.testing.expectEqual(@as(?usize, null), m.sent_len);
+}
+
+test "the probe program and its record decoded; a USDT probe's arguments as usdt.Arg.fetch has them" {
+ // The probed thread's memory is ours: rbp and rdx point into this.
+ var memory: [64]u8 = undefined;
+ for (&memory, 0..) |*b, i| b.* = @truncate(0xf0 + i);
+ const regs: Regs = .{ .rdi = 11, .rsi = 0xffff_fffe, .rdx = @intFromPtr(&memory), .rcx = 3, .r8 = 15, .rbp = @intFromPtr(&memory) + 32, .rip = 0x401234, .rsp = 0x7000, .r10 = 0x1_0000_00ff };
+ var m: Machine = .{ .pid_tgid = 1000 << 32 | 1000, .now = 5 };
+ try m.run(probe(test_maps, 1000, 0xfeed, null).code(), &regs);
+ try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent());
+
+ // A sink that keeps the last event.
+ const Last = struct {
+ ev: Event = undefined,
+ regs: Regs = undefined,
+ fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void {
+ const l: *@This() = @ptrCast(@alignCast(ctx));
+ l.ev = ev.*;
+ l.regs = std.mem.bytesToValue(Regs, payload);
+ }
+ };
+ var last: Last = .{};
+ try decode(m.sent[0..m.sent_len.?], .{ .ctx = &last, .emitFn = Last.emit });
+ try std.testing.expectEqual(@as(u64, 11), last.ev.a[1]);
+ try std.testing.expectEqual(regs.rip, last.regs.rip);
+ try std.testing.expectEqual(@as(u64, 0), last.regs.fs_base);
+
+ const here = struct {
+ pub fn read(_: @This(), addr: u64, buf: []u8) usize {
+ @memcpy(buf, @as([*]const u8, @ptrFromInt(addr)));
+ return buf.len;
+ }
+ }{};
+ const specs = [_][5][]const u8{
+ .{ "-4@%esi", "4@%esi", "1@-16(%rbp)", "8@$9", "-8@$-9" },
+ .{ "-1@8(%rdx,%rcx,8)", "2@(%rdx)", "-2@-32(%rbp)", "1@%r10b", "8@%r10" },
+ };
+ for (specs) |five| {
+ var args: [5]usdt.Arg = undefined;
+ for (&args, five) |*a, spec| a.* = try .parse(spec);
+ try m.run(probe(test_maps, 1000, 0xfeed, &args).code(), &regs);
+ for (args, m.sentEvent().a[1..]) |a, got| try std.testing.expectEqual(a.fetch(&regs, here).?, got);
+ }
+ // Fewer than five: the rest is 0.
+ try m.run(probe(test_maps, 1000, 0xfeed, &.{try .parse("4@%edi")}).code(), &regs);
+ try std.testing.expectEqual([6]u64{ 0xfeed, 11, 0, 0, 0, 0 }, m.sentEvent().a);
+}
+
+test "a ring buffer as the kernel lays it out: finished records only, discarded ones skipped" {
+ var consumer: u64 = 0;
+ var producer: u64 = 0;
+ var data: [64]u8 align(8) = @splat(0);
+ var ring: RingBuf = .{ .consumer = &consumer, .producer = &producer, .data = &data };
+ const records = [_]struct { u32, []const u8 }{ .{ 5, "hello" }, .{ 3 | RingBuf.discarded, "no!" }, .{ 2, "ok" }, .{ 4 | RingBuf.busy, "wait" } };
+ for (records) |record| {
+ std.mem.writeInt(u32, data[producer..][0..4], record[0], .little);
+ @memcpy(data[producer + 8 ..][0..record[1].len], record[1]);
+ producer += std.mem.alignForward(u64, record[1].len + 8, 8);
+ }
+ try std.testing.expectEqualStrings("hello", ring.next().?);
+ try std.testing.expectEqualStrings("ok", ring.next().?);
+ try std.testing.expect(ring.next() == null);
+ ring.commit();
+ try std.testing.expectEqual(@as(u64, 48), consumer);
+ // The producer finishes it.
+ std.mem.writeInt(u32, data[48..52], 4, .little);
+ try std.testing.expectEqualStrings("wait", ring.next().?);
+}
+
+test "task_struct's pid and tgid in this kernel's BTF; privilege is told apart" {
+ var buf: [512]u8 = undefined;
+ if (missing(&buf)) |why| try std.testing.expect(std.mem.indexOf(u8, why, "CAP_") != null);
+
+ const btf = std.Io.Dir.cwd().readFileAlloc(std.testing.io, "/sys/kernel/btf/vmlinux", std.testing.allocator, .unlimited) catch return error.SkipZigTest;
+ defer std.testing.allocator.free(btf);
+ const pid_at = fieldOffset(btf, "task_struct", "pid").?;
+ // Next to each other, some way into a struct of a few kilobytes.
+ try std.testing.expectEqual(pid_at + 4, fieldOffset(btf, "task_struct", "tgid").?);
+ try std.testing.expect(pid_at > 256 and pid_at < 16384);
+ try std.testing.expect(fieldOffset(btf, "task_struct", "no_such_field") == null);
+ try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "rdi")), fieldOffset(btf, "pt_regs", "di"));
+ try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "orig_rax")), fieldOffset(btf, "pt_regs", "orig_ax"));
+}
diff --git a/src/trace/perf.zig b/src/trace/perf.zig
index 7c9ef43..c251741 100644
--- a/src/trace/perf.zig
+++ b/src/trace/perf.zig
@@ -513,6 +513,58 @@ fn userData(raw: []const u8) []const u8 {
return "";
}
+/// A sink in front of another for when two backends record the same run
+/// (this one and BPF, or the agent's ring): each emits in time order, both
+/// with the same clock, and here they become one sequence. Events are held
+/// until `flush` passes on, oldest first, those up to a time that every
+/// backend has reported past.
+pub const Sorted = struct {
+ gpa: std.mem.Allocator,
+ to: event.Sink,
+ held: std.ArrayList(Held) = .empty,
+ payloads: std.ArrayList(u8) = .empty,
+
+ const Held = struct { ev: Event, at: usize, len: usize };
+
+ pub fn sink(s: *Sorted) event.Sink {
+ return .{ .ctx = s, .emitFn = hold };
+ }
+
+ fn hold(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void {
+ const s: *Sorted = @ptrCast(@alignCast(ctx));
+ try s.held.append(s.gpa, .{ .ev = ev.*, .at = s.payloads.items.len, .len = payload.len });
+ try s.payloads.appendSlice(s.gpa, payload);
+ }
+
+ pub fn flush(s: *Sorted, until: u64) !void {
+ // Stable: events of one instant stay in the order their backend gave them.
+ std.mem.sort(Held, s.held.items, {}, struct {
+ fn older(_: void, a: Held, b: Held) bool {
+ return a.ev.ts < b.ev.ts;
+ }
+ }.older);
+ var done: usize = 0;
+ while (done < s.held.items.len and s.held.items[done].ev.ts <= until) : (done += 1) {
+ const h = s.held.items[done];
+ try s.to.emit(h.ev, s.payloads.items[h.at..][0..h.len]);
+ }
+ // What stays moves to the front, its payloads with it.
+ var kept: std.ArrayList(u8) = .empty;
+ for (s.held.items[done..]) |*h| {
+ try kept.appendSlice(s.gpa, s.payloads.items[h.at..][0..h.len]);
+ h.at = kept.items.len - h.len;
+ }
+ s.held.replaceRangeAssumeCapacity(0, done, &.{});
+ s.payloads.deinit(s.gpa);
+ s.payloads = kept;
+ }
+
+ pub fn deinit(s: *Sorted) void {
+ s.held.deinit(s.gpa);
+ s.payloads.deinit(s.gpa);
+ }
+};
+
/// Lets a target held by `ptrace.Process.spawn` run on its own: once the
/// events are attached, ptrace has done its part.
pub fn release(p: ptrace.Process) void {
@@ -596,6 +648,31 @@ fn perfOpen(attr: *linux.perf_event_attr, pid: linux.pid_t, cpu: i32) !linux.fd_
};
}
+test "two backends' events come out as one sequence in time" {
+ const Seen = struct {
+ order: [8]u8 = undefined,
+ len: usize = 0,
+ fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void {
+ const seen: *@This() = @ptrCast(@alignCast(ctx));
+ try std.testing.expectEqual(ev.a[0], payload.len);
+ seen.order[seen.len] = payload[0];
+ seen.len += 1;
+ }
+ };
+ var seen: Seen = .{};
+ var sorted: Sorted = .{ .gpa = std.testing.allocator, .to = .{ .ctx = &seen, .emitFn = Seen.emit } };
+ defer sorted.deinit();
+ // One backend, then the other, each in its own order.
+ for ([_]struct { u64, []const u8 }{ .{ 10, "a" }, .{ 30, "ccc" }, .{ 50, "eeeee" }, .{ 20, "bb" }, .{ 30, "dddd" }, .{ 60, "ffffff" } }) |e| {
+ try sorted.sink().emit(.{ .ts = e[0], .kind = .mark, .a = .{ e[1].len, 0, 0, 0, 0, 0 } }, e[1]);
+ }
+ try sorted.flush(30);
+ try std.testing.expectEqualStrings("abcd", seen.order[0..seen.len]);
+ try sorted.sink().emit(.{ .ts = 55, .kind = .mark, .a = .{ 1, 0, 0, 0, 0, 0 } }, "x");
+ try sorted.flush(std.math.maxInt(u64));
+ try std.testing.expectEqualStrings("abcdexf", seen.order[0..seen.len]);
+}
+
test "a maps line; the user data of a syscall record" {
const m = parseMap("7f80b4224000-7f80b439f000 r-xp 00024000 08:02 1234 /usr/lib/libc.so.6").?;
try std.testing.expectEqual(@as(u64, 0x7f80b4224000), m.start);
diff --git a/src/trace/root.zig b/src/trace/root.zig
index 80745cb..62a7a90 100644
--- a/src/trace/root.zig
+++ b/src/trace/root.zig
@@ -20,6 +20,8 @@ pub const host = @import("host.zig");
pub const perf = @import("perf.zig");
/// Static probes (the ELF notes of <sys/sdt.h>) and their arguments.
pub const usdt = @import("usdt.zig");
+/// The kernel's view, privileged: BPF on tracepoints, kprobes, uprobes, USDT.
+pub const bpf = @import("bpf.zig");
test {
_ = event;
@@ -33,4 +35,5 @@ test {
_ = host;
_ = perf;
_ = usdt;
+ _ = bpf;
}
diff --git a/src/trace/usdt.zig b/src/trace/usdt.zig
index ba03079..d5f1d04 100644
--- a/src/trace/usdt.zig
+++ b/src/trace/usdt.zig
@@ -194,6 +194,7 @@ pub const Arg = struct {
.index = if (parts.next()) |r| try register(r) else null,
.scale = if (parts.next()) |s| std.fmt.parseInt(u8, s, 10) catch return error.BadUsdtArg else 1,
} };
+ if (std.mem.indexOfScalar(u8, &.{ 1, 2, 4, 8 }, arg.at.memory.scale) == null) return error.BadUsdtArg;
}
return arg;
}
@@ -278,7 +279,7 @@ test "argument specs are parsed and fetched" {
.{ "8@0x7000(%rbp)", null },
};
for (cases) |case| try std.testing.expectEqual(case[1], (try Arg.parse(case[0])).fetch(&regs, memory));
- for ([_][]const u8{ "4@counter(%rip)", "3@%eax", "4@%ah", "4", "4@", "4@8(%rdx", "4@%r8x" }) |bad| {
+ for ([_][]const u8{ "4@counter(%rip)", "3@%eax", "4@%ah", "4", "4@", "4@8(%rdx", "4@%r8x", "4@(%rdx,%rax,3)" }) |bad| {
try std.testing.expectError(error.BadUsdtArg, Arg.parse(bad));
}
}