summaryrefslogtreecommitdiff
path: root/src/trace
diff options
context:
space:
mode:
Diffstat (limited to 'src/trace')
-rw-r--r--src/trace/kernel_test.zig346
-rw-r--r--src/trace/perf.zig619
-rw-r--r--src/trace/root.zig6
-rw-r--r--src/trace/usdt.zig284
4 files changed, 1255 insertions, 0 deletions
diff --git a/src/trace/kernel_test.zig b/src/trace/kernel_test.zig
new file mode 100644
index 0000000..6098c58
--- /dev/null
+++ b/src/trace/kernel_test.zig
@@ -0,0 +1,346 @@
+//! The kernel area against real programs: perf events on the targets of
+//! `targets/kernel_*`, the USDT notes of one checked against readelf and
+//! stopped at under ptrace. Built by `build/kernel.zig`, which hands over
+//! the targets' paths.
+const std = @import("std");
+const linux = std.os.linux;
+const zt = @import("ztrace");
+const targets = @import("targets");
+const perf = zt.perf;
+const gpa = std.testing.allocator;
+const expect = std.testing.expect;
+const expectEqual = std.testing.expectEqual;
+
+/// A sink that keeps everything.
+const Log = struct {
+ items: std.ArrayList(Item) = .empty,
+ counts: [8]u64 = @splat(0),
+
+ const Item = struct { ev: zt.Event, payload: []u8 };
+
+ fn sink(l: *Log) zt.Sink {
+ return .{ .ctx = l, .emitFn = emit };
+ }
+
+ fn emit(ctx: *anyopaque, ev: *const zt.Event, payload: []const u8) anyerror!void {
+ const l: *Log = @ptrCast(@alignCast(ctx));
+ try l.items.append(gpa, .{ .ev = ev.*, .payload = try gpa.dupe(u8, payload) });
+ }
+
+ fn deinit(l: *Log) void {
+ for (l.items.items) |item| gpa.free(item.payload);
+ l.items.deinit(gpa);
+ }
+
+ fn count(l: *const Log, kind: zt.Kind) usize {
+ var n: usize = 0;
+ for (l.items.items) |item| n += @intFromBool(item.ev.kind == kind);
+ return n;
+ }
+
+ /// The string a `string` event gave `id`.
+ fn string(l: *const Log, id: u64) ?[]const u8 {
+ for (l.items.items) |item| if (item.ev.kind == .string and item.ev.a[0] == id) return item.payload;
+ return null;
+ }
+};
+
+const Armed = struct { symbol: []const u8, mode: perf.Watch.Mode };
+
+/// Runs a target to its end with a session on it. `log.counts` gets the
+/// counters. Skips the test where the kernel or CPU refuses the events
+/// (perf_event_paranoid above 2, a VM without a PMU).
+fn record(log: *Log, path: []const u8, args: []const []const u8, options: perf.Options, armed: []const Armed) !void {
+ var arena_state: std.heap.ArenaAllocator = .init(gpa);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ const argv = try arena.allocSentinel(?[*:0]const u8, args.len + 1, null);
+ argv[0] = try arena.dupeSentinel(u8, path, 0);
+ for (argv[1..], args) |*dst, arg| dst.* = try arena.dupeSentinel(u8, arg, 0);
+ const envp = [_:null]?[*:0]const u8{};
+
+ var proc = try zt.ptrace.Process.spawn(argv[0].?, argv, &envp);
+ errdefer proc.kill();
+ var session = perf.Session.open(gpa, proc.pid, options, log.sink()) catch |err| switch (err) {
+ error.PerfAccess, error.PerfNoSuchEvent => {
+ proc.kill();
+ return error.SkipZigTest;
+ },
+ else => |e| return e,
+ };
+ defer session.close();
+ for (armed) |a| {
+ const sym = (try perf.lookup(gpa, proc.pid, a.symbol)).?;
+ try session.watch(.{ .addr = sym.addr, .len = @intCast(@min(sym.size, 8)), .mode = a.mode, .name = a.symbol });
+ }
+ perf.release(proc);
+ while (try session.poll(100)) {}
+ try session.readCounters();
+ @memcpy(log.counts[0..session.counts.len], session.counts);
+ try expectEqual(@as(u64, 0), session.lost);
+ var status: u32 = 0;
+ _ = linux.waitpid(proc.pid, @ptrCast(&status), 0);
+ try expect(linux.W.IFEXITED(status) and linux.W.EXITSTATUS(status) == 0);
+}
+
+test "counters: the instructions of a loop of known length" {
+ var log: Log = .{};
+ defer log.deinit();
+ try record(&log, targets.spin, &.{"5000000"}, .{ .counters = &.{ .instructions, .branches, .@"task-clock" } }, &.{});
+ // Four instructions and one branch an iteration, plus the loader and libc starting up.
+ try expect(log.counts[0] >= 20_000_000 and log.counts[0] < 20_600_000);
+ try expect(log.counts[1] >= 5_000_000 and log.counts[1] < 5_150_000);
+ try expect(log.counts[2] > 100_000);
+ // ... and the same as events, named.
+ const last = log.items.items[log.items.items.len - 3];
+ try expectEqual(zt.Kind.sample, last.ev.kind);
+ try expectEqual(perf.reading, last.ev.aux);
+ try expectEqual(log.counts[0], last.ev.a[2]);
+ try std.testing.expectEqualStrings("instructions", log.string(last.ev.a[1]).?);
+}
+
+test "faults: every fresh page touched, with the registers; maps: the program itself and what it maps" {
+ var log: Log = .{};
+ defer log.deinit();
+ const pages = 64;
+ try record(&log, targets.touch, &.{std.fmt.comptimePrint("{d}", .{pages})}, .{ .faults = true }, &.{});
+
+ // Text of the program itself (mapped before we came: from /proc), of
+ // libc (mapped after: from the kernel's records), and anonymous memory.
+ var text: [2]u64 = .{ 0, 0 };
+ var libc = false;
+ var anonymous: std.ArrayList([2]u64) = .empty;
+ defer anonymous.deinit(gpa);
+ var touched: usize = 0;
+ for (log.items.items) |item| switch (item.ev.kind) {
+ .map => {
+ const range: [2]u64 = .{ item.ev.a[0], item.ev.a[0] + item.ev.a[1] };
+ const executable = item.ev.a[2] & 4 != 0;
+ if (std.mem.endsWith(u8, item.payload, "/kernel_touch") and executable) text = range;
+ if (std.mem.indexOf(u8, item.payload, "libc.so") != null and executable) libc = true;
+ // The target's own mmap is in one of these: the kernel reports
+ // it merged with its neighbour, not by itself.
+ if (item.payload.len == 0 and item.ev.a[1] >= pages * 4096) try anonymous.append(gpa, range);
+ },
+ .fault => {
+ const regs = std.mem.bytesToValue(zt.Regs, item.payload);
+ try expectEqual(item.ev.a[1], regs.rip);
+ try expect(regs.rsp != 0);
+ // Its loop: from its own code, the first byte of a page of anonymous memory.
+ if (regs.rip < text[0] or regs.rip >= text[1] or item.ev.a[0] % 4096 != 0) continue;
+ for (anonymous.items) |range| {
+ if (item.ev.a[0] >= range[0] and item.ev.a[0] < range[1]) break;
+ } else continue;
+ touched += 1;
+ },
+ else => {},
+ };
+ try expect(text[0] != 0 and libc);
+ try expect(touched >= pages);
+ try expect(touched <= pages + 2);
+}
+
+test "debug registers: a watchpoint sees every write of a global, a breakpoint every call and its arguments" {
+ var log: Log = .{};
+ defer log.deinit();
+ const steps = 100;
+ try record(&log, targets.touch, &.{ "1", std.fmt.comptimePrint("{d}", .{steps}) }, .{}, &.{
+ .{ .symbol = "counter", .mode = .write },
+ .{ .symbol = "step", .mode = .exec },
+ });
+ var writes: u64 = 0;
+ var calls: u64 = 0;
+ var sum: u64 = 0;
+ var step_pc: u64 = 0;
+ for (log.items.items) |item| switch (item.ev.kind) {
+ .probe => {
+ try std.testing.expectEqualStrings("step", log.string(item.ev.a[0]).?);
+ // step(i, the sum so far)
+ try expectEqual(calls, item.ev.a[1]);
+ try expectEqual(sum, item.ev.a[2]);
+ step_pc = std.mem.bytesToValue(zt.Regs, item.payload).rip;
+ sum += calls;
+ calls += 1;
+ },
+ .mem_write => {
+ try expectEqual(@as(u64, 8), item.ev.a[1]);
+ // The pc is the instruction after the store, a few bytes into step.
+ try expect(item.ev.a[2] > step_pc and item.ev.a[2] < step_pc + 32);
+ // The value is read when the record is: what was written, or something later.
+ if (item.payload.len == 8) try expect(std.mem.readInt(u64, item.payload[0..8], .little) >= sum);
+ writes += 1;
+ },
+ else => {},
+ };
+ // The thread's three calls are not this task's: nobody asked to follow.
+ try expectEqual(@as(u64, steps), calls);
+ try expectEqual(@as(u64, steps), writes);
+}
+
+test "syscalls with their paths, context switches, threads followed" {
+ var log: Log = .{};
+ defer log.deinit();
+ try record(&log, targets.touch, &.{ "1", "2" }, .{ .syscalls = true, .switches = true, .follow = true }, &.{.{ .symbol = "step", .mode = .exec }});
+
+ var fd: ?u64 = null;
+ var opened = false;
+ var wrote = false;
+ var slept = false;
+ var thread: u32 = 0;
+ var thread_calls: usize = 0;
+ var last_ts: u64 = 0;
+ for (log.items.items, 0..) |item, i| {
+ const ev = item.ev;
+ // One stream in time order, whichever CPU's ring a record came from.
+ if (ev.kind != .string and ev.kind != .map) {
+ try expect(ev.ts >= last_ts);
+ last_ts = ev.ts;
+ }
+ switch (ev.kind) {
+ .syscall_enter => switch (@as(linux.SYS, @fromBackingInt(@intCast(ev.aux)))) {
+ // Linux 6.19 and later copy the path and the bytes written into the record.
+ .openat => if (ev.a[1] != 0 and (item.payload.len == 0 or std.mem.eql(u8, item.payload, "/dev/null")) and ev.a[2] & 3 == 1) {
+ opened = true;
+ // Its exit is the same thread's next event.
+ for (log.items.items[i + 1 ..]) |next| if (next.ev.tid == ev.tid) {
+ try expectEqual(zt.Kind.syscall_exit, next.ev.kind);
+ try expectEqual(ev.aux, next.ev.aux);
+ fd = next.ev.a[0];
+ break;
+ };
+ },
+ .write => if (fd != null and ev.a[0] == fd.? and ev.a[2] == 5) {
+ try expect(item.payload.len == 0 or std.mem.eql(u8, item.payload, "hello"));
+ wrote = true;
+ },
+ .nanosleep, .clock_nanosleep => slept = true,
+ else => {},
+ },
+ // It left the CPU to sleep.
+ .sched => if (slept and ev.a[0] == ev.tid) {
+ slept = false;
+ thread_calls += 1000;
+ },
+ .thread_start => thread = @intCast(ev.a[0]),
+ .probe => if (ev.tid == thread and thread != 0) {
+ try expect(ev.a[1] >= 1000);
+ thread_calls += 1;
+ },
+ else => {},
+ }
+ }
+ try expect(opened and wrote and thread != 0);
+ try expectEqual(@as(usize, 1003), thread_calls);
+ try expect(log.count(.syscall_enter) == log.count(.syscall_exit) + 1 or log.count(.syscall_enter) == log.count(.syscall_exit) + 2);
+ try expectEqual(@as(usize, 2), log.count(.thread_exit));
+}
+
+test "a profile: samples in the program's code, their callchains starting there" {
+ var log: Log = .{};
+ defer log.deinit();
+ try record(&log, targets.spin, &.{"100000000"}, .{ .sample_hz = 5000, .callchain = true }, &.{});
+ var text: [2]u64 = .{ 0, 0 };
+ var in_text: usize = 0;
+ for (log.items.items) |item| switch (item.ev.kind) {
+ .map => if (std.mem.endsWith(u8, item.payload, "/kernel_spin") and item.ev.a[2] & 4 != 0) {
+ text = .{ item.ev.a[0], item.ev.a[0] + item.ev.a[1] };
+ },
+ .sample => {
+ try std.testing.expectEqualStrings("cycles", log.string(item.ev.a[1]).?);
+ try expect(item.payload.len >= 8 and item.payload.len % 8 == 0);
+ try expectEqual(item.ev.a[0], std.mem.readInt(u64, item.payload[0..8], .little));
+ in_text += @intFromBool(item.ev.a[0] >= text[0] and item.ev.a[0] < text[1]);
+ },
+ else => {},
+ };
+ // All but the few taken while the loader ran.
+ try expect(in_text >= 20 and in_text + 10 >= log.count(.sample));
+}
+
+test "usdt: the notes say what readelf says; arguments are fetched at a probe" {
+ const io = std.testing.io;
+ const path = try gpa.dupeSentinel(u8, targets.usdt, 0);
+ defer gpa.free(path);
+ const elf = try zt.usdt.Elf.open(path);
+ defer elf.close();
+ const found = try zt.usdt.probes(gpa, elf, 0);
+ defer gpa.free(found);
+ try expectEqual(@as(usize, 3), found.len);
+
+ const result = std.process.run(gpa, io, .{ .argv = &.{ "readelf", "-n", targets.usdt } }) catch return error.SkipZigTest;
+ defer gpa.free(result.stdout);
+ defer gpa.free(result.stderr);
+ var lines = std.mem.tokenizeScalar(u8, result.stdout, '\n');
+ var seen: usize = 0;
+ while (lines.next()) |line| {
+ const provider = after(line, "Provider: ") orelse continue;
+ const probe = found[seen];
+ seen += 1;
+ try std.testing.expectEqualStrings(provider, probe.provider);
+ try std.testing.expectEqualStrings(after(lines.next().?, "Name: ").?, probe.name);
+ // Location: 0x..., Base: 0x..., Semaphore: 0x...
+ var numbers = std.mem.tokenizeAny(u8, lines.next().?, " ,");
+ var values: [3]u64 = undefined;
+ for (&values) |*v| {
+ _ = numbers.next();
+ v.* = try std.fmt.parseInt(u64, numbers.next().?, 0);
+ }
+ try expectEqual(values[0], probe.pc);
+ try expectEqual(values[2], probe.semaphore);
+ try std.testing.expectEqualStrings(after(lines.next().?, "Arguments: ").?, probe.args);
+ }
+ try expectEqual(found.len, seen);
+
+ // What the debugger will do: an int3 on each nop, the semaphore raised,
+ // and at every stop the arguments read from the registers and memory.
+ const argv = [_:null]?[*:0]const u8{ path, "4", null };
+ const envp = [_:null]?[*:0]const u8{};
+ var proc = try zt.ptrace.Process.spawn(path, &argv, &envp);
+ errdefer proc.kill();
+ const maps = try perf.readMaps(gpa, proc.pid);
+ defer gpa.free(maps);
+ // The program itself is the first thing mapped.
+ const live = try zt.usdt.probes(gpa, elf, elf.bias(perf.parseMap(maps[0..std.mem.indexOfScalar(u8, maps, '\n').?]).?.start));
+ defer gpa.free(live);
+ for (live) |probe| {
+ var nop: [1]u8 = undefined;
+ try expectEqual(@as(usize, 1), proc.read(probe.pc, &nop));
+ try expectEqual(@as(u8, 0x90), nop[0]);
+ try proc.write(probe.pc, &.{0xcc});
+ if (probe.semaphore != 0) try proc.write(probe.semaphore, &.{ 1, 0 });
+ }
+ // start(n), then tick(i, the sum of squares so far) n times, then done(n, sum).
+ const expected = [_]struct { []const u8, []const u64 }{
+ .{ "start", &.{4} }, .{ "tick", &.{ 0, 0 } }, .{ "tick", &.{ 1, 1 } },
+ .{ "tick", &.{ 2, 5 } }, .{ "tick", &.{ 3, 14 } }, .{ "done", &.{ 4, 14 } },
+ };
+ var stops: usize = 0;
+ while (true) switch (try proc.cont()) {
+ .trap => {
+ // The int3 was the whole instruction: the thread is already past the probe.
+ const regs = try proc.regs();
+ const probe = for (live) |probe| {
+ if (probe.pc == regs.rip - 1) break probe;
+ } else return error.StoppedElsewhere;
+ try std.testing.expectEqualStrings(expected[stops][0], probe.name);
+ var specs = std.mem.tokenizeScalar(u8, probe.args, ' ');
+ for (expected[stops][1]) |value| {
+ const arg = try zt.usdt.Arg.parse(specs.next().?);
+ try expectEqual(@as(?u64, value), arg.fetch(&regs, proc));
+ }
+ try expect(specs.next() == null);
+ stops += 1;
+ },
+ .exited => |code| {
+ try expectEqual(@as(u8, 0), code);
+ break;
+ },
+ else => {},
+ };
+ try expectEqual(expected.len, stops);
+}
+
+fn after(line: []const u8, label: []const u8) ?[]const u8 {
+ const at = std.mem.indexOf(u8, line, label) orelse return null;
+ return line[at + label.len ..];
+}
diff --git a/src/trace/perf.zig b/src/trace/perf.zig
new file mode 100644
index 0000000..7c9ef43
--- /dev/null
+++ b/src/trace/perf.zig
@@ -0,0 +1,619 @@
+//! The kernel's view of a task without privilege: `perf_event_open` as far
+//! as `perf_event_paranoid=2` lets a user go on their own processes (user
+//! space only). Counters, a sampling profile, every page fault, the memory
+//! maps, threads, context switches, every syscall, and the debug registers
+//! as watchpoints. The target runs at full speed: the kernel writes records
+//! into ring buffers, stamped with CLOCK_MONOTONIC like every other
+//! backend's events, and `Session` decodes them into a `Sink`.
+//!
+//! What becomes what:
+//! counter reading sample aux=`reading`, a1=name string id, a2=total so far
+//! profile sample sample a0=pc a1=name string id a2=period a3=data address. Payload: the callchain, u64 pcs
+//! page fault fault a0=address a1=pc, aux=`fault_major` or 0. Payload: `Regs`
+//! mapping map as the contract says; the path is `[heap]`, `[stack]`... for those
+//! clone, exit thread_start (tid=parent, a0=new tid, a1=its pid), thread_exit
+//! comm mark "comm NAME", or "exec NAME"
+//! context switch sched out: a0=tid, in: a1=tid; a2=cpu, aux=1 when preempted
+//! syscall syscall_enter, syscall_exit. Payload of an enter: the path or buffer the kernel captured
+//! watchpoint mem_write (payload: the bytes there when the record is decoded), mem_read; a2=pc after the access
+//! execute breakpoint probe a0=name string id a1..a5=rdi rsi rdx rcx r8. Payload: `Regs`
+const std = @import("std");
+const linux = std.os.linux;
+const event = @import("event.zig");
+const ptrace = @import("ptrace.zig");
+const usdt = @import("usdt.zig");
+const Event = event.Event;
+const Regs = event.Regs;
+const PERF = linux.PERF;
+
+/// `aux` of a `sample` that is a counter's total, not a profile sample.
+pub const reading: u16 = 1;
+/// `aux` of a `fault` that needed I/O.
+pub const fault_major: u16 = 1;
+
+/// The id a `string` event gives `text`: its hash, so that backends agree
+/// without a registry.
+pub fn stringId(text: []const u8) u64 {
+ return std.hash.Wyhash.hash(0, text);
+}
+
+/// What can be counted. Context switches and migrations are missing: their
+/// software events fire in kernel mode, which paranoid 2 excludes, so they
+/// always read 0 (`Options.switches` gives the switches as events instead).
+pub const Counter = enum {
+ instructions,
+ cycles,
+ branches,
+ @"branch-misses",
+ @"cache-references",
+ @"cache-misses",
+ /// Faults taken in user mode only.
+ @"page-faults",
+ /// Nanoseconds on a CPU.
+ @"task-clock",
+
+ fn attr(c: Counter) linux.perf_event_attr {
+ const HW = PERF.COUNT.HW;
+ const SW = PERF.COUNT.SW;
+ return switch (c) {
+ .instructions => .{ .type = .HARDWARE, .config = @backingInt(HW.INSTRUCTIONS) },
+ .cycles => .{ .type = .HARDWARE, .config = @backingInt(HW.CPU_CYCLES) },
+ .branches => .{ .type = .HARDWARE, .config = @backingInt(HW.BRANCH_INSTRUCTIONS) },
+ .@"branch-misses" => .{ .type = .HARDWARE, .config = @backingInt(HW.BRANCH_MISSES) },
+ .@"cache-references" => .{ .type = .HARDWARE, .config = @backingInt(HW.CACHE_REFERENCES) },
+ .@"cache-misses" => .{ .type = .HARDWARE, .config = @backingInt(HW.CACHE_MISSES) },
+ .@"page-faults" => .{ .type = .SOFTWARE, .config = @backingInt(SW.PAGE_FAULTS) },
+ .@"task-clock" => .{ .type = .SOFTWARE, .config = @backingInt(SW.TASK_CLOCK) },
+ };
+ }
+};
+
+/// A debug register on the target: the CPU reports every access, the target
+/// never stops. There are four; a data watchpoint covers 1, 2, 4 or 8 bytes
+/// at an address aligned to that; the record is written after the access, so
+/// `pc` is the instruction after it and the value is not part of it.
+pub const Watch = struct {
+ pub const Mode = enum(u32) { write = 2, access = 3, exec = 4 };
+
+ addr: u64,
+ len: u8 = 8,
+ mode: Mode = .write,
+ /// What an `exec` hit is called.
+ name: []const u8 = "",
+};
+
+pub const Options = struct {
+ counters: []const Counter = &.{},
+ /// Profile samples a second (of cycles); 0 for none.
+ sample_hz: u32 = 0,
+ /// Frame-pointer callchains on profile samples.
+ callchain: bool = false,
+ faults: bool = false,
+ switches: bool = false,
+ syscalls: bool = false,
+ /// Also the threads and children the target creates from now on. One
+ /// ring per CPU then, merged by time, as inherited events need.
+ follow: bool = false,
+ /// Per ring; a power of two, at least a page.
+ ring_bytes: usize = 128 * 4096,
+};
+
+const base_sample = PERF.SAMPLE.IDENTIFIER | PERF.SAMPLE.IP | PERF.SAMPLE.TID | PERF.SAMPLE.TIME | PERF.SAMPLE.ADDR | PERF.SAMPLE.CPU | PERF.SAMPLE.PERIOD;
+/// The fixed head of a sample with `base_sample`: id, ip, pid, tid, time, addr, cpu, period.
+const sample_head = 56;
+/// What follows every other record: pid, tid, time, cpu, id.
+const trailer = 32;
+
+/// The registers perf hands out for a 64-bit task, in its bit order (the
+/// segment registers 12..15 are refused).
+const perf_regs = [_][]const u8{ "rax", "rbx", "rcx", "rdx", "rsi", "rdi", "rbp", "rsp", "rip", "eflags", "cs", "ss", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" };
+const perf_regs_mask: u64 = 0xff0fff;
+
+/// What an event writing into a ring means, found by the id in its records.
+const Source = union(enum) {
+ sideband,
+ profile,
+ fault: u16,
+ watch: struct { len: u8, mode: Watch.Mode, name: u64 },
+ syscall_enter,
+ syscall_exit,
+};
+
+const Ring = struct {
+ cpu: i32,
+ /// The event that owns the mapping; the others output into it.
+ fd: linux.fd_t = -1,
+ meta: *linux.perf_event_mmap_page = undefined,
+ data: []u8 = &.{},
+ head: u64 = 0,
+ tail: u64 = 0,
+
+ /// The record at the read position, copied to `scratch` when it wraps.
+ fn peek(r: *const Ring, scratch: []u8) ?[]const u8 {
+ if (r.tail == r.head) return null;
+ const at: usize = @intCast(r.tail & (r.data.len - 1));
+ // Records are multiples of 8 bytes: the header itself never wraps.
+ const size = std.mem.readInt(u16, r.data[at + 6 ..][0..2], .little);
+ if (at + size <= r.data.len) return r.data[at..][0..size];
+ const first = r.data.len - at;
+ @memcpy(scratch[0..first], r.data[at..]);
+ @memcpy(scratch[first..size], r.data[0 .. size - first]);
+ return scratch[0..size];
+ }
+};
+
+pub const Session = struct {
+ gpa: std.mem.Allocator,
+ pid: linux.pid_t,
+ sink: event.Sink,
+ options: Options,
+ rings: []Ring = &.{},
+ pollfds: []linux.pollfd = &.{},
+ fds: std.ArrayList(linux.fd_t) = .empty,
+ sources: std.AutoHashMapUnmanaged(u64, Source) = .empty,
+ counter_fds: []linux.fd_t = &.{},
+ /// The totals `readCounters` last saw, in `options.counters` order.
+ counts: []u64 = &.{},
+ /// Records the kernel dropped because a ring was full.
+ lost: u64 = 0,
+ scratch: []u8 = &.{},
+
+ /// Attaches to `pid` (a thread id), which should be stopped: events
+ /// count from here. Emits the names used and the mappings it already has.
+ pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, options: Options, sink: event.Sink) !Session {
+ std.debug.assert(std.math.isPowerOfTwo(options.ring_bytes) and options.ring_bytes >= 4096);
+ var s: Session = .{ .gpa = gpa, .pid = pid, .sink = sink, .options = options };
+ errdefer s.close();
+ s.scratch = try gpa.alloc(u8, 1 << 16);
+ // shortcut: CPUs 0..count-1, right unless some are offline or outside our affinity mask.
+ const cpus = if (options.follow) try std.Thread.getCpuCount() else 1;
+ s.rings = try gpa.alloc(Ring, cpus);
+ for (s.rings, 0..) |*r, i| r.* = .{ .cpu = if (options.follow) @intCast(i) else -1 };
+
+ // An event that counts nothing carries the records about the task
+ // itself, and comes first: it owns the rings.
+ var side: linux.perf_event_attr = .{ .type = .SOFTWARE, .config = @backingInt(PERF.COUNT.SW.DUMMY) };
+ side.flags.mmap = true;
+ side.flags.mmap2 = true;
+ side.flags.mmap_data = true;
+ side.flags.comm = true;
+ side.flags.comm_exec = true;
+ side.flags.task = true;
+ side.flags.context_switch = options.switches;
+ try s.add(side, .sideband);
+ s.pollfds = try gpa.alloc(linux.pollfd, cpus);
+ for (s.pollfds, s.rings) |*p, r| p.* = .{ .fd = r.fd, .events = linux.POLL.IN, .revents = 0 };
+
+ s.counter_fds = try gpa.alloc(linux.fd_t, options.counters.len);
+ @memset(s.counter_fds, -1);
+ s.counts = try gpa.alloc(u64, options.counters.len);
+ @memset(s.counts, 0);
+ for (options.counters, s.counter_fds) |c, *fd| {
+ var attr = c.attr();
+ attr.read_format = 1 | 2; // TOTAL_TIME_ENABLED | TOTAL_TIME_RUNNING
+ attr.flags.inherit = options.follow;
+ fd.* = try perfOpen(&attr, pid, -1);
+ try s.name(@tagName(c));
+ }
+
+ if (options.sample_hz != 0) {
+ var attr = Counter.cycles.attr();
+ attr.sample_period_or_freq = options.sample_hz;
+ attr.flags.freq = true;
+ attr.flags.exclude_callchain_kernel = true;
+ if (options.callchain) attr.sample_type = PERF.SAMPLE.CALLCHAIN;
+ try s.add(attr, .profile);
+ try s.name("cycles");
+ }
+ if (options.faults) for ([_]PERF.COUNT.SW{ .PAGE_FAULTS_MIN, .PAGE_FAULTS_MAJ }, [_]u16{ 0, fault_major }) |config, aux| {
+ try s.add(.{ .type = .SOFTWARE, .config = @backingInt(config), .sample_period_or_freq = 1, .sample_type = PERF.SAMPLE.REGS_USER }, .{ .fault = aux });
+ };
+ if (options.syscalls) try s.addSyscalls();
+ try s.maps();
+ return s;
+ }
+
+ pub fn close(s: *Session) void {
+ for (s.rings) |r| if (r.fd >= 0) {
+ _ = linux.munmap(@ptrCast(r.meta), 4096 + r.data.len);
+ };
+ for (s.fds.items) |fd| _ = linux.close(fd);
+ for (s.counter_fds) |fd| if (fd >= 0) {
+ _ = linux.close(fd);
+ };
+ s.fds.deinit(s.gpa);
+ s.sources.deinit(s.gpa);
+ s.gpa.free(s.rings);
+ s.gpa.free(s.pollfds);
+ s.gpa.free(s.counter_fds);
+ s.gpa.free(s.counts);
+ s.gpa.free(s.scratch);
+ }
+
+ /// Arms a debug register, on a session already running too.
+ /// `error.NoFreeBreakpoint` when all four are taken.
+ pub fn watch(s: *Session, w: Watch) !void {
+ var attr: linux.perf_event_attr = .{ .type = .BREAKPOINT, .sample_period_or_freq = 1, .bp_type = @backingInt(w.mode), .config1 = w.addr, .config2 = if (w.mode == .exec) 8 else w.len };
+ // Where it executes is known; what it was called with is not.
+ if (w.mode == .exec) attr.sample_type = PERF.SAMPLE.REGS_USER;
+ try s.add(attr, .{ .watch = .{ .len = w.len, .mode = w.mode, .name = stringId(w.name) } });
+ if (w.mode == .exec) try s.name(w.name);
+ }
+
+ /// One event on every ring, its records told apart by `source`.
+ fn add(s: *Session, template: linux.perf_event_attr, source: Source) !void {
+ var attr = template;
+ attr.sample_type |= base_sample;
+ if (attr.sample_type & PERF.SAMPLE.REGS_USER != 0) attr.sample_regs_user = perf_regs_mask;
+ attr.flags.sample_id_all = true;
+ attr.flags.inherit = s.options.follow;
+ attr.flags.use_clockid = true;
+ attr.clockid = .MONOTONIC;
+ // Wake the reader when a ring is a quarter full (it also looks on its
+ // own), but at once for a watchpoint: the value is read on waking.
+ attr.flags.watermark = source != .watch;
+ attr.wakeup_events_or_watermark = if (source == .watch) 1 else @intCast(s.options.ring_bytes / 4);
+ for (s.rings) |*r| {
+ const fd = try perfOpen(&attr, s.pid, r.cpu);
+ try s.fds.append(s.gpa, fd);
+ if (r.fd < 0) {
+ const base = linux.mmap(null, 4096 + s.options.ring_bytes, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0);
+ if (linux.errno(base) != .SUCCESS) return error.PerfRingMap;
+ r.fd = fd;
+ r.meta = @ptrFromInt(base);
+ r.data = @as([*]u8, @ptrFromInt(base + 4096))[0..s.options.ring_bytes];
+ } else if (linux.errno(linux.ioctl(fd, PERF.EVENT_IOC.SET_OUTPUT, @intCast(r.fd))) != .SUCCESS) return error.PerfSetOutput;
+ var id: u64 = 0;
+ if (linux.errno(linux.ioctl(fd, PERF.EVENT_IOC.ID, @intFromPtr(&id))) != .SUCCESS) return error.PerfEventId;
+ try s.sources.put(s.gpa, id, source);
+ }
+ }
+
+ /// Every syscall, through the per-syscall tracepoints: the only
+ /// tracepoints an unprivileged user may read, and only on a task of
+ /// their own. Their ids are in tracefs, which we may not be able to
+ /// read, so every id is tried: about 700 open, two a syscall.
+ fn addSyscalls(s: *Session) !void {
+ var limit: linux.rlimit = undefined;
+ if (linux.errno(linux.getrlimit(.NOFILE, &limit)) == .SUCCESS and limit.cur < limit.max) {
+ limit.cur = limit.max;
+ _ = linux.setrlimit(.NOFILE, &limit);
+ }
+ // shortcut: ids below 4096; scan further if a kernel has more trace events than that.
+ for (0..4096) |id| {
+ var attr: linux.perf_event_attr = .{ .type = .TRACEPOINT, .config = id, .sample_period_or_freq = 1, .sample_type = PERF.SAMPLE.RAW };
+ // Only a syscall's exit has a field `ret`: a filter on it is
+ // accepted there and nowhere else. On an event of its own, since
+ // a filter cannot be taken off again.
+ const probe = perfOpen(&attr, s.pid, -1) catch |err| switch (err) {
+ error.PerfAccess, error.PerfNoSuchEvent => continue,
+ else => |e| return e,
+ };
+ const is_exit = linux.errno(linux.ioctl(probe, PERF.EVENT_IOC.SET_FILTER, @intFromPtr("ret==0"))) == .SUCCESS;
+ _ = linux.close(probe);
+ try s.add(attr, if (is_exit) .syscall_exit else .syscall_enter);
+ }
+ }
+
+ fn name(s: *Session, text: []const u8) !void {
+ try s.sink.emit(.{ .ts = event.now(), .tid = @intCast(s.pid), .kind = .string, .a = .{ stringId(text), 0, 0, 0, 0, 0 } }, text);
+ }
+
+ /// The mappings the target has now, from /proc: the kernel only reports
+ /// the ones made after an event exists.
+ fn maps(s: *Session) !void {
+ const text = try readMaps(s.gpa, s.pid);
+ defer s.gpa.free(text);
+ const ts = event.now();
+ var lines = std.mem.tokenizeScalar(u8, text, '\n');
+ while (lines.next()) |line| {
+ const m = parseMap(line) orelse continue;
+ try s.sink.emit(.{ .ts = ts, .tid = @intCast(s.pid), .kind = .map, .a = .{ m.start, m.end - m.start, m.prot, m.offset, 0, 0 } }, m.path);
+ }
+ }
+
+ /// Each counter's total so far, as `sample` events and in `counts`.
+ pub fn readCounters(s: *Session) !void {
+ const ts = event.now();
+ for (s.options.counters, s.counter_fds, s.counts) |c, fd, *count| {
+ // value, time enabled, time running
+ var v: [3]u64 = undefined;
+ if (linux.read(fd, @ptrCast(&v), 24) != 24) continue;
+ // A counter that had to share its hardware counted part of the time: scale it.
+ count.* = if (v[2] != 0 and v[2] < v[1]) @intCast(@as(u128, v[0]) * v[1] / v[2]) else v[0];
+ try s.sink.emit(.{ .ts = ts, .tid = @intCast(s.pid), .kind = .sample, .aux = reading, .a = .{ 0, stringId(@tagName(c)), count.*, 0, 0, 0 } }, "");
+ }
+ }
+
+ /// Waits for records, the target's end, or the timeout, then decodes
+ /// what is there. False once every traced task has exited.
+ pub fn poll(s: *Session, timeout_ms: i32) !bool {
+ _ = linux.poll(s.pollfds.ptr, s.pollfds.len, timeout_ms);
+ _ = try s.drain();
+ for (s.pollfds) |p| if (p.revents & linux.POLL.HUP == 0) return true;
+ return false;
+ }
+
+ /// Decodes the records in the rings into the sink, oldest first.
+ /// Returns how many.
+ pub fn drain(s: *Session) !usize {
+ // A record stamped after this waits for the next call, so a ring
+ // read late cannot get ahead of one read early.
+ const limit = event.now();
+ for (s.rings) |*r| r.head = @atomicLoad(u64, &r.meta.data_head, .acquire);
+ var n: usize = 0;
+ while (true) : (n += 1) {
+ var oldest: ?*Ring = null;
+ var oldest_ts = limit;
+ for (s.rings) |*r| {
+ const ts = time(r.peek(s.scratch) orelse continue);
+ if (ts > oldest_ts) continue;
+ oldest = r;
+ oldest_ts = ts;
+ }
+ const r = oldest orelse break;
+ const rec = r.peek(s.scratch).?;
+ try s.decode(rec);
+ r.tail += rec.len;
+ }
+ for (s.rings) |*r| @atomicStore(u64, &r.meta.data_tail, r.tail, .release);
+ return n;
+ }
+
+ fn decode(s: *Session, rec: []const u8) !void {
+ const body = rec[8..];
+ const misc = std.mem.readInt(u16, rec[4..6], .little);
+ const tail = rec[rec.len - trailer ..];
+ var ev: Event = .{ .ts = word(tail, 8), .tid = half(tail, 4), .kind = undefined };
+ switch (@as(PERF.RECORD, @fromBackingInt(@intCast(half(rec, 0))))) {
+ .SAMPLE => return s.sample(body),
+ .MMAP2 => {
+ ev.kind = .map;
+ ev.a = .{ word(body, 8), word(body, 16), half(body, 56), word(body, 24), 0, 0 };
+ const path = std.mem.sliceTo(body[64 .. body.len - trailer], 0);
+ try s.sink.emit(ev, if (std.mem.eql(u8, path, "//anon")) "" else path);
+ },
+ .COMM => {
+ var text: [64]u8 = undefined;
+ const comm = std.mem.sliceTo(body[8 .. body.len - trailer], 0);
+ ev.kind = .mark;
+ try s.sink.emit(ev, try std.fmt.bufPrint(&text, "{s} {s}", .{ if (misc & 1 << 13 != 0) "exec" else "comm", comm }));
+ },
+ .FORK => {
+ ev.kind = .thread_start;
+ ev.tid = half(body, 12);
+ ev.a[0] = half(body, 8);
+ ev.a[1] = half(body, 0);
+ try s.sink.emit(ev, "");
+ },
+ .EXIT => {
+ ev.kind = .thread_exit;
+ try s.sink.emit(ev, "");
+ },
+ .SWITCH => {
+ const out = misc & 1 << 13 != 0;
+ ev.kind = .sched;
+ ev.aux = @intFromBool(misc & 1 << 14 != 0);
+ ev.a = .{ if (out) ev.tid else 0, if (out) 0 else ev.tid, half(tail, 16), 0, 0, 0 };
+ try s.sink.emit(ev, "");
+ },
+ .LOST => s.lost += word(body, 8),
+ .LOST_SAMPLES => s.lost += word(body, 0),
+ else => {},
+ }
+ }
+
+ fn sample(s: *Session, body: []const u8) !void {
+ const source = s.sources.get(word(body, 0)) orelse return;
+ const pc = word(body, 8);
+ const addr = word(body, 32);
+ var ev: Event = .{ .ts = word(body, 24), .tid = half(body, 20), .kind = undefined };
+ const rest = body[sample_head..];
+ switch (source) {
+ .sideband => {},
+ .profile => {
+ // The counter overflowed on the way into the kernel: not a place in the program.
+ if (pc >> 63 != 0) return;
+ ev.kind = .sample;
+ ev.a = .{ pc, stringId("cycles"), word(body, 48), addr, 0, 0 };
+ var chain: []const u8 = "";
+ if (s.options.callchain) {
+ // The first entry says "user space from here", not a pc.
+ const frames: usize = @intCast(word(rest, 0));
+ if (frames > 1) chain = rest[16..][0 .. (frames - 1) * 8];
+ // The recorder takes a payload the size of `Regs` for registers.
+ if (chain.len == @sizeOf(Regs)) chain.len -= 8;
+ }
+ try s.sink.emit(ev, chain);
+ },
+ .fault => |aux| {
+ const regs = userRegs(rest);
+ ev.kind = .fault;
+ ev.aux = aux;
+ ev.a = .{ addr, pc, 0, 0, 0, 0 };
+ try s.sink.emit(ev, if (regs) |*r| std.mem.asBytes(r) else "");
+ },
+ .watch => |w| switch (w.mode) {
+ .write => {
+ // The value now: the target has run on since it wrote.
+ var bytes: [8]u8 = undefined;
+ const n = (ptrace.Process{ .pid = s.pid }).read(addr, bytes[0..w.len]);
+ ev.kind = .mem_write;
+ ev.a = .{ addr, w.len, pc, 0, 0, 0 };
+ try s.sink.emit(ev, bytes[0..n]);
+ },
+ .access => {
+ ev.kind = .mem_read;
+ ev.a = .{ addr, w.len, pc, 0, 0, 0 };
+ try s.sink.emit(ev, "");
+ },
+ .exec => {
+ const regs = userRegs(rest) orelse Regs{};
+ ev.kind = .probe;
+ ev.a = .{ w.name, regs.rdi, regs.rsi, regs.rdx, regs.rcx, regs.r8 };
+ try s.sink.emit(ev, std.mem.asBytes(&regs));
+ },
+ },
+ .syscall_enter, .syscall_exit => {
+ // u32 size, then the trace event: 8 common bytes, the number, the arguments.
+ const raw = rest[4..][0..half(rest, 0)];
+ ev.kind = if (source == .syscall_enter) .syscall_enter else .syscall_exit;
+ ev.aux = @truncate(half(raw, 8));
+ for (&ev.a, 0..) |*a, i| {
+ if (16 + 8 * i + 8 <= raw.len) a.* = word(raw, 16 + 8 * i);
+ }
+ try s.sink.emit(ev, if (source == .syscall_enter) userData(raw) else "");
+ },
+ }
+ }
+};
+
+fn word(b: []const u8, at: usize) u64 {
+ return std.mem.readInt(u64, b[at..][0..8], .little);
+}
+
+fn half(b: []const u8, at: usize) u32 {
+ return std.mem.readInt(u32, b[at..][0..4], .little);
+}
+
+/// When a record happened: at a fixed place in a sample, at the end of the rest.
+fn time(rec: []const u8) u64 {
+ return if (half(rec, 0) == @backingInt(PERF.RECORD.SAMPLE)) word(rec, 8 + 24) else word(rec, rec.len - trailer + 8);
+}
+
+/// The registers of a sample taken with `perf_regs_mask`: an ABI word (0
+/// when the task had no user registers), then the values.
+fn userRegs(b: []const u8) ?Regs {
+ if (word(b, 0) == 0) return null;
+ var regs: Regs = .{ .orig_rax = std.math.maxInt(u64) };
+ inline for (perf_regs, 1..) |reg, i| @field(regs, reg) = word(b, 8 * i);
+ return regs;
+}
+
+/// The first piece of user memory the kernel copied into a syscall's entry
+/// record (a path, the bytes written; Linux 6.19 and later), else nothing.
+/// After the arguments come up to three u32 (offset | length << 16) and the
+/// data they point at; how many arguments there are is not in the record,
+/// so each count is tried until one accounts for the record's exact size.
+fn userData(raw: []const u8) []const u8 {
+ for (0..7) |args| for (1..4) |pieces| {
+ const metas = 16 + 8 * args;
+ var at = metas + 4 * pieces;
+ if (at > raw.len) continue;
+ const first = half(raw, metas);
+ for (0..pieces) |i| {
+ const meta = half(raw, metas + 4 * i);
+ if (meta & 0xffff != at) break;
+ at += meta >> 16;
+ } else if (std.mem.alignForward(usize, at + 4, 8) - 4 == raw.len) {
+ const data = raw[first & 0xffff ..][0 .. first >> 16];
+ return if (std.mem.endsWith(u8, data, "\x00")) data[0 .. data.len - 1] else data;
+ }
+ };
+ return "";
+}
+
+/// Lets a target held by `ptrace.Process.spawn` run on its own: once the
+/// events are attached, ptrace has done its part.
+pub fn release(p: ptrace.Process) void {
+ _ = linux.ptrace(linux.PTRACE.DETACH, p.pid, 0, 0, 0);
+}
+
+pub const Symbol = struct { addr: u64, size: u64 };
+
+/// Where `pid` has the symbol `name` now, looking through every ELF file
+/// it has mapped, in address order. Shared libraries are there once the
+/// dynamic loader has run, not at the first instruction.
+pub fn lookup(gpa: std.mem.Allocator, pid: linux.pid_t, name: []const u8) !?Symbol {
+ const text = try readMaps(gpa, pid);
+ defer gpa.free(text);
+ var lines = std.mem.tokenizeScalar(u8, text, '\n');
+ while (lines.next()) |line| {
+ const m = parseMap(line) orelse continue;
+ if (m.offset != 0 or m.path.len == 0 or m.path[0] != '/') continue;
+ const path = try gpa.dupeSentinel(u8, m.path, 0);
+ defer gpa.free(path);
+ const elf = usdt.Elf.open(path) catch continue;
+ defer elf.close();
+ const sym = elf.symbol(name) orelse continue;
+ return .{ .addr = sym.value +% elf.bias(m.start), .size = sym.size };
+ }
+ return null;
+}
+
+/// /proc/PID/maps, whole. Free with `gpa`.
+pub fn readMaps(gpa: std.mem.Allocator, pid: linux.pid_t) ![]u8 {
+ var path_buf: [64]u8 = undefined;
+ const path = try std.fmt.bufPrintSentinel(&path_buf, "/proc/{d}/maps", .{pid}, 0);
+ const fd = linux.open(path, .{}, 0);
+ if (linux.errno(fd) != .SUCCESS) return error.NoSuchProcess;
+ defer _ = linux.close(@intCast(fd));
+ var text: std.ArrayList(u8) = .empty;
+ errdefer text.deinit(gpa);
+ while (true) {
+ try text.ensureUnusedCapacity(gpa, 4096);
+ const n = linux.read(@intCast(fd), text.unusedCapacitySlice().ptr, text.unusedCapacitySlice().len);
+ if (linux.errno(n) != .SUCCESS or n == 0) break;
+ text.items.len += n;
+ }
+ return text.toOwnedSlice(gpa);
+}
+
+pub const Map = struct { start: u64, end: u64, prot: u64, offset: u64, path: []const u8 };
+
+/// A line of /proc/PID/maps.
+pub fn parseMap(line: []const u8) ?Map {
+ var fields = std.mem.tokenizeScalar(u8, line, ' ');
+ const range = fields.next() orelse return null;
+ const perms = fields.next() orelse return null;
+ const offset = fields.next() orelse return null;
+ _ = fields.next();
+ _ = fields.next();
+ const dash = std.mem.indexOfScalar(u8, range, '-') orelse return null;
+ if (perms.len < 3) return null;
+ return .{
+ .start = std.fmt.parseInt(u64, range[0..dash], 16) catch return null,
+ .end = std.fmt.parseInt(u64, range[dash + 1 ..], 16) catch return null,
+ .prot = @as(u64, @intFromBool(perms[0] == 'r')) | @as(u64, @intFromBool(perms[1] == 'w')) << 1 | @as(u64, @intFromBool(perms[2] == 'x')) << 2,
+ .offset = std.fmt.parseInt(u64, offset, 16) catch return null,
+ .path = fields.rest(),
+ };
+}
+
+fn perfOpen(attr: *linux.perf_event_attr, pid: linux.pid_t, cpu: i32) !linux.fd_t {
+ // Paranoid 2 refuses anything that could see the kernel.
+ attr.flags.exclude_kernel = true;
+ attr.flags.exclude_hv = true;
+ const rc = linux.perf_event_open(attr, pid, cpu, -1, PERF.FLAG.FD_CLOEXEC);
+ return switch (linux.errno(rc)) {
+ .SUCCESS => @intCast(rc),
+ .ACCES, .PERM => error.PerfAccess,
+ .NOENT, .INVAL, .NODEV, .OPNOTSUPP => error.PerfNoSuchEvent,
+ .NOSPC, .BUSY => error.NoFreeBreakpoint,
+ .SRCH => error.NoSuchProcess,
+ .MFILE, .NFILE => error.ProcessFdQuotaExceeded,
+ else => error.PerfEventOpen,
+ };
+}
+
+test "a maps line; the user data of a syscall record" {
+ const m = parseMap("7f80b4224000-7f80b439f000 r-xp 00024000 08:02 1234 /usr/lib/libc.so.6").?;
+ try std.testing.expectEqual(@as(u64, 0x7f80b4224000), m.start);
+ try std.testing.expectEqual(@as(u64, 0x17b000), m.end - m.start);
+ try std.testing.expectEqual(@as(u64, 5), m.prot);
+ try std.testing.expectEqual(@as(u64, 0x24000), m.offset);
+ try std.testing.expectEqualStrings("/usr/lib/libc.so.6", m.path);
+ try std.testing.expectEqualStrings("", parseMap("7ffd1000-7ffd2000 rw-p 00000000 00:00 0").?.path);
+
+ // openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC, 0) as Linux 7.2 records it.
+ var raw: [76]u8 = @splat(0);
+ std.mem.writeInt(u32, raw[8..12], 257, .little);
+ std.mem.writeInt(u32, raw[48..52], 52 | 17 << 16, .little);
+ @memcpy(raw[52..68], "/etc/ld.so.cache");
+ try std.testing.expectEqualStrings("/etc/ld.so.cache", userData(&raw));
+ // An argument that looks like a pointer into the record is not one: the sizes disagree.
+ std.mem.writeInt(u32, raw[16..20], 20 | 4 << 16, .little);
+ try std.testing.expectEqualStrings("/etc/ld.so.cache", userData(&raw));
+ // An older kernel: arguments only.
+ try std.testing.expectEqualStrings("", userData(raw[0..52]));
+}
diff --git a/src/trace/root.zig b/src/trace/root.zig
index fd91d9a..80745cb 100644
--- a/src/trace/root.zig
+++ b/src/trace/root.zig
@@ -16,6 +16,10 @@ pub const syscalls = @import("syscalls.zig");
pub const client = @import("client.zig");
/// Inside the tracer: a target running with a ring.
pub const host = @import("host.zig");
+/// The kernel's view, unprivileged: counters, samples, faults, maps, syscalls, watchpoints.
+pub const perf = @import("perf.zig");
+/// Static probes (the ELF notes of <sys/sdt.h>) and their arguments.
+pub const usdt = @import("usdt.zig");
test {
_ = event;
@@ -27,4 +31,6 @@ test {
_ = syscalls;
_ = client;
_ = host;
+ _ = perf;
+ _ = usdt;
}
diff --git a/src/trace/usdt.zig b/src/trace/usdt.zig
new file mode 100644
index 0000000..ba03079
--- /dev/null
+++ b/src/trace/usdt.zig
@@ -0,0 +1,284 @@
+//! USDT, the static probes of DTrace and SystemTap: a program built with
+//! <sys/sdt.h> has a nop where each probe is and an ELF note saying where
+//! that is and where the probe's arguments live then. This reads the notes
+//! and fetches the arguments from a stopped thread's registers and memory.
+//! Getting stopped there is the caller's business: a breakpoint under
+//! ptrace, a uprobe with BPF. Also the little of ELF both need: sections,
+//! symbols, where a file's addresses land in a process. x86-64.
+const std = @import("std");
+const linux = std.os.linux;
+const Regs = @import("event.zig").Regs;
+const E = std.elf.Elf64;
+
+/// A 64-bit little-endian ELF file in memory.
+pub const Elf = struct {
+ bytes: []align(4096) const u8,
+
+ /// Maps the file at `path`.
+ pub fn open(path: [*:0]const u8) !Elf {
+ const fd = linux.open(path, .{}, 0);
+ if (linux.errno(fd) != .SUCCESS) return error.FileNotFound;
+ defer _ = linux.close(@intCast(fd));
+ const size = linux.lseek(@intCast(fd), 0, linux.SEEK.END);
+ if (linux.errno(size) != .SUCCESS or size < @sizeOf(E.Ehdr)) return error.NotElf;
+ const base = linux.mmap(null, size, .{ .READ = true }, .{ .TYPE = .PRIVATE }, @intCast(fd), 0);
+ if (linux.errno(base) != .SUCCESS) return error.NotElf;
+ const e: Elf = .{ .bytes = @as([*]align(4096) const u8, @ptrFromInt(base))[0..size] };
+ const ident = e.header().ident;
+ if (!std.mem.eql(u8, &ident.magic, std.elf.MAGIC) or ident.class != .@"64" or ident.data != .@"2LSB") {
+ e.close();
+ return error.NotElf;
+ }
+ return e;
+ }
+
+ pub fn close(e: Elf) void {
+ _ = linux.munmap(e.bytes.ptr, e.bytes.len);
+ }
+
+ fn header(e: Elf) *const E.Ehdr {
+ return @ptrCast(e.bytes.ptr);
+ }
+
+ fn table(e: Elf, comptime T: type, at: u64, count: u64) []align(1) const T {
+ if (at > e.bytes.len or count > (e.bytes.len - at) / @sizeOf(T)) return &.{};
+ return @as([*]align(1) const T, @ptrCast(e.bytes.ptr + at))[0..count];
+ }
+
+ fn contents(e: Elf, s: E.Shdr) []const u8 {
+ return if (s.type == .NOBITS) "" else e.table(u8, s.offset, s.size);
+ }
+
+ pub fn section(e: Elf, name: []const u8) ?E.Shdr {
+ const h = e.header();
+ const all = e.table(E.Shdr, h.shoff, h.shnum);
+ if (h.shstrndx >= all.len) return null;
+ const names = e.contents(all[h.shstrndx]);
+ for (all) |s| {
+ if (s.name < names.len and std.mem.eql(u8, std.mem.sliceTo(names[s.name..], 0), name)) return s;
+ }
+ return null;
+ }
+
+ /// A symbol by name, from the full table when the file still has one,
+ /// else from the dynamic one.
+ pub fn symbol(e: Elf, name: []const u8) ?E.Sym {
+ for ([_][]const u8{ ".symtab", ".dynsym" }) |table_name| {
+ const s = e.section(table_name) orelse continue;
+ const all = e.table(E.Shdr, e.header().shoff, e.header().shnum);
+ if (s.link >= all.len) continue;
+ const names = e.contents(all[s.link]);
+ for (e.table(E.Sym, s.offset, s.size / @sizeOf(E.Sym))) |sym| {
+ if (sym.shndx == 0 or sym.name >= names.len) continue;
+ if (std.mem.eql(u8, std.mem.sliceTo(names[sym.name..], 0), name)) return sym;
+ }
+ }
+ return null;
+ }
+
+ /// The first loaded segment, or the one that has `containing`.
+ fn load(e: Elf, containing: ?u64) ?E.Phdr {
+ for (e.table(E.Phdr, e.header().phoff, e.header().phnum)) |p| {
+ if (p.type != .LOAD) continue;
+ if (containing) |addr| if (addr < p.vaddr or addr >= p.vaddr + p.memsz) continue;
+ return p;
+ }
+ return null;
+ }
+
+ /// What to add to the file's addresses in a process that has the start
+ /// of the file mapped at `start`: 0 for a fixed-position executable.
+ pub fn bias(e: Elf, start: u64) u64 {
+ const p = e.load(null) orelse return 0;
+ return start -% (p.vaddr - p.offset);
+ }
+
+ /// Where in the file the byte at address `addr` (as the file has it) is:
+ /// what a uprobe is placed by.
+ pub fn offset(e: Elf, addr: u64) ?u64 {
+ const p = e.load(addr) orelse return null;
+ return addr - p.vaddr + p.offset;
+ }
+};
+
+pub const Probe = struct {
+ provider: []const u8,
+ name: []const u8,
+ /// Where the nop is.
+ pc: u64,
+ /// The address of the u16 that a tracer increments to enable the probe,
+ /// for a program that tests it before computing arguments; 0 without.
+ semaphore: u64,
+ /// The arguments as the note has them: "-4@%edi 8@-24(%rbp)".
+ args: []const u8,
+};
+
+/// The probes of `elf`, their addresses moved by `bias` (see `Elf.bias`; 0
+/// gives them as the file and `readelf -n` have them). The strings point
+/// into the file. Free the slice with `gpa`.
+pub fn probes(gpa: std.mem.Allocator, elf: Elf, bias: u64) ![]Probe {
+ var list: std.ArrayList(Probe) = .empty;
+ errdefer list.deinit(gpa);
+ const notes = elf.contents(elf.section(".note.stapsdt") orelse return &.{});
+ // A note records where .stapsdt.base was when it was written: a
+ // prelinker moves the section and not the notes.
+ const base_now = if (elf.section(".stapsdt.base")) |s| s.addr else 0;
+ var at: usize = 0;
+ while (at + 12 <= notes.len) {
+ const name_len = std.mem.readInt(u32, notes[at..][0..4], .little);
+ const desc_len = std.mem.readInt(u32, notes[at + 4 ..][0..4], .little);
+ const kind = std.mem.readInt(u32, notes[at + 8 ..][0..4], .little);
+ const name_at = at + 12;
+ const desc_at = name_at + std.mem.alignForward(usize, name_len, 4);
+ at = desc_at + std.mem.alignForward(usize, desc_len, 4);
+ if (at > notes.len) break;
+ const desc = notes[desc_at..][0..desc_len];
+ if (kind != 3 or desc_len < 24 or !std.mem.eql(u8, notes[name_at..][0..name_len], "stapsdt\x00")) continue;
+ const moved = (if (base_now != 0) base_now -% std.mem.readInt(u64, desc[8..16], .little) else 0) +% bias;
+ const semaphore = std.mem.readInt(u64, desc[16..24], .little);
+ var strings = std.mem.splitScalar(u8, desc[24..], 0);
+ try list.append(gpa, .{
+ .pc = std.mem.readInt(u64, desc[0..8], .little) +% moved,
+ .semaphore = if (semaphore == 0) 0 else semaphore +% moved,
+ .provider = strings.next() orelse "",
+ .name = strings.next() orelse "",
+ .args = strings.next() orelse "",
+ });
+ }
+ return list.toOwnedSlice(gpa);
+}
+
+/// One argument of a probe: its size and where it is at the probe.
+pub const Arg = struct {
+ /// 1, 2, 4 or 8 bytes.
+ size: u8,
+ signed: bool,
+ at: union(enum) {
+ constant: i64,
+ register: Register,
+ /// disp(base, index, scale)
+ memory: struct { base: Register, index: ?Register = null, scale: u8 = 1, disp: i64 = 0 },
+ },
+
+ const Register = struct {
+ /// Which u64 of `Regs`.
+ slot: u8,
+ bytes: u8,
+
+ fn value(r: Register, regs: *const Regs) u64 {
+ const all: *const [@sizeOf(Regs) / 8]u64 = @ptrCast(regs);
+ return if (r.bytes == 8) all[r.slot] else all[r.slot] & (@as(u64, 1) << @intCast(r.bytes * 8)) - 1;
+ }
+ };
+
+ /// "-4@%edi", "8@-24(%rbp)", "4@$7", "8@(%rdx,%rax,8)".
+ /// shortcut: no symbol(%rip) operands and no %ah-style registers, like libbpf; add them when a real probe uses one.
+ pub fn parse(text: []const u8) !Arg {
+ const split = std.mem.indexOfScalar(u8, text, '@') orelse return error.BadUsdtArg;
+ const size = std.fmt.parseInt(i8, text[0..split], 10) catch return error.BadUsdtArg;
+ var arg: Arg = .{ .size = @abs(size), .signed = size < 0, .at = undefined };
+ if (std.mem.indexOfScalar(u8, &.{ 1, 2, 4, 8 }, arg.size) == null) return error.BadUsdtArg;
+ const op = text[split + 1 ..];
+ if (op.len == 0) return error.BadUsdtArg;
+ if (op[0] == '$') {
+ arg.at = .{ .constant = std.fmt.parseInt(i64, op[1..], 0) catch return error.BadUsdtArg };
+ } else if (op[0] == '%') {
+ arg.at = .{ .register = try register(op) };
+ } else {
+ const open = std.mem.indexOfScalar(u8, op, '(') orelse return error.BadUsdtArg;
+ if (op[op.len - 1] != ')') return error.BadUsdtArg;
+ var parts = std.mem.splitScalar(u8, op[open + 1 .. op.len - 1], ',');
+ arg.at = .{ .memory = .{
+ .disp = if (open == 0) 0 else std.fmt.parseInt(i64, op[0..open], 0) catch return error.BadUsdtArg,
+ .base = try register(parts.next().?),
+ .index = if (parts.next()) |r| try register(r) else null,
+ .scale = if (parts.next()) |s| std.fmt.parseInt(u8, s, 10) catch return error.BadUsdtArg else 1,
+ } };
+ }
+ return arg;
+ }
+
+ /// The argument's value, sign- or zero-extended, for a thread stopped
+ /// at the probe with `regs`. `memory` is anything with
+ /// `read(addr: u64, buf: []u8) usize` (a `ptrace.Process`). Null when
+ /// the memory cannot be read.
+ pub fn fetch(arg: Arg, regs: *const Regs, memory: anytype) ?u64 {
+ var bytes: [8]u8 = undefined;
+ switch (arg.at) {
+ .constant => |c| std.mem.writeInt(i64, &bytes, c, .little),
+ .register => |r| std.mem.writeInt(u64, &bytes, r.value(regs), .little),
+ .memory => |m| {
+ const index = if (m.index) |r| r.value(regs) *% m.scale else 0;
+ const addr = m.base.value(regs) +% index +% @as(u64, @bitCast(m.disp));
+ if (memory.read(addr, bytes[0..arg.size]) != arg.size) return null;
+ },
+ }
+ return switch (arg.size) {
+ inline 1, 2, 4, 8 => |n| {
+ const U = @Int(.unsigned, n * 8);
+ const v = std.mem.readInt(U, bytes[0..n], .little);
+ return if (arg.signed) @as(u64, @bitCast(@as(i64, @as(@Int(.signed, n * 8), @bitCast(v))))) else v;
+ },
+ else => unreachable,
+ };
+ }
+};
+
+/// "%rax", "%eax", "%ax", "%al", "%r8d"...
+fn register(text: []const u8) !Arg.Register {
+ if (text.len < 3 or text[0] != '%') return error.BadUsdtArg;
+ const name = text[1..];
+ // The same register at 8, 4, 2 and 1 bytes.
+ const legacy = [_][4][]const u8{
+ .{ "rax", "eax", "ax", "al" }, .{ "rbx", "ebx", "bx", "bl" },
+ .{ "rcx", "ecx", "cx", "cl" }, .{ "rdx", "edx", "dx", "dl" },
+ .{ "rsi", "esi", "si", "sil" }, .{ "rdi", "edi", "di", "dil" },
+ .{ "rbp", "ebp", "bp", "bpl" }, .{ "rsp", "esp", "sp", "spl" },
+ };
+ inline for (legacy) |names| inline for (names, [_]u8{ 8, 4, 2, 1 }) |n, bytes| {
+ if (std.mem.eql(u8, name, n)) return .{ .slot = @offsetOf(Regs, names[0]) / 8, .bytes = bytes };
+ };
+ inline for (8..16) |number| {
+ const full = std.fmt.comptimePrint("r{d}", .{number});
+ if (std.mem.startsWith(u8, name, full)) {
+ const bytes: u8 = if (name.len == full.len) 8 else if (name.len != full.len + 1) 0 else switch (name[full.len]) {
+ 'd' => 4,
+ 'w' => 2,
+ 'b' => 1,
+ else => 0,
+ };
+ if (bytes != 0) return .{ .slot = @offsetOf(Regs, full) / 8, .bytes = bytes };
+ }
+ }
+ return error.BadUsdtArg;
+}
+
+test "argument specs are parsed and fetched" {
+ const regs: Regs = .{ .rdi = 0xffff_ffff_ffff_fffe, .rbp = 0x1000, .rdx = 0x2000, .rax = 3, .r10 = 0x1_0000_00ff };
+ // A memory of its own: every address holds its own low byte, counting up.
+ const memory = struct {
+ fn read(_: @This(), addr: u64, buf: []u8) usize {
+ if (addr >= 0x8000) return 0;
+ for (buf, 0..) |*b, i| b.* = @truncate(addr + i);
+ return buf.len;
+ }
+ }{};
+ const cases = [_]struct { []const u8, ?u64 }{
+ .{ "-4@%edi", @bitCast(@as(i64, -2)) },
+ .{ "4@%edi", 0xffff_fffe },
+ .{ "8@%rdi", 0xffff_ffff_ffff_fffe },
+ .{ "1@%r10b", 0xff },
+ .{ "4@%r10d", 0xff },
+ .{ "-4@$-7", @bitCast(@as(i64, -7)) },
+ .{ "8@$0x10", 16 },
+ .{ "2@-24(%rbp)", 0xe9e8 },
+ .{ "1@(%rdx)", 0 },
+ .{ "4@8(%rdx,%rax,8)", 0x23222120 },
+ .{ "-1@-1(%rbp)", @bitCast(@as(i64, -1)) },
+ .{ "8@0x7000(%rbp)", null },
+ };
+ for (cases) |case| try std.testing.expectEqual(case[1], (try Arg.parse(case[0])).fetch(&regs, memory));
+ for ([_][]const u8{ "4@counter(%rip)", "3@%eax", "4@%ah", "4", "4@", "4@8(%rdx", "4@%r8x" }) |bad| {
+ try std.testing.expectError(error.BadUsdtArg, Arg.parse(bad));
+ }
+}