diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/cli/main.zig | 2 | ||||
| -rw-r--r-- | src/cli/perf.zig | 195 | ||||
| -rw-r--r-- | src/trace/kernel_test.zig | 346 | ||||
| -rw-r--r-- | src/trace/perf.zig | 619 | ||||
| -rw-r--r-- | src/trace/root.zig | 6 | ||||
| -rw-r--r-- | src/trace/usdt.zig | 284 |
6 files changed, 1452 insertions, 0 deletions
diff --git a/src/cli/main.zig b/src/cli/main.zig index 1ac019f..9e6b0e5 100644 --- a/src/cli/main.zig +++ b/src/cli/main.zig @@ -7,6 +7,7 @@ const commands = .{ .record = @import("record.zig"), .dbg = @import("dbg.zig"), .run = @import("run.zig"), + .perf = @import("perf.zig"), .dump = @import("dump.zig"), .show = @import("show.zig"), .cc = @import("cc.zig"), @@ -18,6 +19,7 @@ const usage = \\ record run a program under ptrace into a recording \\ dbg debug a program, recording it: step, break, look, go back \\ run record a program that carries a ring producer (agent, instrumentation) + \\ perf record what the kernel sees of a program at full speed (perf events) \\ dump print a recording's events \\ show print a recording's summary, or the program's state at a moment \\ cc compile a program with instrumentation that records it diff --git a/src/cli/perf.zig b/src/cli/perf.zig new file mode 100644 index 0000000..57ad11a --- /dev/null +++ b/src/cli/perf.zig @@ -0,0 +1,195 @@ +//! ztrace perf: a program at full speed, the kernel taking notes (perf +//! events). No privilege needed, no ptrace once it runs. +const std = @import("std"); +const linux = std.os.linux; +const zt = @import("ztrace"); +const util = @import("util.zig"); +const fail = util.fail; +const Kind = zt.Kind; + +pub const usage = + \\usage: ztrace perf [-o FILE] [OPTIONS] -- PROGRAM [ARGS] + \\ + \\Runs PROGRAM at full speed while the kernel records it through perf + \\events, as any user may on their own processes, into a recording + \\(default out.ztrace). Always recorded: its memory maps, the threads it + \\starts and ends, its exit. + \\ + \\ -e NAME,... counters, read when it ends (default instructions,cycles): + \\ instructions cycles branches branch-misses + \\ cache-references cache-misses page-faults task-clock + \\ --interval MS ... and every MS milliseconds + \\ --sample HZ a profile: where it is, HZ times a second + \\ --callchain ... and who called that (needs frame pointers) + \\ --faults every page fault: address, pc, registers + \\ --switches every time it leaves or gets a CPU + \\ --syscalls every syscall: arguments, result, the path it names + \\ --watch WHAT[:LEN][:rw] + \\ a debug register on a symbol or an address: every + \\ write to its LEN bytes (1, 2, 4, 8; default the symbol's + \\ size) with the value found there; with :rw every access + \\ --break WHAT ... on code: every time it runs, with the registers + \\ --threads also the threads and children it creates + \\ --ring BYTES the kernel's buffer, per CPU (a power of two, default 512 KiB) + \\ + \\There are four debug registers. Only user space is seen: a page the + \\kernel touches first (a read() into fresh memory) faults unseen. + \\ +; + +const Spec = struct { what: []const u8, len: ?u8 = null, mode: zt.perf.Watch.Mode }; + +pub fn run(init: std.process.Init, args: []const [:0]const u8) !void { + const arena = init.arena.allocator(); + var out: []const u8 = "out.ztrace"; + var options: zt.perf.Options = .{ .counters = &.{ .instructions, .cycles } }; + var interval_ms: u32 = 0; + var specs: std.ArrayList(Spec) = .empty; + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + const value: ?[]const u8 = if (i + 1 < args.len) args[i + 1] else null; + if (std.mem.eql(u8, arg, "--")) { + i += 1; + break; + } else if (std.mem.eql(u8, arg, "--callchain")) { + options.callchain = true; + } else if (std.mem.eql(u8, arg, "--faults")) { + options.faults = true; + } else if (std.mem.eql(u8, arg, "--switches")) { + options.switches = true; + } else if (std.mem.eql(u8, arg, "--syscalls")) { + options.syscalls = true; + } else if (std.mem.eql(u8, arg, "--threads")) { + options.follow = true; + } else if (value == null) { + fail(usage, .{}); + } else if (std.mem.eql(u8, arg, "-o")) { + out = value.?; + i += 1; + } else if (std.mem.eql(u8, arg, "-e")) { + var counters: std.ArrayList(zt.perf.Counter) = .empty; + var names = std.mem.tokenizeScalar(u8, value.?, ','); + while (names.next()) |name| try counters.append(arena, std.meta.stringToEnum(zt.perf.Counter, name) orelse fail("-e {s}: no such counter\n", .{name})); + options.counters = counters.items; + i += 1; + } else if (std.mem.eql(u8, arg, "--interval")) { + interval_ms = std.fmt.parseInt(u32, value.?, 0) catch fail("--interval needs milliseconds\n", .{}); + i += 1; + } else if (std.mem.eql(u8, arg, "--sample")) { + options.sample_hz = std.fmt.parseInt(u32, value.?, 0) catch fail("--sample needs a rate\n", .{}); + i += 1; + } else if (std.mem.eql(u8, arg, "--ring")) { + options.ring_bytes = std.fmt.parseInt(usize, value.?, 0) catch 0; + if (!std.math.isPowerOfTwo(options.ring_bytes) or options.ring_bytes < 4096) fail("--ring needs a power of two, 4096 or more\n", .{}); + i += 1; + } else if (std.mem.eql(u8, arg, "--break")) { + try specs.append(arena, .{ .what = value.?, .mode = .exec }); + i += 1; + } else if (std.mem.eql(u8, arg, "--watch")) { + var spec: Spec = .{ .what = value.?, .mode = .write }; + var parts = std.mem.splitScalar(u8, value.?, ':'); + spec.what = parts.first(); + while (parts.next()) |part| { + if (std.mem.eql(u8, part, "rw")) spec.mode = .access else spec.len = std.fmt.parseInt(u8, part, 0) catch fail("--watch {s}: {s} is neither a length nor rw\n", .{ value.?, part }); + } + try specs.append(arena, spec); + i += 1; + } else fail(usage, .{}); + } + if (i >= args.len) fail(usage, .{}); + + const path = try util.resolve(init, args[i]); + const argv = try util.targetArgv(arena, args[i..]); + // Held at its first instruction until the events are on it. + var proc = try zt.ptrace.Process.spawn(path, argv, init.minimal.environ.block.slice.ptr); + errdefer proc.kill(); + var rec = try zt.Recorder.create(init.gpa, init.io, out); + // What changes the state a moment holds (registers, memory) ends one. + rec.cut |= Kind.fault.bit() | Kind.probe.bit() | Kind.mem_write.bit() | Kind.mem_read.bit(); + var session = zt.perf.Session.open(init.gpa, proc.pid, options, rec.sink()) catch |err| switch (err) { + error.PerfAccess => fail("perf_event_open was refused: kernel.perf_event_paranoid is above 2, or a security module forbids it\n", .{}), + error.PerfNoSuchEvent => fail("perf_event_open: this CPU or kernel lacks one of the events asked for (in a VM without a PMU, try -e task-clock; --sample takes at most kernel.perf_event_max_sample_rate)\n", .{}), + else => |e| return e, + }; + defer session.close(); + + var at_entry = false; + for (specs.items) |spec| { + const sym: zt.perf.Symbol = if (std.fmt.parseInt(u64, spec.what, 0)) |addr| .{ .addr = addr, .size = 8 } else |_| found: { + if (try zt.perf.lookup(init.gpa, proc.pid, spec.what)) |sym| break :found sym; + // A library's symbol: there once the loader has mapped it. + if (!at_entry) try runToEntry(&proc); + at_entry = true; + break :found try zt.perf.lookup(init.gpa, proc.pid, spec.what) orelse fail("{s}: no such symbol in what the program has mapped\n", .{spec.what}); + }; + const len = spec.len orelse if (std.mem.indexOfScalar(u64, &.{ 1, 2, 4 }, sym.size)) |_| @as(u8, @intCast(sym.size)) else 8; + session.watch(.{ .addr = sym.addr, .len = len, .mode = spec.mode, .name = spec.what }) catch |err| switch (err) { + error.NoFreeBreakpoint => fail("{s}: no debug register left, there are four\n", .{spec.what}), + error.PerfNoSuchEvent => fail("{s}: a watchpoint covers 1, 2, 4 or 8 bytes at an address aligned to that, not {d} at {x}\n", .{ spec.what, len, sym.addr }), + else => |e| return e, + }; + } + + zt.perf.release(proc); + const start = zt.event.now(); + var next_read = start; + while (true) { + var wait_ms: i32 = 100; + if (interval_ms != 0) { + const now = zt.event.now(); + if (now >= next_read) { + try session.readCounters(); + next_read += @as(u64, interval_ms) * std.time.ns_per_ms; + } + wait_ms = @intCast((next_read -| now) / std.time.ns_per_ms + 1); + } + if (!try session.poll(wait_ms)) break; + } + try session.readCounters(); + var status: u32 = 0; + _ = linux.waitpid(proc.pid, @ptrCast(&status), 0); + const exit_code: u64 = if (linux.W.IFEXITED(status)) linux.W.EXITSTATUS(status) else 128 + @as(u64, @backingInt(linux.W.TERMSIG(status))); + const ns = zt.event.now() - start; + try rec.emit(.{ .ts = zt.event.now(), .tid = @intCast(proc.pid), .kind = .exit, .a = .{ exit_code, 0, 0, 0, 0, 0 } }, ""); + + const events = rec.count; + const moments = rec.moments + @intFromBool(rec.delta.items.len > 0); + try rec.close(); + const size = (try std.Io.Dir.cwd().statFile(init.io, out, .{})).size; + for (options.counters, session.counts) |c, count| std.debug.print("{d:>16} {t}\n", .{ count, c }); + std.debug.print("{s}: {d} moments, {d} events ({d} lost), {d} bytes, {d:.1} ms, {d:.0} events/s, exit {d}\n", .{ + out, moments, + events, session.lost, + size, @as(f64, @floatFromInt(ns)) / 1e6, + @as(f64, @floatFromInt(events)) / (@as(f64, @floatFromInt(ns)) / 1e9), exit_code, + }); +} + +/// Runs the held target up to its own entry point, where the dynamic +/// loader is done and its libraries are mapped. +fn runToEntry(proc: *zt.ptrace.Process) !void { + var path_buf: [64]u8 = undefined; + const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/proc/{d}/auxv", .{proc.pid}, 0), .{}, 0); + if (linux.errno(fd) != .SUCCESS) return error.NoSuchProcess; + defer _ = linux.close(@intCast(fd)); + var auxv: [64][2]u64 = undefined; + const n = linux.read(@intCast(fd), @ptrCast(&auxv), @sizeOf(@TypeOf(auxv))); + if (linux.errno(n) != .SUCCESS) return error.NoSuchProcess; + const entry = for (auxv[0 .. n / 16]) |pair| { + if (pair[0] == std.elf.AT.ENTRY) break pair[1]; + } else return error.NoEntryPoint; + + var saved: [1]u8 = undefined; + if (proc.read(entry, &saved) != 1) return error.BadAddress; + try proc.write(entry, &.{0xcc}); + while (true) switch (try proc.cont()) { + .trap => break, + .exited, .killed => fail("the program ended before its entry point\n", .{}), + else => {}, + }; + try proc.write(entry, &saved); + var regs = try proc.regs(); + regs.rip = entry; + try proc.setRegs(®s); +} diff --git a/src/trace/kernel_test.zig b/src/trace/kernel_test.zig new file mode 100644 index 0000000..6098c58 --- /dev/null +++ b/src/trace/kernel_test.zig @@ -0,0 +1,346 @@ +//! The kernel area against real programs: perf events on the targets of +//! `targets/kernel_*`, the USDT notes of one checked against readelf and +//! stopped at under ptrace. Built by `build/kernel.zig`, which hands over +//! the targets' paths. +const std = @import("std"); +const linux = std.os.linux; +const zt = @import("ztrace"); +const targets = @import("targets"); +const perf = zt.perf; +const gpa = std.testing.allocator; +const expect = std.testing.expect; +const expectEqual = std.testing.expectEqual; + +/// A sink that keeps everything. +const Log = struct { + items: std.ArrayList(Item) = .empty, + counts: [8]u64 = @splat(0), + + const Item = struct { ev: zt.Event, payload: []u8 }; + + fn sink(l: *Log) zt.Sink { + return .{ .ctx = l, .emitFn = emit }; + } + + fn emit(ctx: *anyopaque, ev: *const zt.Event, payload: []const u8) anyerror!void { + const l: *Log = @ptrCast(@alignCast(ctx)); + try l.items.append(gpa, .{ .ev = ev.*, .payload = try gpa.dupe(u8, payload) }); + } + + fn deinit(l: *Log) void { + for (l.items.items) |item| gpa.free(item.payload); + l.items.deinit(gpa); + } + + fn count(l: *const Log, kind: zt.Kind) usize { + var n: usize = 0; + for (l.items.items) |item| n += @intFromBool(item.ev.kind == kind); + return n; + } + + /// The string a `string` event gave `id`. + fn string(l: *const Log, id: u64) ?[]const u8 { + for (l.items.items) |item| if (item.ev.kind == .string and item.ev.a[0] == id) return item.payload; + return null; + } +}; + +const Armed = struct { symbol: []const u8, mode: perf.Watch.Mode }; + +/// Runs a target to its end with a session on it. `log.counts` gets the +/// counters. Skips the test where the kernel or CPU refuses the events +/// (perf_event_paranoid above 2, a VM without a PMU). +fn record(log: *Log, path: []const u8, args: []const []const u8, options: perf.Options, armed: []const Armed) !void { + var arena_state: std.heap.ArenaAllocator = .init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const argv = try arena.allocSentinel(?[*:0]const u8, args.len + 1, null); + argv[0] = try arena.dupeSentinel(u8, path, 0); + for (argv[1..], args) |*dst, arg| dst.* = try arena.dupeSentinel(u8, arg, 0); + const envp = [_:null]?[*:0]const u8{}; + + var proc = try zt.ptrace.Process.spawn(argv[0].?, argv, &envp); + errdefer proc.kill(); + var session = perf.Session.open(gpa, proc.pid, options, log.sink()) catch |err| switch (err) { + error.PerfAccess, error.PerfNoSuchEvent => { + proc.kill(); + return error.SkipZigTest; + }, + else => |e| return e, + }; + defer session.close(); + for (armed) |a| { + const sym = (try perf.lookup(gpa, proc.pid, a.symbol)).?; + try session.watch(.{ .addr = sym.addr, .len = @intCast(@min(sym.size, 8)), .mode = a.mode, .name = a.symbol }); + } + perf.release(proc); + while (try session.poll(100)) {} + try session.readCounters(); + @memcpy(log.counts[0..session.counts.len], session.counts); + try expectEqual(@as(u64, 0), session.lost); + var status: u32 = 0; + _ = linux.waitpid(proc.pid, @ptrCast(&status), 0); + try expect(linux.W.IFEXITED(status) and linux.W.EXITSTATUS(status) == 0); +} + +test "counters: the instructions of a loop of known length" { + var log: Log = .{}; + defer log.deinit(); + try record(&log, targets.spin, &.{"5000000"}, .{ .counters = &.{ .instructions, .branches, .@"task-clock" } }, &.{}); + // Four instructions and one branch an iteration, plus the loader and libc starting up. + try expect(log.counts[0] >= 20_000_000 and log.counts[0] < 20_600_000); + try expect(log.counts[1] >= 5_000_000 and log.counts[1] < 5_150_000); + try expect(log.counts[2] > 100_000); + // ... and the same as events, named. + const last = log.items.items[log.items.items.len - 3]; + try expectEqual(zt.Kind.sample, last.ev.kind); + try expectEqual(perf.reading, last.ev.aux); + try expectEqual(log.counts[0], last.ev.a[2]); + try std.testing.expectEqualStrings("instructions", log.string(last.ev.a[1]).?); +} + +test "faults: every fresh page touched, with the registers; maps: the program itself and what it maps" { + var log: Log = .{}; + defer log.deinit(); + const pages = 64; + try record(&log, targets.touch, &.{std.fmt.comptimePrint("{d}", .{pages})}, .{ .faults = true }, &.{}); + + // Text of the program itself (mapped before we came: from /proc), of + // libc (mapped after: from the kernel's records), and anonymous memory. + var text: [2]u64 = .{ 0, 0 }; + var libc = false; + var anonymous: std.ArrayList([2]u64) = .empty; + defer anonymous.deinit(gpa); + var touched: usize = 0; + for (log.items.items) |item| switch (item.ev.kind) { + .map => { + const range: [2]u64 = .{ item.ev.a[0], item.ev.a[0] + item.ev.a[1] }; + const executable = item.ev.a[2] & 4 != 0; + if (std.mem.endsWith(u8, item.payload, "/kernel_touch") and executable) text = range; + if (std.mem.indexOf(u8, item.payload, "libc.so") != null and executable) libc = true; + // The target's own mmap is in one of these: the kernel reports + // it merged with its neighbour, not by itself. + if (item.payload.len == 0 and item.ev.a[1] >= pages * 4096) try anonymous.append(gpa, range); + }, + .fault => { + const regs = std.mem.bytesToValue(zt.Regs, item.payload); + try expectEqual(item.ev.a[1], regs.rip); + try expect(regs.rsp != 0); + // Its loop: from its own code, the first byte of a page of anonymous memory. + if (regs.rip < text[0] or regs.rip >= text[1] or item.ev.a[0] % 4096 != 0) continue; + for (anonymous.items) |range| { + if (item.ev.a[0] >= range[0] and item.ev.a[0] < range[1]) break; + } else continue; + touched += 1; + }, + else => {}, + }; + try expect(text[0] != 0 and libc); + try expect(touched >= pages); + try expect(touched <= pages + 2); +} + +test "debug registers: a watchpoint sees every write of a global, a breakpoint every call and its arguments" { + var log: Log = .{}; + defer log.deinit(); + const steps = 100; + try record(&log, targets.touch, &.{ "1", std.fmt.comptimePrint("{d}", .{steps}) }, .{}, &.{ + .{ .symbol = "counter", .mode = .write }, + .{ .symbol = "step", .mode = .exec }, + }); + var writes: u64 = 0; + var calls: u64 = 0; + var sum: u64 = 0; + var step_pc: u64 = 0; + for (log.items.items) |item| switch (item.ev.kind) { + .probe => { + try std.testing.expectEqualStrings("step", log.string(item.ev.a[0]).?); + // step(i, the sum so far) + try expectEqual(calls, item.ev.a[1]); + try expectEqual(sum, item.ev.a[2]); + step_pc = std.mem.bytesToValue(zt.Regs, item.payload).rip; + sum += calls; + calls += 1; + }, + .mem_write => { + try expectEqual(@as(u64, 8), item.ev.a[1]); + // The pc is the instruction after the store, a few bytes into step. + try expect(item.ev.a[2] > step_pc and item.ev.a[2] < step_pc + 32); + // The value is read when the record is: what was written, or something later. + if (item.payload.len == 8) try expect(std.mem.readInt(u64, item.payload[0..8], .little) >= sum); + writes += 1; + }, + else => {}, + }; + // The thread's three calls are not this task's: nobody asked to follow. + try expectEqual(@as(u64, steps), calls); + try expectEqual(@as(u64, steps), writes); +} + +test "syscalls with their paths, context switches, threads followed" { + var log: Log = .{}; + defer log.deinit(); + try record(&log, targets.touch, &.{ "1", "2" }, .{ .syscalls = true, .switches = true, .follow = true }, &.{.{ .symbol = "step", .mode = .exec }}); + + var fd: ?u64 = null; + var opened = false; + var wrote = false; + var slept = false; + var thread: u32 = 0; + var thread_calls: usize = 0; + var last_ts: u64 = 0; + for (log.items.items, 0..) |item, i| { + const ev = item.ev; + // One stream in time order, whichever CPU's ring a record came from. + if (ev.kind != .string and ev.kind != .map) { + try expect(ev.ts >= last_ts); + last_ts = ev.ts; + } + switch (ev.kind) { + .syscall_enter => switch (@as(linux.SYS, @fromBackingInt(@intCast(ev.aux)))) { + // Linux 6.19 and later copy the path and the bytes written into the record. + .openat => if (ev.a[1] != 0 and (item.payload.len == 0 or std.mem.eql(u8, item.payload, "/dev/null")) and ev.a[2] & 3 == 1) { + opened = true; + // Its exit is the same thread's next event. + for (log.items.items[i + 1 ..]) |next| if (next.ev.tid == ev.tid) { + try expectEqual(zt.Kind.syscall_exit, next.ev.kind); + try expectEqual(ev.aux, next.ev.aux); + fd = next.ev.a[0]; + break; + }; + }, + .write => if (fd != null and ev.a[0] == fd.? and ev.a[2] == 5) { + try expect(item.payload.len == 0 or std.mem.eql(u8, item.payload, "hello")); + wrote = true; + }, + .nanosleep, .clock_nanosleep => slept = true, + else => {}, + }, + // It left the CPU to sleep. + .sched => if (slept and ev.a[0] == ev.tid) { + slept = false; + thread_calls += 1000; + }, + .thread_start => thread = @intCast(ev.a[0]), + .probe => if (ev.tid == thread and thread != 0) { + try expect(ev.a[1] >= 1000); + thread_calls += 1; + }, + else => {}, + } + } + try expect(opened and wrote and thread != 0); + try expectEqual(@as(usize, 1003), thread_calls); + try expect(log.count(.syscall_enter) == log.count(.syscall_exit) + 1 or log.count(.syscall_enter) == log.count(.syscall_exit) + 2); + try expectEqual(@as(usize, 2), log.count(.thread_exit)); +} + +test "a profile: samples in the program's code, their callchains starting there" { + var log: Log = .{}; + defer log.deinit(); + try record(&log, targets.spin, &.{"100000000"}, .{ .sample_hz = 5000, .callchain = true }, &.{}); + var text: [2]u64 = .{ 0, 0 }; + var in_text: usize = 0; + for (log.items.items) |item| switch (item.ev.kind) { + .map => if (std.mem.endsWith(u8, item.payload, "/kernel_spin") and item.ev.a[2] & 4 != 0) { + text = .{ item.ev.a[0], item.ev.a[0] + item.ev.a[1] }; + }, + .sample => { + try std.testing.expectEqualStrings("cycles", log.string(item.ev.a[1]).?); + try expect(item.payload.len >= 8 and item.payload.len % 8 == 0); + try expectEqual(item.ev.a[0], std.mem.readInt(u64, item.payload[0..8], .little)); + in_text += @intFromBool(item.ev.a[0] >= text[0] and item.ev.a[0] < text[1]); + }, + else => {}, + }; + // All but the few taken while the loader ran. + try expect(in_text >= 20 and in_text + 10 >= log.count(.sample)); +} + +test "usdt: the notes say what readelf says; arguments are fetched at a probe" { + const io = std.testing.io; + const path = try gpa.dupeSentinel(u8, targets.usdt, 0); + defer gpa.free(path); + const elf = try zt.usdt.Elf.open(path); + defer elf.close(); + const found = try zt.usdt.probes(gpa, elf, 0); + defer gpa.free(found); + try expectEqual(@as(usize, 3), found.len); + + const result = std.process.run(gpa, io, .{ .argv = &.{ "readelf", "-n", targets.usdt } }) catch return error.SkipZigTest; + defer gpa.free(result.stdout); + defer gpa.free(result.stderr); + var lines = std.mem.tokenizeScalar(u8, result.stdout, '\n'); + var seen: usize = 0; + while (lines.next()) |line| { + const provider = after(line, "Provider: ") orelse continue; + const probe = found[seen]; + seen += 1; + try std.testing.expectEqualStrings(provider, probe.provider); + try std.testing.expectEqualStrings(after(lines.next().?, "Name: ").?, probe.name); + // Location: 0x..., Base: 0x..., Semaphore: 0x... + var numbers = std.mem.tokenizeAny(u8, lines.next().?, " ,"); + var values: [3]u64 = undefined; + for (&values) |*v| { + _ = numbers.next(); + v.* = try std.fmt.parseInt(u64, numbers.next().?, 0); + } + try expectEqual(values[0], probe.pc); + try expectEqual(values[2], probe.semaphore); + try std.testing.expectEqualStrings(after(lines.next().?, "Arguments: ").?, probe.args); + } + try expectEqual(found.len, seen); + + // What the debugger will do: an int3 on each nop, the semaphore raised, + // and at every stop the arguments read from the registers and memory. + const argv = [_:null]?[*:0]const u8{ path, "4", null }; + const envp = [_:null]?[*:0]const u8{}; + var proc = try zt.ptrace.Process.spawn(path, &argv, &envp); + errdefer proc.kill(); + const maps = try perf.readMaps(gpa, proc.pid); + defer gpa.free(maps); + // The program itself is the first thing mapped. + const live = try zt.usdt.probes(gpa, elf, elf.bias(perf.parseMap(maps[0..std.mem.indexOfScalar(u8, maps, '\n').?]).?.start)); + defer gpa.free(live); + for (live) |probe| { + var nop: [1]u8 = undefined; + try expectEqual(@as(usize, 1), proc.read(probe.pc, &nop)); + try expectEqual(@as(u8, 0x90), nop[0]); + try proc.write(probe.pc, &.{0xcc}); + if (probe.semaphore != 0) try proc.write(probe.semaphore, &.{ 1, 0 }); + } + // start(n), then tick(i, the sum of squares so far) n times, then done(n, sum). + const expected = [_]struct { []const u8, []const u64 }{ + .{ "start", &.{4} }, .{ "tick", &.{ 0, 0 } }, .{ "tick", &.{ 1, 1 } }, + .{ "tick", &.{ 2, 5 } }, .{ "tick", &.{ 3, 14 } }, .{ "done", &.{ 4, 14 } }, + }; + var stops: usize = 0; + while (true) switch (try proc.cont()) { + .trap => { + // The int3 was the whole instruction: the thread is already past the probe. + const regs = try proc.regs(); + const probe = for (live) |probe| { + if (probe.pc == regs.rip - 1) break probe; + } else return error.StoppedElsewhere; + try std.testing.expectEqualStrings(expected[stops][0], probe.name); + var specs = std.mem.tokenizeScalar(u8, probe.args, ' '); + for (expected[stops][1]) |value| { + const arg = try zt.usdt.Arg.parse(specs.next().?); + try expectEqual(@as(?u64, value), arg.fetch(®s, proc)); + } + try expect(specs.next() == null); + stops += 1; + }, + .exited => |code| { + try expectEqual(@as(u8, 0), code); + break; + }, + else => {}, + }; + try expectEqual(expected.len, stops); +} + +fn after(line: []const u8, label: []const u8) ?[]const u8 { + const at = std.mem.indexOf(u8, line, label) orelse return null; + return line[at + label.len ..]; +} diff --git a/src/trace/perf.zig b/src/trace/perf.zig new file mode 100644 index 0000000..7c9ef43 --- /dev/null +++ b/src/trace/perf.zig @@ -0,0 +1,619 @@ +//! The kernel's view of a task without privilege: `perf_event_open` as far +//! as `perf_event_paranoid=2` lets a user go on their own processes (user +//! space only). Counters, a sampling profile, every page fault, the memory +//! maps, threads, context switches, every syscall, and the debug registers +//! as watchpoints. The target runs at full speed: the kernel writes records +//! into ring buffers, stamped with CLOCK_MONOTONIC like every other +//! backend's events, and `Session` decodes them into a `Sink`. +//! +//! What becomes what: +//! counter reading sample aux=`reading`, a1=name string id, a2=total so far +//! profile sample sample a0=pc a1=name string id a2=period a3=data address. Payload: the callchain, u64 pcs +//! page fault fault a0=address a1=pc, aux=`fault_major` or 0. Payload: `Regs` +//! mapping map as the contract says; the path is `[heap]`, `[stack]`... for those +//! clone, exit thread_start (tid=parent, a0=new tid, a1=its pid), thread_exit +//! comm mark "comm NAME", or "exec NAME" +//! context switch sched out: a0=tid, in: a1=tid; a2=cpu, aux=1 when preempted +//! syscall syscall_enter, syscall_exit. Payload of an enter: the path or buffer the kernel captured +//! watchpoint mem_write (payload: the bytes there when the record is decoded), mem_read; a2=pc after the access +//! execute breakpoint probe a0=name string id a1..a5=rdi rsi rdx rcx r8. Payload: `Regs` +const std = @import("std"); +const linux = std.os.linux; +const event = @import("event.zig"); +const ptrace = @import("ptrace.zig"); +const usdt = @import("usdt.zig"); +const Event = event.Event; +const Regs = event.Regs; +const PERF = linux.PERF; + +/// `aux` of a `sample` that is a counter's total, not a profile sample. +pub const reading: u16 = 1; +/// `aux` of a `fault` that needed I/O. +pub const fault_major: u16 = 1; + +/// The id a `string` event gives `text`: its hash, so that backends agree +/// without a registry. +pub fn stringId(text: []const u8) u64 { + return std.hash.Wyhash.hash(0, text); +} + +/// What can be counted. Context switches and migrations are missing: their +/// software events fire in kernel mode, which paranoid 2 excludes, so they +/// always read 0 (`Options.switches` gives the switches as events instead). +pub const Counter = enum { + instructions, + cycles, + branches, + @"branch-misses", + @"cache-references", + @"cache-misses", + /// Faults taken in user mode only. + @"page-faults", + /// Nanoseconds on a CPU. + @"task-clock", + + fn attr(c: Counter) linux.perf_event_attr { + const HW = PERF.COUNT.HW; + const SW = PERF.COUNT.SW; + return switch (c) { + .instructions => .{ .type = .HARDWARE, .config = @backingInt(HW.INSTRUCTIONS) }, + .cycles => .{ .type = .HARDWARE, .config = @backingInt(HW.CPU_CYCLES) }, + .branches => .{ .type = .HARDWARE, .config = @backingInt(HW.BRANCH_INSTRUCTIONS) }, + .@"branch-misses" => .{ .type = .HARDWARE, .config = @backingInt(HW.BRANCH_MISSES) }, + .@"cache-references" => .{ .type = .HARDWARE, .config = @backingInt(HW.CACHE_REFERENCES) }, + .@"cache-misses" => .{ .type = .HARDWARE, .config = @backingInt(HW.CACHE_MISSES) }, + .@"page-faults" => .{ .type = .SOFTWARE, .config = @backingInt(SW.PAGE_FAULTS) }, + .@"task-clock" => .{ .type = .SOFTWARE, .config = @backingInt(SW.TASK_CLOCK) }, + }; + } +}; + +/// A debug register on the target: the CPU reports every access, the target +/// never stops. There are four; a data watchpoint covers 1, 2, 4 or 8 bytes +/// at an address aligned to that; the record is written after the access, so +/// `pc` is the instruction after it and the value is not part of it. +pub const Watch = struct { + pub const Mode = enum(u32) { write = 2, access = 3, exec = 4 }; + + addr: u64, + len: u8 = 8, + mode: Mode = .write, + /// What an `exec` hit is called. + name: []const u8 = "", +}; + +pub const Options = struct { + counters: []const Counter = &.{}, + /// Profile samples a second (of cycles); 0 for none. + sample_hz: u32 = 0, + /// Frame-pointer callchains on profile samples. + callchain: bool = false, + faults: bool = false, + switches: bool = false, + syscalls: bool = false, + /// Also the threads and children the target creates from now on. One + /// ring per CPU then, merged by time, as inherited events need. + follow: bool = false, + /// Per ring; a power of two, at least a page. + ring_bytes: usize = 128 * 4096, +}; + +const base_sample = PERF.SAMPLE.IDENTIFIER | PERF.SAMPLE.IP | PERF.SAMPLE.TID | PERF.SAMPLE.TIME | PERF.SAMPLE.ADDR | PERF.SAMPLE.CPU | PERF.SAMPLE.PERIOD; +/// The fixed head of a sample with `base_sample`: id, ip, pid, tid, time, addr, cpu, period. +const sample_head = 56; +/// What follows every other record: pid, tid, time, cpu, id. +const trailer = 32; + +/// The registers perf hands out for a 64-bit task, in its bit order (the +/// segment registers 12..15 are refused). +const perf_regs = [_][]const u8{ "rax", "rbx", "rcx", "rdx", "rsi", "rdi", "rbp", "rsp", "rip", "eflags", "cs", "ss", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" }; +const perf_regs_mask: u64 = 0xff0fff; + +/// What an event writing into a ring means, found by the id in its records. +const Source = union(enum) { + sideband, + profile, + fault: u16, + watch: struct { len: u8, mode: Watch.Mode, name: u64 }, + syscall_enter, + syscall_exit, +}; + +const Ring = struct { + cpu: i32, + /// The event that owns the mapping; the others output into it. + fd: linux.fd_t = -1, + meta: *linux.perf_event_mmap_page = undefined, + data: []u8 = &.{}, + head: u64 = 0, + tail: u64 = 0, + + /// The record at the read position, copied to `scratch` when it wraps. + fn peek(r: *const Ring, scratch: []u8) ?[]const u8 { + if (r.tail == r.head) return null; + const at: usize = @intCast(r.tail & (r.data.len - 1)); + // Records are multiples of 8 bytes: the header itself never wraps. + const size = std.mem.readInt(u16, r.data[at + 6 ..][0..2], .little); + if (at + size <= r.data.len) return r.data[at..][0..size]; + const first = r.data.len - at; + @memcpy(scratch[0..first], r.data[at..]); + @memcpy(scratch[first..size], r.data[0 .. size - first]); + return scratch[0..size]; + } +}; + +pub const Session = struct { + gpa: std.mem.Allocator, + pid: linux.pid_t, + sink: event.Sink, + options: Options, + rings: []Ring = &.{}, + pollfds: []linux.pollfd = &.{}, + fds: std.ArrayList(linux.fd_t) = .empty, + sources: std.AutoHashMapUnmanaged(u64, Source) = .empty, + counter_fds: []linux.fd_t = &.{}, + /// The totals `readCounters` last saw, in `options.counters` order. + counts: []u64 = &.{}, + /// Records the kernel dropped because a ring was full. + lost: u64 = 0, + scratch: []u8 = &.{}, + + /// Attaches to `pid` (a thread id), which should be stopped: events + /// count from here. Emits the names used and the mappings it already has. + pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, options: Options, sink: event.Sink) !Session { + std.debug.assert(std.math.isPowerOfTwo(options.ring_bytes) and options.ring_bytes >= 4096); + var s: Session = .{ .gpa = gpa, .pid = pid, .sink = sink, .options = options }; + errdefer s.close(); + s.scratch = try gpa.alloc(u8, 1 << 16); + // shortcut: CPUs 0..count-1, right unless some are offline or outside our affinity mask. + const cpus = if (options.follow) try std.Thread.getCpuCount() else 1; + s.rings = try gpa.alloc(Ring, cpus); + for (s.rings, 0..) |*r, i| r.* = .{ .cpu = if (options.follow) @intCast(i) else -1 }; + + // An event that counts nothing carries the records about the task + // itself, and comes first: it owns the rings. + var side: linux.perf_event_attr = .{ .type = .SOFTWARE, .config = @backingInt(PERF.COUNT.SW.DUMMY) }; + side.flags.mmap = true; + side.flags.mmap2 = true; + side.flags.mmap_data = true; + side.flags.comm = true; + side.flags.comm_exec = true; + side.flags.task = true; + side.flags.context_switch = options.switches; + try s.add(side, .sideband); + s.pollfds = try gpa.alloc(linux.pollfd, cpus); + for (s.pollfds, s.rings) |*p, r| p.* = .{ .fd = r.fd, .events = linux.POLL.IN, .revents = 0 }; + + s.counter_fds = try gpa.alloc(linux.fd_t, options.counters.len); + @memset(s.counter_fds, -1); + s.counts = try gpa.alloc(u64, options.counters.len); + @memset(s.counts, 0); + for (options.counters, s.counter_fds) |c, *fd| { + var attr = c.attr(); + attr.read_format = 1 | 2; // TOTAL_TIME_ENABLED | TOTAL_TIME_RUNNING + attr.flags.inherit = options.follow; + fd.* = try perfOpen(&attr, pid, -1); + try s.name(@tagName(c)); + } + + if (options.sample_hz != 0) { + var attr = Counter.cycles.attr(); + attr.sample_period_or_freq = options.sample_hz; + attr.flags.freq = true; + attr.flags.exclude_callchain_kernel = true; + if (options.callchain) attr.sample_type = PERF.SAMPLE.CALLCHAIN; + try s.add(attr, .profile); + try s.name("cycles"); + } + if (options.faults) for ([_]PERF.COUNT.SW{ .PAGE_FAULTS_MIN, .PAGE_FAULTS_MAJ }, [_]u16{ 0, fault_major }) |config, aux| { + try s.add(.{ .type = .SOFTWARE, .config = @backingInt(config), .sample_period_or_freq = 1, .sample_type = PERF.SAMPLE.REGS_USER }, .{ .fault = aux }); + }; + if (options.syscalls) try s.addSyscalls(); + try s.maps(); + return s; + } + + pub fn close(s: *Session) void { + for (s.rings) |r| if (r.fd >= 0) { + _ = linux.munmap(@ptrCast(r.meta), 4096 + r.data.len); + }; + for (s.fds.items) |fd| _ = linux.close(fd); + for (s.counter_fds) |fd| if (fd >= 0) { + _ = linux.close(fd); + }; + s.fds.deinit(s.gpa); + s.sources.deinit(s.gpa); + s.gpa.free(s.rings); + s.gpa.free(s.pollfds); + s.gpa.free(s.counter_fds); + s.gpa.free(s.counts); + s.gpa.free(s.scratch); + } + + /// Arms a debug register, on a session already running too. + /// `error.NoFreeBreakpoint` when all four are taken. + pub fn watch(s: *Session, w: Watch) !void { + var attr: linux.perf_event_attr = .{ .type = .BREAKPOINT, .sample_period_or_freq = 1, .bp_type = @backingInt(w.mode), .config1 = w.addr, .config2 = if (w.mode == .exec) 8 else w.len }; + // Where it executes is known; what it was called with is not. + if (w.mode == .exec) attr.sample_type = PERF.SAMPLE.REGS_USER; + try s.add(attr, .{ .watch = .{ .len = w.len, .mode = w.mode, .name = stringId(w.name) } }); + if (w.mode == .exec) try s.name(w.name); + } + + /// One event on every ring, its records told apart by `source`. + fn add(s: *Session, template: linux.perf_event_attr, source: Source) !void { + var attr = template; + attr.sample_type |= base_sample; + if (attr.sample_type & PERF.SAMPLE.REGS_USER != 0) attr.sample_regs_user = perf_regs_mask; + attr.flags.sample_id_all = true; + attr.flags.inherit = s.options.follow; + attr.flags.use_clockid = true; + attr.clockid = .MONOTONIC; + // Wake the reader when a ring is a quarter full (it also looks on its + // own), but at once for a watchpoint: the value is read on waking. + attr.flags.watermark = source != .watch; + attr.wakeup_events_or_watermark = if (source == .watch) 1 else @intCast(s.options.ring_bytes / 4); + for (s.rings) |*r| { + const fd = try perfOpen(&attr, s.pid, r.cpu); + try s.fds.append(s.gpa, fd); + if (r.fd < 0) { + const base = linux.mmap(null, 4096 + s.options.ring_bytes, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0); + if (linux.errno(base) != .SUCCESS) return error.PerfRingMap; + r.fd = fd; + r.meta = @ptrFromInt(base); + r.data = @as([*]u8, @ptrFromInt(base + 4096))[0..s.options.ring_bytes]; + } else if (linux.errno(linux.ioctl(fd, PERF.EVENT_IOC.SET_OUTPUT, @intCast(r.fd))) != .SUCCESS) return error.PerfSetOutput; + var id: u64 = 0; + if (linux.errno(linux.ioctl(fd, PERF.EVENT_IOC.ID, @intFromPtr(&id))) != .SUCCESS) return error.PerfEventId; + try s.sources.put(s.gpa, id, source); + } + } + + /// Every syscall, through the per-syscall tracepoints: the only + /// tracepoints an unprivileged user may read, and only on a task of + /// their own. Their ids are in tracefs, which we may not be able to + /// read, so every id is tried: about 700 open, two a syscall. + fn addSyscalls(s: *Session) !void { + var limit: linux.rlimit = undefined; + if (linux.errno(linux.getrlimit(.NOFILE, &limit)) == .SUCCESS and limit.cur < limit.max) { + limit.cur = limit.max; + _ = linux.setrlimit(.NOFILE, &limit); + } + // shortcut: ids below 4096; scan further if a kernel has more trace events than that. + for (0..4096) |id| { + var attr: linux.perf_event_attr = .{ .type = .TRACEPOINT, .config = id, .sample_period_or_freq = 1, .sample_type = PERF.SAMPLE.RAW }; + // Only a syscall's exit has a field `ret`: a filter on it is + // accepted there and nowhere else. On an event of its own, since + // a filter cannot be taken off again. + const probe = perfOpen(&attr, s.pid, -1) catch |err| switch (err) { + error.PerfAccess, error.PerfNoSuchEvent => continue, + else => |e| return e, + }; + const is_exit = linux.errno(linux.ioctl(probe, PERF.EVENT_IOC.SET_FILTER, @intFromPtr("ret==0"))) == .SUCCESS; + _ = linux.close(probe); + try s.add(attr, if (is_exit) .syscall_exit else .syscall_enter); + } + } + + fn name(s: *Session, text: []const u8) !void { + try s.sink.emit(.{ .ts = event.now(), .tid = @intCast(s.pid), .kind = .string, .a = .{ stringId(text), 0, 0, 0, 0, 0 } }, text); + } + + /// The mappings the target has now, from /proc: the kernel only reports + /// the ones made after an event exists. + fn maps(s: *Session) !void { + const text = try readMaps(s.gpa, s.pid); + defer s.gpa.free(text); + const ts = event.now(); + var lines = std.mem.tokenizeScalar(u8, text, '\n'); + while (lines.next()) |line| { + const m = parseMap(line) orelse continue; + try s.sink.emit(.{ .ts = ts, .tid = @intCast(s.pid), .kind = .map, .a = .{ m.start, m.end - m.start, m.prot, m.offset, 0, 0 } }, m.path); + } + } + + /// Each counter's total so far, as `sample` events and in `counts`. + pub fn readCounters(s: *Session) !void { + const ts = event.now(); + for (s.options.counters, s.counter_fds, s.counts) |c, fd, *count| { + // value, time enabled, time running + var v: [3]u64 = undefined; + if (linux.read(fd, @ptrCast(&v), 24) != 24) continue; + // A counter that had to share its hardware counted part of the time: scale it. + count.* = if (v[2] != 0 and v[2] < v[1]) @intCast(@as(u128, v[0]) * v[1] / v[2]) else v[0]; + try s.sink.emit(.{ .ts = ts, .tid = @intCast(s.pid), .kind = .sample, .aux = reading, .a = .{ 0, stringId(@tagName(c)), count.*, 0, 0, 0 } }, ""); + } + } + + /// Waits for records, the target's end, or the timeout, then decodes + /// what is there. False once every traced task has exited. + pub fn poll(s: *Session, timeout_ms: i32) !bool { + _ = linux.poll(s.pollfds.ptr, s.pollfds.len, timeout_ms); + _ = try s.drain(); + for (s.pollfds) |p| if (p.revents & linux.POLL.HUP == 0) return true; + return false; + } + + /// Decodes the records in the rings into the sink, oldest first. + /// Returns how many. + pub fn drain(s: *Session) !usize { + // A record stamped after this waits for the next call, so a ring + // read late cannot get ahead of one read early. + const limit = event.now(); + for (s.rings) |*r| r.head = @atomicLoad(u64, &r.meta.data_head, .acquire); + var n: usize = 0; + while (true) : (n += 1) { + var oldest: ?*Ring = null; + var oldest_ts = limit; + for (s.rings) |*r| { + const ts = time(r.peek(s.scratch) orelse continue); + if (ts > oldest_ts) continue; + oldest = r; + oldest_ts = ts; + } + const r = oldest orelse break; + const rec = r.peek(s.scratch).?; + try s.decode(rec); + r.tail += rec.len; + } + for (s.rings) |*r| @atomicStore(u64, &r.meta.data_tail, r.tail, .release); + return n; + } + + fn decode(s: *Session, rec: []const u8) !void { + const body = rec[8..]; + const misc = std.mem.readInt(u16, rec[4..6], .little); + const tail = rec[rec.len - trailer ..]; + var ev: Event = .{ .ts = word(tail, 8), .tid = half(tail, 4), .kind = undefined }; + switch (@as(PERF.RECORD, @fromBackingInt(@intCast(half(rec, 0))))) { + .SAMPLE => return s.sample(body), + .MMAP2 => { + ev.kind = .map; + ev.a = .{ word(body, 8), word(body, 16), half(body, 56), word(body, 24), 0, 0 }; + const path = std.mem.sliceTo(body[64 .. body.len - trailer], 0); + try s.sink.emit(ev, if (std.mem.eql(u8, path, "//anon")) "" else path); + }, + .COMM => { + var text: [64]u8 = undefined; + const comm = std.mem.sliceTo(body[8 .. body.len - trailer], 0); + ev.kind = .mark; + try s.sink.emit(ev, try std.fmt.bufPrint(&text, "{s} {s}", .{ if (misc & 1 << 13 != 0) "exec" else "comm", comm })); + }, + .FORK => { + ev.kind = .thread_start; + ev.tid = half(body, 12); + ev.a[0] = half(body, 8); + ev.a[1] = half(body, 0); + try s.sink.emit(ev, ""); + }, + .EXIT => { + ev.kind = .thread_exit; + try s.sink.emit(ev, ""); + }, + .SWITCH => { + const out = misc & 1 << 13 != 0; + ev.kind = .sched; + ev.aux = @intFromBool(misc & 1 << 14 != 0); + ev.a = .{ if (out) ev.tid else 0, if (out) 0 else ev.tid, half(tail, 16), 0, 0, 0 }; + try s.sink.emit(ev, ""); + }, + .LOST => s.lost += word(body, 8), + .LOST_SAMPLES => s.lost += word(body, 0), + else => {}, + } + } + + fn sample(s: *Session, body: []const u8) !void { + const source = s.sources.get(word(body, 0)) orelse return; + const pc = word(body, 8); + const addr = word(body, 32); + var ev: Event = .{ .ts = word(body, 24), .tid = half(body, 20), .kind = undefined }; + const rest = body[sample_head..]; + switch (source) { + .sideband => {}, + .profile => { + // The counter overflowed on the way into the kernel: not a place in the program. + if (pc >> 63 != 0) return; + ev.kind = .sample; + ev.a = .{ pc, stringId("cycles"), word(body, 48), addr, 0, 0 }; + var chain: []const u8 = ""; + if (s.options.callchain) { + // The first entry says "user space from here", not a pc. + const frames: usize = @intCast(word(rest, 0)); + if (frames > 1) chain = rest[16..][0 .. (frames - 1) * 8]; + // The recorder takes a payload the size of `Regs` for registers. + if (chain.len == @sizeOf(Regs)) chain.len -= 8; + } + try s.sink.emit(ev, chain); + }, + .fault => |aux| { + const regs = userRegs(rest); + ev.kind = .fault; + ev.aux = aux; + ev.a = .{ addr, pc, 0, 0, 0, 0 }; + try s.sink.emit(ev, if (regs) |*r| std.mem.asBytes(r) else ""); + }, + .watch => |w| switch (w.mode) { + .write => { + // The value now: the target has run on since it wrote. + var bytes: [8]u8 = undefined; + const n = (ptrace.Process{ .pid = s.pid }).read(addr, bytes[0..w.len]); + ev.kind = .mem_write; + ev.a = .{ addr, w.len, pc, 0, 0, 0 }; + try s.sink.emit(ev, bytes[0..n]); + }, + .access => { + ev.kind = .mem_read; + ev.a = .{ addr, w.len, pc, 0, 0, 0 }; + try s.sink.emit(ev, ""); + }, + .exec => { + const regs = userRegs(rest) orelse Regs{}; + ev.kind = .probe; + ev.a = .{ w.name, regs.rdi, regs.rsi, regs.rdx, regs.rcx, regs.r8 }; + try s.sink.emit(ev, std.mem.asBytes(®s)); + }, + }, + .syscall_enter, .syscall_exit => { + // u32 size, then the trace event: 8 common bytes, the number, the arguments. + const raw = rest[4..][0..half(rest, 0)]; + ev.kind = if (source == .syscall_enter) .syscall_enter else .syscall_exit; + ev.aux = @truncate(half(raw, 8)); + for (&ev.a, 0..) |*a, i| { + if (16 + 8 * i + 8 <= raw.len) a.* = word(raw, 16 + 8 * i); + } + try s.sink.emit(ev, if (source == .syscall_enter) userData(raw) else ""); + }, + } + } +}; + +fn word(b: []const u8, at: usize) u64 { + return std.mem.readInt(u64, b[at..][0..8], .little); +} + +fn half(b: []const u8, at: usize) u32 { + return std.mem.readInt(u32, b[at..][0..4], .little); +} + +/// When a record happened: at a fixed place in a sample, at the end of the rest. +fn time(rec: []const u8) u64 { + return if (half(rec, 0) == @backingInt(PERF.RECORD.SAMPLE)) word(rec, 8 + 24) else word(rec, rec.len - trailer + 8); +} + +/// The registers of a sample taken with `perf_regs_mask`: an ABI word (0 +/// when the task had no user registers), then the values. +fn userRegs(b: []const u8) ?Regs { + if (word(b, 0) == 0) return null; + var regs: Regs = .{ .orig_rax = std.math.maxInt(u64) }; + inline for (perf_regs, 1..) |reg, i| @field(regs, reg) = word(b, 8 * i); + return regs; +} + +/// The first piece of user memory the kernel copied into a syscall's entry +/// record (a path, the bytes written; Linux 6.19 and later), else nothing. +/// After the arguments come up to three u32 (offset | length << 16) and the +/// data they point at; how many arguments there are is not in the record, +/// so each count is tried until one accounts for the record's exact size. +fn userData(raw: []const u8) []const u8 { + for (0..7) |args| for (1..4) |pieces| { + const metas = 16 + 8 * args; + var at = metas + 4 * pieces; + if (at > raw.len) continue; + const first = half(raw, metas); + for (0..pieces) |i| { + const meta = half(raw, metas + 4 * i); + if (meta & 0xffff != at) break; + at += meta >> 16; + } else if (std.mem.alignForward(usize, at + 4, 8) - 4 == raw.len) { + const data = raw[first & 0xffff ..][0 .. first >> 16]; + return if (std.mem.endsWith(u8, data, "\x00")) data[0 .. data.len - 1] else data; + } + }; + return ""; +} + +/// Lets a target held by `ptrace.Process.spawn` run on its own: once the +/// events are attached, ptrace has done its part. +pub fn release(p: ptrace.Process) void { + _ = linux.ptrace(linux.PTRACE.DETACH, p.pid, 0, 0, 0); +} + +pub const Symbol = struct { addr: u64, size: u64 }; + +/// Where `pid` has the symbol `name` now, looking through every ELF file +/// it has mapped, in address order. Shared libraries are there once the +/// dynamic loader has run, not at the first instruction. +pub fn lookup(gpa: std.mem.Allocator, pid: linux.pid_t, name: []const u8) !?Symbol { + const text = try readMaps(gpa, pid); + defer gpa.free(text); + var lines = std.mem.tokenizeScalar(u8, text, '\n'); + while (lines.next()) |line| { + const m = parseMap(line) orelse continue; + if (m.offset != 0 or m.path.len == 0 or m.path[0] != '/') continue; + const path = try gpa.dupeSentinel(u8, m.path, 0); + defer gpa.free(path); + const elf = usdt.Elf.open(path) catch continue; + defer elf.close(); + const sym = elf.symbol(name) orelse continue; + return .{ .addr = sym.value +% elf.bias(m.start), .size = sym.size }; + } + return null; +} + +/// /proc/PID/maps, whole. Free with `gpa`. +pub fn readMaps(gpa: std.mem.Allocator, pid: linux.pid_t) ![]u8 { + var path_buf: [64]u8 = undefined; + const path = try std.fmt.bufPrintSentinel(&path_buf, "/proc/{d}/maps", .{pid}, 0); + const fd = linux.open(path, .{}, 0); + if (linux.errno(fd) != .SUCCESS) return error.NoSuchProcess; + defer _ = linux.close(@intCast(fd)); + var text: std.ArrayList(u8) = .empty; + errdefer text.deinit(gpa); + while (true) { + try text.ensureUnusedCapacity(gpa, 4096); + const n = linux.read(@intCast(fd), text.unusedCapacitySlice().ptr, text.unusedCapacitySlice().len); + if (linux.errno(n) != .SUCCESS or n == 0) break; + text.items.len += n; + } + return text.toOwnedSlice(gpa); +} + +pub const Map = struct { start: u64, end: u64, prot: u64, offset: u64, path: []const u8 }; + +/// A line of /proc/PID/maps. +pub fn parseMap(line: []const u8) ?Map { + var fields = std.mem.tokenizeScalar(u8, line, ' '); + const range = fields.next() orelse return null; + const perms = fields.next() orelse return null; + const offset = fields.next() orelse return null; + _ = fields.next(); + _ = fields.next(); + const dash = std.mem.indexOfScalar(u8, range, '-') orelse return null; + if (perms.len < 3) return null; + return .{ + .start = std.fmt.parseInt(u64, range[0..dash], 16) catch return null, + .end = std.fmt.parseInt(u64, range[dash + 1 ..], 16) catch return null, + .prot = @as(u64, @intFromBool(perms[0] == 'r')) | @as(u64, @intFromBool(perms[1] == 'w')) << 1 | @as(u64, @intFromBool(perms[2] == 'x')) << 2, + .offset = std.fmt.parseInt(u64, offset, 16) catch return null, + .path = fields.rest(), + }; +} + +fn perfOpen(attr: *linux.perf_event_attr, pid: linux.pid_t, cpu: i32) !linux.fd_t { + // Paranoid 2 refuses anything that could see the kernel. + attr.flags.exclude_kernel = true; + attr.flags.exclude_hv = true; + const rc = linux.perf_event_open(attr, pid, cpu, -1, PERF.FLAG.FD_CLOEXEC); + return switch (linux.errno(rc)) { + .SUCCESS => @intCast(rc), + .ACCES, .PERM => error.PerfAccess, + .NOENT, .INVAL, .NODEV, .OPNOTSUPP => error.PerfNoSuchEvent, + .NOSPC, .BUSY => error.NoFreeBreakpoint, + .SRCH => error.NoSuchProcess, + .MFILE, .NFILE => error.ProcessFdQuotaExceeded, + else => error.PerfEventOpen, + }; +} + +test "a maps line; the user data of a syscall record" { + const m = parseMap("7f80b4224000-7f80b439f000 r-xp 00024000 08:02 1234 /usr/lib/libc.so.6").?; + try std.testing.expectEqual(@as(u64, 0x7f80b4224000), m.start); + try std.testing.expectEqual(@as(u64, 0x17b000), m.end - m.start); + try std.testing.expectEqual(@as(u64, 5), m.prot); + try std.testing.expectEqual(@as(u64, 0x24000), m.offset); + try std.testing.expectEqualStrings("/usr/lib/libc.so.6", m.path); + try std.testing.expectEqualStrings("", parseMap("7ffd1000-7ffd2000 rw-p 00000000 00:00 0").?.path); + + // openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC, 0) as Linux 7.2 records it. + var raw: [76]u8 = @splat(0); + std.mem.writeInt(u32, raw[8..12], 257, .little); + std.mem.writeInt(u32, raw[48..52], 52 | 17 << 16, .little); + @memcpy(raw[52..68], "/etc/ld.so.cache"); + try std.testing.expectEqualStrings("/etc/ld.so.cache", userData(&raw)); + // An argument that looks like a pointer into the record is not one: the sizes disagree. + std.mem.writeInt(u32, raw[16..20], 20 | 4 << 16, .little); + try std.testing.expectEqualStrings("/etc/ld.so.cache", userData(&raw)); + // An older kernel: arguments only. + try std.testing.expectEqualStrings("", userData(raw[0..52])); +} diff --git a/src/trace/root.zig b/src/trace/root.zig index fd91d9a..80745cb 100644 --- a/src/trace/root.zig +++ b/src/trace/root.zig @@ -16,6 +16,10 @@ pub const syscalls = @import("syscalls.zig"); pub const client = @import("client.zig"); /// Inside the tracer: a target running with a ring. pub const host = @import("host.zig"); +/// The kernel's view, unprivileged: counters, samples, faults, maps, syscalls, watchpoints. +pub const perf = @import("perf.zig"); +/// Static probes (the ELF notes of <sys/sdt.h>) and their arguments. +pub const usdt = @import("usdt.zig"); test { _ = event; @@ -27,4 +31,6 @@ test { _ = syscalls; _ = client; _ = host; + _ = perf; + _ = usdt; } diff --git a/src/trace/usdt.zig b/src/trace/usdt.zig new file mode 100644 index 0000000..ba03079 --- /dev/null +++ b/src/trace/usdt.zig @@ -0,0 +1,284 @@ +//! USDT, the static probes of DTrace and SystemTap: a program built with +//! <sys/sdt.h> has a nop where each probe is and an ELF note saying where +//! that is and where the probe's arguments live then. This reads the notes +//! and fetches the arguments from a stopped thread's registers and memory. +//! Getting stopped there is the caller's business: a breakpoint under +//! ptrace, a uprobe with BPF. Also the little of ELF both need: sections, +//! symbols, where a file's addresses land in a process. x86-64. +const std = @import("std"); +const linux = std.os.linux; +const Regs = @import("event.zig").Regs; +const E = std.elf.Elf64; + +/// A 64-bit little-endian ELF file in memory. +pub const Elf = struct { + bytes: []align(4096) const u8, + + /// Maps the file at `path`. + pub fn open(path: [*:0]const u8) !Elf { + const fd = linux.open(path, .{}, 0); + if (linux.errno(fd) != .SUCCESS) return error.FileNotFound; + defer _ = linux.close(@intCast(fd)); + const size = linux.lseek(@intCast(fd), 0, linux.SEEK.END); + if (linux.errno(size) != .SUCCESS or size < @sizeOf(E.Ehdr)) return error.NotElf; + const base = linux.mmap(null, size, .{ .READ = true }, .{ .TYPE = .PRIVATE }, @intCast(fd), 0); + if (linux.errno(base) != .SUCCESS) return error.NotElf; + const e: Elf = .{ .bytes = @as([*]align(4096) const u8, @ptrFromInt(base))[0..size] }; + const ident = e.header().ident; + if (!std.mem.eql(u8, &ident.magic, std.elf.MAGIC) or ident.class != .@"64" or ident.data != .@"2LSB") { + e.close(); + return error.NotElf; + } + return e; + } + + pub fn close(e: Elf) void { + _ = linux.munmap(e.bytes.ptr, e.bytes.len); + } + + fn header(e: Elf) *const E.Ehdr { + return @ptrCast(e.bytes.ptr); + } + + fn table(e: Elf, comptime T: type, at: u64, count: u64) []align(1) const T { + if (at > e.bytes.len or count > (e.bytes.len - at) / @sizeOf(T)) return &.{}; + return @as([*]align(1) const T, @ptrCast(e.bytes.ptr + at))[0..count]; + } + + fn contents(e: Elf, s: E.Shdr) []const u8 { + return if (s.type == .NOBITS) "" else e.table(u8, s.offset, s.size); + } + + pub fn section(e: Elf, name: []const u8) ?E.Shdr { + const h = e.header(); + const all = e.table(E.Shdr, h.shoff, h.shnum); + if (h.shstrndx >= all.len) return null; + const names = e.contents(all[h.shstrndx]); + for (all) |s| { + if (s.name < names.len and std.mem.eql(u8, std.mem.sliceTo(names[s.name..], 0), name)) return s; + } + return null; + } + + /// A symbol by name, from the full table when the file still has one, + /// else from the dynamic one. + pub fn symbol(e: Elf, name: []const u8) ?E.Sym { + for ([_][]const u8{ ".symtab", ".dynsym" }) |table_name| { + const s = e.section(table_name) orelse continue; + const all = e.table(E.Shdr, e.header().shoff, e.header().shnum); + if (s.link >= all.len) continue; + const names = e.contents(all[s.link]); + for (e.table(E.Sym, s.offset, s.size / @sizeOf(E.Sym))) |sym| { + if (sym.shndx == 0 or sym.name >= names.len) continue; + if (std.mem.eql(u8, std.mem.sliceTo(names[sym.name..], 0), name)) return sym; + } + } + return null; + } + + /// The first loaded segment, or the one that has `containing`. + fn load(e: Elf, containing: ?u64) ?E.Phdr { + for (e.table(E.Phdr, e.header().phoff, e.header().phnum)) |p| { + if (p.type != .LOAD) continue; + if (containing) |addr| if (addr < p.vaddr or addr >= p.vaddr + p.memsz) continue; + return p; + } + return null; + } + + /// What to add to the file's addresses in a process that has the start + /// of the file mapped at `start`: 0 for a fixed-position executable. + pub fn bias(e: Elf, start: u64) u64 { + const p = e.load(null) orelse return 0; + return start -% (p.vaddr - p.offset); + } + + /// Where in the file the byte at address `addr` (as the file has it) is: + /// what a uprobe is placed by. + pub fn offset(e: Elf, addr: u64) ?u64 { + const p = e.load(addr) orelse return null; + return addr - p.vaddr + p.offset; + } +}; + +pub const Probe = struct { + provider: []const u8, + name: []const u8, + /// Where the nop is. + pc: u64, + /// The address of the u16 that a tracer increments to enable the probe, + /// for a program that tests it before computing arguments; 0 without. + semaphore: u64, + /// The arguments as the note has them: "-4@%edi 8@-24(%rbp)". + args: []const u8, +}; + +/// The probes of `elf`, their addresses moved by `bias` (see `Elf.bias`; 0 +/// gives them as the file and `readelf -n` have them). The strings point +/// into the file. Free the slice with `gpa`. +pub fn probes(gpa: std.mem.Allocator, elf: Elf, bias: u64) ![]Probe { + var list: std.ArrayList(Probe) = .empty; + errdefer list.deinit(gpa); + const notes = elf.contents(elf.section(".note.stapsdt") orelse return &.{}); + // A note records where .stapsdt.base was when it was written: a + // prelinker moves the section and not the notes. + const base_now = if (elf.section(".stapsdt.base")) |s| s.addr else 0; + var at: usize = 0; + while (at + 12 <= notes.len) { + const name_len = std.mem.readInt(u32, notes[at..][0..4], .little); + const desc_len = std.mem.readInt(u32, notes[at + 4 ..][0..4], .little); + const kind = std.mem.readInt(u32, notes[at + 8 ..][0..4], .little); + const name_at = at + 12; + const desc_at = name_at + std.mem.alignForward(usize, name_len, 4); + at = desc_at + std.mem.alignForward(usize, desc_len, 4); + if (at > notes.len) break; + const desc = notes[desc_at..][0..desc_len]; + if (kind != 3 or desc_len < 24 or !std.mem.eql(u8, notes[name_at..][0..name_len], "stapsdt\x00")) continue; + const moved = (if (base_now != 0) base_now -% std.mem.readInt(u64, desc[8..16], .little) else 0) +% bias; + const semaphore = std.mem.readInt(u64, desc[16..24], .little); + var strings = std.mem.splitScalar(u8, desc[24..], 0); + try list.append(gpa, .{ + .pc = std.mem.readInt(u64, desc[0..8], .little) +% moved, + .semaphore = if (semaphore == 0) 0 else semaphore +% moved, + .provider = strings.next() orelse "", + .name = strings.next() orelse "", + .args = strings.next() orelse "", + }); + } + return list.toOwnedSlice(gpa); +} + +/// One argument of a probe: its size and where it is at the probe. +pub const Arg = struct { + /// 1, 2, 4 or 8 bytes. + size: u8, + signed: bool, + at: union(enum) { + constant: i64, + register: Register, + /// disp(base, index, scale) + memory: struct { base: Register, index: ?Register = null, scale: u8 = 1, disp: i64 = 0 }, + }, + + const Register = struct { + /// Which u64 of `Regs`. + slot: u8, + bytes: u8, + + fn value(r: Register, regs: *const Regs) u64 { + const all: *const [@sizeOf(Regs) / 8]u64 = @ptrCast(regs); + return if (r.bytes == 8) all[r.slot] else all[r.slot] & (@as(u64, 1) << @intCast(r.bytes * 8)) - 1; + } + }; + + /// "-4@%edi", "8@-24(%rbp)", "4@$7", "8@(%rdx,%rax,8)". + /// shortcut: no symbol(%rip) operands and no %ah-style registers, like libbpf; add them when a real probe uses one. + pub fn parse(text: []const u8) !Arg { + const split = std.mem.indexOfScalar(u8, text, '@') orelse return error.BadUsdtArg; + const size = std.fmt.parseInt(i8, text[0..split], 10) catch return error.BadUsdtArg; + var arg: Arg = .{ .size = @abs(size), .signed = size < 0, .at = undefined }; + if (std.mem.indexOfScalar(u8, &.{ 1, 2, 4, 8 }, arg.size) == null) return error.BadUsdtArg; + const op = text[split + 1 ..]; + if (op.len == 0) return error.BadUsdtArg; + if (op[0] == '$') { + arg.at = .{ .constant = std.fmt.parseInt(i64, op[1..], 0) catch return error.BadUsdtArg }; + } else if (op[0] == '%') { + arg.at = .{ .register = try register(op) }; + } else { + const open = std.mem.indexOfScalar(u8, op, '(') orelse return error.BadUsdtArg; + if (op[op.len - 1] != ')') return error.BadUsdtArg; + var parts = std.mem.splitScalar(u8, op[open + 1 .. op.len - 1], ','); + arg.at = .{ .memory = .{ + .disp = if (open == 0) 0 else std.fmt.parseInt(i64, op[0..open], 0) catch return error.BadUsdtArg, + .base = try register(parts.next().?), + .index = if (parts.next()) |r| try register(r) else null, + .scale = if (parts.next()) |s| std.fmt.parseInt(u8, s, 10) catch return error.BadUsdtArg else 1, + } }; + } + return arg; + } + + /// The argument's value, sign- or zero-extended, for a thread stopped + /// at the probe with `regs`. `memory` is anything with + /// `read(addr: u64, buf: []u8) usize` (a `ptrace.Process`). Null when + /// the memory cannot be read. + pub fn fetch(arg: Arg, regs: *const Regs, memory: anytype) ?u64 { + var bytes: [8]u8 = undefined; + switch (arg.at) { + .constant => |c| std.mem.writeInt(i64, &bytes, c, .little), + .register => |r| std.mem.writeInt(u64, &bytes, r.value(regs), .little), + .memory => |m| { + const index = if (m.index) |r| r.value(regs) *% m.scale else 0; + const addr = m.base.value(regs) +% index +% @as(u64, @bitCast(m.disp)); + if (memory.read(addr, bytes[0..arg.size]) != arg.size) return null; + }, + } + return switch (arg.size) { + inline 1, 2, 4, 8 => |n| { + const U = @Int(.unsigned, n * 8); + const v = std.mem.readInt(U, bytes[0..n], .little); + return if (arg.signed) @as(u64, @bitCast(@as(i64, @as(@Int(.signed, n * 8), @bitCast(v))))) else v; + }, + else => unreachable, + }; + } +}; + +/// "%rax", "%eax", "%ax", "%al", "%r8d"... +fn register(text: []const u8) !Arg.Register { + if (text.len < 3 or text[0] != '%') return error.BadUsdtArg; + const name = text[1..]; + // The same register at 8, 4, 2 and 1 bytes. + const legacy = [_][4][]const u8{ + .{ "rax", "eax", "ax", "al" }, .{ "rbx", "ebx", "bx", "bl" }, + .{ "rcx", "ecx", "cx", "cl" }, .{ "rdx", "edx", "dx", "dl" }, + .{ "rsi", "esi", "si", "sil" }, .{ "rdi", "edi", "di", "dil" }, + .{ "rbp", "ebp", "bp", "bpl" }, .{ "rsp", "esp", "sp", "spl" }, + }; + inline for (legacy) |names| inline for (names, [_]u8{ 8, 4, 2, 1 }) |n, bytes| { + if (std.mem.eql(u8, name, n)) return .{ .slot = @offsetOf(Regs, names[0]) / 8, .bytes = bytes }; + }; + inline for (8..16) |number| { + const full = std.fmt.comptimePrint("r{d}", .{number}); + if (std.mem.startsWith(u8, name, full)) { + const bytes: u8 = if (name.len == full.len) 8 else if (name.len != full.len + 1) 0 else switch (name[full.len]) { + 'd' => 4, + 'w' => 2, + 'b' => 1, + else => 0, + }; + if (bytes != 0) return .{ .slot = @offsetOf(Regs, full) / 8, .bytes = bytes }; + } + } + return error.BadUsdtArg; +} + +test "argument specs are parsed and fetched" { + const regs: Regs = .{ .rdi = 0xffff_ffff_ffff_fffe, .rbp = 0x1000, .rdx = 0x2000, .rax = 3, .r10 = 0x1_0000_00ff }; + // A memory of its own: every address holds its own low byte, counting up. + const memory = struct { + fn read(_: @This(), addr: u64, buf: []u8) usize { + if (addr >= 0x8000) return 0; + for (buf, 0..) |*b, i| b.* = @truncate(addr + i); + return buf.len; + } + }{}; + const cases = [_]struct { []const u8, ?u64 }{ + .{ "-4@%edi", @bitCast(@as(i64, -2)) }, + .{ "4@%edi", 0xffff_fffe }, + .{ "8@%rdi", 0xffff_ffff_ffff_fffe }, + .{ "1@%r10b", 0xff }, + .{ "4@%r10d", 0xff }, + .{ "-4@$-7", @bitCast(@as(i64, -7)) }, + .{ "8@$0x10", 16 }, + .{ "2@-24(%rbp)", 0xe9e8 }, + .{ "1@(%rdx)", 0 }, + .{ "4@8(%rdx,%rax,8)", 0x23222120 }, + .{ "-1@-1(%rbp)", @bitCast(@as(i64, -1)) }, + .{ "8@0x7000(%rbp)", null }, + }; + for (cases) |case| try std.testing.expectEqual(case[1], (try Arg.parse(case[0])).fetch(®s, memory)); + for ([_][]const u8{ "4@counter(%rip)", "3@%eax", "4@%ah", "4", "4@", "4@8(%rdx", "4@%r8x" }) |bad| { + try std.testing.expectError(error.BadUsdtArg, Arg.parse(bad)); + } +} |
