summaryrefslogtreecommitdiff
path: root/src/trace/perf.zig
blob: 7c9ef433bb2067e7f51062803f0cabafc61ae345 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
//! The kernel's view of a task without privilege: `perf_event_open` as far
//! as `perf_event_paranoid=2` lets a user go on their own processes (user
//! space only). Counters, a sampling profile, every page fault, the memory
//! maps, threads, context switches, every syscall, and the debug registers
//! as watchpoints. The target runs at full speed: the kernel writes records
//! into ring buffers, stamped with CLOCK_MONOTONIC like every other
//! backend's events, and `Session` decodes them into a `Sink`.
//!
//! What becomes what:
//!   counter reading     sample   aux=`reading`, a1=name string id, a2=total so far
//!   profile sample      sample   a0=pc a1=name string id a2=period a3=data address. Payload: the callchain, u64 pcs
//!   page fault          fault    a0=address a1=pc, aux=`fault_major` or 0. Payload: `Regs`
//!   mapping             map      as the contract says; the path is `[heap]`, `[stack]`... for those
//!   clone, exit         thread_start (tid=parent, a0=new tid, a1=its pid), thread_exit
//!   comm                mark     "comm NAME", or "exec NAME"
//!   context switch      sched    out: a0=tid, in: a1=tid; a2=cpu, aux=1 when preempted
//!   syscall             syscall_enter, syscall_exit. Payload of an enter: the path or buffer the kernel captured
//!   watchpoint          mem_write (payload: the bytes there when the record is decoded), mem_read; a2=pc after the access
//!   execute breakpoint  probe    a0=name string id a1..a5=rdi rsi rdx rcx r8. Payload: `Regs`
const std = @import("std");
const linux = std.os.linux;
const event = @import("event.zig");
const ptrace = @import("ptrace.zig");
const usdt = @import("usdt.zig");
const Event = event.Event;
const Regs = event.Regs;
const PERF = linux.PERF;

/// `aux` of a `sample` that is a counter's total, not a profile sample.
pub const reading: u16 = 1;
/// `aux` of a `fault` that needed I/O.
pub const fault_major: u16 = 1;

/// The id a `string` event gives `text`: its hash, so that backends agree
/// without a registry.
pub fn stringId(text: []const u8) u64 {
    return std.hash.Wyhash.hash(0, text);
}

/// What can be counted. Context switches and migrations are missing: their
/// software events fire in kernel mode, which paranoid 2 excludes, so they
/// always read 0 (`Options.switches` gives the switches as events instead).
pub const Counter = enum {
    instructions,
    cycles,
    branches,
    @"branch-misses",
    @"cache-references",
    @"cache-misses",
    /// Faults taken in user mode only.
    @"page-faults",
    /// Nanoseconds on a CPU.
    @"task-clock",

    fn attr(c: Counter) linux.perf_event_attr {
        const HW = PERF.COUNT.HW;
        const SW = PERF.COUNT.SW;
        return switch (c) {
            .instructions => .{ .type = .HARDWARE, .config = @backingInt(HW.INSTRUCTIONS) },
            .cycles => .{ .type = .HARDWARE, .config = @backingInt(HW.CPU_CYCLES) },
            .branches => .{ .type = .HARDWARE, .config = @backingInt(HW.BRANCH_INSTRUCTIONS) },
            .@"branch-misses" => .{ .type = .HARDWARE, .config = @backingInt(HW.BRANCH_MISSES) },
            .@"cache-references" => .{ .type = .HARDWARE, .config = @backingInt(HW.CACHE_REFERENCES) },
            .@"cache-misses" => .{ .type = .HARDWARE, .config = @backingInt(HW.CACHE_MISSES) },
            .@"page-faults" => .{ .type = .SOFTWARE, .config = @backingInt(SW.PAGE_FAULTS) },
            .@"task-clock" => .{ .type = .SOFTWARE, .config = @backingInt(SW.TASK_CLOCK) },
        };
    }
};

/// A debug register on the target: the CPU reports every access, the target
/// never stops. There are four; a data watchpoint covers 1, 2, 4 or 8 bytes
/// at an address aligned to that; the record is written after the access, so
/// `pc` is the instruction after it and the value is not part of it.
pub const Watch = struct {
    pub const Mode = enum(u32) { write = 2, access = 3, exec = 4 };

    addr: u64,
    len: u8 = 8,
    mode: Mode = .write,
    /// What an `exec` hit is called.
    name: []const u8 = "",
};

pub const Options = struct {
    counters: []const Counter = &.{},
    /// Profile samples a second (of cycles); 0 for none.
    sample_hz: u32 = 0,
    /// Frame-pointer callchains on profile samples.
    callchain: bool = false,
    faults: bool = false,
    switches: bool = false,
    syscalls: bool = false,
    /// Also the threads and children the target creates from now on. One
    /// ring per CPU then, merged by time, as inherited events need.
    follow: bool = false,
    /// Per ring; a power of two, at least a page.
    ring_bytes: usize = 128 * 4096,
};

const base_sample = PERF.SAMPLE.IDENTIFIER | PERF.SAMPLE.IP | PERF.SAMPLE.TID | PERF.SAMPLE.TIME | PERF.SAMPLE.ADDR | PERF.SAMPLE.CPU | PERF.SAMPLE.PERIOD;
/// The fixed head of a sample with `base_sample`: id, ip, pid, tid, time, addr, cpu, period.
const sample_head = 56;
/// What follows every other record: pid, tid, time, cpu, id.
const trailer = 32;

/// The registers perf hands out for a 64-bit task, in its bit order (the
/// segment registers 12..15 are refused).
const perf_regs = [_][]const u8{ "rax", "rbx", "rcx", "rdx", "rsi", "rdi", "rbp", "rsp", "rip", "eflags", "cs", "ss", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" };
const perf_regs_mask: u64 = 0xff0fff;

/// What an event writing into a ring means, found by the id in its records.
const Source = union(enum) {
    sideband,
    profile,
    fault: u16,
    watch: struct { len: u8, mode: Watch.Mode, name: u64 },
    syscall_enter,
    syscall_exit,
};

const Ring = struct {
    cpu: i32,
    /// The event that owns the mapping; the others output into it.
    fd: linux.fd_t = -1,
    meta: *linux.perf_event_mmap_page = undefined,
    data: []u8 = &.{},
    head: u64 = 0,
    tail: u64 = 0,

    /// The record at the read position, copied to `scratch` when it wraps.
    fn peek(r: *const Ring, scratch: []u8) ?[]const u8 {
        if (r.tail == r.head) return null;
        const at: usize = @intCast(r.tail & (r.data.len - 1));
        // Records are multiples of 8 bytes: the header itself never wraps.
        const size = std.mem.readInt(u16, r.data[at + 6 ..][0..2], .little);
        if (at + size <= r.data.len) return r.data[at..][0..size];
        const first = r.data.len - at;
        @memcpy(scratch[0..first], r.data[at..]);
        @memcpy(scratch[first..size], r.data[0 .. size - first]);
        return scratch[0..size];
    }
};

pub const Session = struct {
    gpa: std.mem.Allocator,
    pid: linux.pid_t,
    sink: event.Sink,
    options: Options,
    rings: []Ring = &.{},
    pollfds: []linux.pollfd = &.{},
    fds: std.ArrayList(linux.fd_t) = .empty,
    sources: std.AutoHashMapUnmanaged(u64, Source) = .empty,
    counter_fds: []linux.fd_t = &.{},
    /// The totals `readCounters` last saw, in `options.counters` order.
    counts: []u64 = &.{},
    /// Records the kernel dropped because a ring was full.
    lost: u64 = 0,
    scratch: []u8 = &.{},

    /// Attaches to `pid` (a thread id), which should be stopped: events
    /// count from here. Emits the names used and the mappings it already has.
    pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, options: Options, sink: event.Sink) !Session {
        std.debug.assert(std.math.isPowerOfTwo(options.ring_bytes) and options.ring_bytes >= 4096);
        var s: Session = .{ .gpa = gpa, .pid = pid, .sink = sink, .options = options };
        errdefer s.close();
        s.scratch = try gpa.alloc(u8, 1 << 16);
        // shortcut: CPUs 0..count-1, right unless some are offline or outside our affinity mask.
        const cpus = if (options.follow) try std.Thread.getCpuCount() else 1;
        s.rings = try gpa.alloc(Ring, cpus);
        for (s.rings, 0..) |*r, i| r.* = .{ .cpu = if (options.follow) @intCast(i) else -1 };

        // An event that counts nothing carries the records about the task
        // itself, and comes first: it owns the rings.
        var side: linux.perf_event_attr = .{ .type = .SOFTWARE, .config = @backingInt(PERF.COUNT.SW.DUMMY) };
        side.flags.mmap = true;
        side.flags.mmap2 = true;
        side.flags.mmap_data = true;
        side.flags.comm = true;
        side.flags.comm_exec = true;
        side.flags.task = true;
        side.flags.context_switch = options.switches;
        try s.add(side, .sideband);
        s.pollfds = try gpa.alloc(linux.pollfd, cpus);
        for (s.pollfds, s.rings) |*p, r| p.* = .{ .fd = r.fd, .events = linux.POLL.IN, .revents = 0 };

        s.counter_fds = try gpa.alloc(linux.fd_t, options.counters.len);
        @memset(s.counter_fds, -1);
        s.counts = try gpa.alloc(u64, options.counters.len);
        @memset(s.counts, 0);
        for (options.counters, s.counter_fds) |c, *fd| {
            var attr = c.attr();
            attr.read_format = 1 | 2; // TOTAL_TIME_ENABLED | TOTAL_TIME_RUNNING
            attr.flags.inherit = options.follow;
            fd.* = try perfOpen(&attr, pid, -1);
            try s.name(@tagName(c));
        }

        if (options.sample_hz != 0) {
            var attr = Counter.cycles.attr();
            attr.sample_period_or_freq = options.sample_hz;
            attr.flags.freq = true;
            attr.flags.exclude_callchain_kernel = true;
            if (options.callchain) attr.sample_type = PERF.SAMPLE.CALLCHAIN;
            try s.add(attr, .profile);
            try s.name("cycles");
        }
        if (options.faults) for ([_]PERF.COUNT.SW{ .PAGE_FAULTS_MIN, .PAGE_FAULTS_MAJ }, [_]u16{ 0, fault_major }) |config, aux| {
            try s.add(.{ .type = .SOFTWARE, .config = @backingInt(config), .sample_period_or_freq = 1, .sample_type = PERF.SAMPLE.REGS_USER }, .{ .fault = aux });
        };
        if (options.syscalls) try s.addSyscalls();
        try s.maps();
        return s;
    }

    pub fn close(s: *Session) void {
        for (s.rings) |r| if (r.fd >= 0) {
            _ = linux.munmap(@ptrCast(r.meta), 4096 + r.data.len);
        };
        for (s.fds.items) |fd| _ = linux.close(fd);
        for (s.counter_fds) |fd| if (fd >= 0) {
            _ = linux.close(fd);
        };
        s.fds.deinit(s.gpa);
        s.sources.deinit(s.gpa);
        s.gpa.free(s.rings);
        s.gpa.free(s.pollfds);
        s.gpa.free(s.counter_fds);
        s.gpa.free(s.counts);
        s.gpa.free(s.scratch);
    }

    /// Arms a debug register, on a session already running too.
    /// `error.NoFreeBreakpoint` when all four are taken.
    pub fn watch(s: *Session, w: Watch) !void {
        var attr: linux.perf_event_attr = .{ .type = .BREAKPOINT, .sample_period_or_freq = 1, .bp_type = @backingInt(w.mode), .config1 = w.addr, .config2 = if (w.mode == .exec) 8 else w.len };
        // Where it executes is known; what it was called with is not.
        if (w.mode == .exec) attr.sample_type = PERF.SAMPLE.REGS_USER;
        try s.add(attr, .{ .watch = .{ .len = w.len, .mode = w.mode, .name = stringId(w.name) } });
        if (w.mode == .exec) try s.name(w.name);
    }

    /// One event on every ring, its records told apart by `source`.
    fn add(s: *Session, template: linux.perf_event_attr, source: Source) !void {
        var attr = template;
        attr.sample_type |= base_sample;
        if (attr.sample_type & PERF.SAMPLE.REGS_USER != 0) attr.sample_regs_user = perf_regs_mask;
        attr.flags.sample_id_all = true;
        attr.flags.inherit = s.options.follow;
        attr.flags.use_clockid = true;
        attr.clockid = .MONOTONIC;
        // Wake the reader when a ring is a quarter full (it also looks on its
        // own), but at once for a watchpoint: the value is read on waking.
        attr.flags.watermark = source != .watch;
        attr.wakeup_events_or_watermark = if (source == .watch) 1 else @intCast(s.options.ring_bytes / 4);
        for (s.rings) |*r| {
            const fd = try perfOpen(&attr, s.pid, r.cpu);
            try s.fds.append(s.gpa, fd);
            if (r.fd < 0) {
                const base = linux.mmap(null, 4096 + s.options.ring_bytes, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0);
                if (linux.errno(base) != .SUCCESS) return error.PerfRingMap;
                r.fd = fd;
                r.meta = @ptrFromInt(base);
                r.data = @as([*]u8, @ptrFromInt(base + 4096))[0..s.options.ring_bytes];
            } else if (linux.errno(linux.ioctl(fd, PERF.EVENT_IOC.SET_OUTPUT, @intCast(r.fd))) != .SUCCESS) return error.PerfSetOutput;
            var id: u64 = 0;
            if (linux.errno(linux.ioctl(fd, PERF.EVENT_IOC.ID, @intFromPtr(&id))) != .SUCCESS) return error.PerfEventId;
            try s.sources.put(s.gpa, id, source);
        }
    }

    /// Every syscall, through the per-syscall tracepoints: the only
    /// tracepoints an unprivileged user may read, and only on a task of
    /// their own. Their ids are in tracefs, which we may not be able to
    /// read, so every id is tried: about 700 open, two a syscall.
    fn addSyscalls(s: *Session) !void {
        var limit: linux.rlimit = undefined;
        if (linux.errno(linux.getrlimit(.NOFILE, &limit)) == .SUCCESS and limit.cur < limit.max) {
            limit.cur = limit.max;
            _ = linux.setrlimit(.NOFILE, &limit);
        }
        // shortcut: ids below 4096; scan further if a kernel has more trace events than that.
        for (0..4096) |id| {
            var attr: linux.perf_event_attr = .{ .type = .TRACEPOINT, .config = id, .sample_period_or_freq = 1, .sample_type = PERF.SAMPLE.RAW };
            // Only a syscall's exit has a field `ret`: a filter on it is
            // accepted there and nowhere else. On an event of its own, since
            // a filter cannot be taken off again.
            const probe = perfOpen(&attr, s.pid, -1) catch |err| switch (err) {
                error.PerfAccess, error.PerfNoSuchEvent => continue,
                else => |e| return e,
            };
            const is_exit = linux.errno(linux.ioctl(probe, PERF.EVENT_IOC.SET_FILTER, @intFromPtr("ret==0"))) == .SUCCESS;
            _ = linux.close(probe);
            try s.add(attr, if (is_exit) .syscall_exit else .syscall_enter);
        }
    }

    fn name(s: *Session, text: []const u8) !void {
        try s.sink.emit(.{ .ts = event.now(), .tid = @intCast(s.pid), .kind = .string, .a = .{ stringId(text), 0, 0, 0, 0, 0 } }, text);
    }

    /// The mappings the target has now, from /proc: the kernel only reports
    /// the ones made after an event exists.
    fn maps(s: *Session) !void {
        const text = try readMaps(s.gpa, s.pid);
        defer s.gpa.free(text);
        const ts = event.now();
        var lines = std.mem.tokenizeScalar(u8, text, '\n');
        while (lines.next()) |line| {
            const m = parseMap(line) orelse continue;
            try s.sink.emit(.{ .ts = ts, .tid = @intCast(s.pid), .kind = .map, .a = .{ m.start, m.end - m.start, m.prot, m.offset, 0, 0 } }, m.path);
        }
    }

    /// Each counter's total so far, as `sample` events and in `counts`.
    pub fn readCounters(s: *Session) !void {
        const ts = event.now();
        for (s.options.counters, s.counter_fds, s.counts) |c, fd, *count| {
            // value, time enabled, time running
            var v: [3]u64 = undefined;
            if (linux.read(fd, @ptrCast(&v), 24) != 24) continue;
            // A counter that had to share its hardware counted part of the time: scale it.
            count.* = if (v[2] != 0 and v[2] < v[1]) @intCast(@as(u128, v[0]) * v[1] / v[2]) else v[0];
            try s.sink.emit(.{ .ts = ts, .tid = @intCast(s.pid), .kind = .sample, .aux = reading, .a = .{ 0, stringId(@tagName(c)), count.*, 0, 0, 0 } }, "");
        }
    }

    /// Waits for records, the target's end, or the timeout, then decodes
    /// what is there. False once every traced task has exited.
    pub fn poll(s: *Session, timeout_ms: i32) !bool {
        _ = linux.poll(s.pollfds.ptr, s.pollfds.len, timeout_ms);
        _ = try s.drain();
        for (s.pollfds) |p| if (p.revents & linux.POLL.HUP == 0) return true;
        return false;
    }

    /// Decodes the records in the rings into the sink, oldest first.
    /// Returns how many.
    pub fn drain(s: *Session) !usize {
        // A record stamped after this waits for the next call, so a ring
        // read late cannot get ahead of one read early.
        const limit = event.now();
        for (s.rings) |*r| r.head = @atomicLoad(u64, &r.meta.data_head, .acquire);
        var n: usize = 0;
        while (true) : (n += 1) {
            var oldest: ?*Ring = null;
            var oldest_ts = limit;
            for (s.rings) |*r| {
                const ts = time(r.peek(s.scratch) orelse continue);
                if (ts > oldest_ts) continue;
                oldest = r;
                oldest_ts = ts;
            }
            const r = oldest orelse break;
            const rec = r.peek(s.scratch).?;
            try s.decode(rec);
            r.tail += rec.len;
        }
        for (s.rings) |*r| @atomicStore(u64, &r.meta.data_tail, r.tail, .release);
        return n;
    }

    fn decode(s: *Session, rec: []const u8) !void {
        const body = rec[8..];
        const misc = std.mem.readInt(u16, rec[4..6], .little);
        const tail = rec[rec.len - trailer ..];
        var ev: Event = .{ .ts = word(tail, 8), .tid = half(tail, 4), .kind = undefined };
        switch (@as(PERF.RECORD, @fromBackingInt(@intCast(half(rec, 0))))) {
            .SAMPLE => return s.sample(body),
            .MMAP2 => {
                ev.kind = .map;
                ev.a = .{ word(body, 8), word(body, 16), half(body, 56), word(body, 24), 0, 0 };
                const path = std.mem.sliceTo(body[64 .. body.len - trailer], 0);
                try s.sink.emit(ev, if (std.mem.eql(u8, path, "//anon")) "" else path);
            },
            .COMM => {
                var text: [64]u8 = undefined;
                const comm = std.mem.sliceTo(body[8 .. body.len - trailer], 0);
                ev.kind = .mark;
                try s.sink.emit(ev, try std.fmt.bufPrint(&text, "{s} {s}", .{ if (misc & 1 << 13 != 0) "exec" else "comm", comm }));
            },
            .FORK => {
                ev.kind = .thread_start;
                ev.tid = half(body, 12);
                ev.a[0] = half(body, 8);
                ev.a[1] = half(body, 0);
                try s.sink.emit(ev, "");
            },
            .EXIT => {
                ev.kind = .thread_exit;
                try s.sink.emit(ev, "");
            },
            .SWITCH => {
                const out = misc & 1 << 13 != 0;
                ev.kind = .sched;
                ev.aux = @intFromBool(misc & 1 << 14 != 0);
                ev.a = .{ if (out) ev.tid else 0, if (out) 0 else ev.tid, half(tail, 16), 0, 0, 0 };
                try s.sink.emit(ev, "");
            },
            .LOST => s.lost += word(body, 8),
            .LOST_SAMPLES => s.lost += word(body, 0),
            else => {},
        }
    }

    fn sample(s: *Session, body: []const u8) !void {
        const source = s.sources.get(word(body, 0)) orelse return;
        const pc = word(body, 8);
        const addr = word(body, 32);
        var ev: Event = .{ .ts = word(body, 24), .tid = half(body, 20), .kind = undefined };
        const rest = body[sample_head..];
        switch (source) {
            .sideband => {},
            .profile => {
                // The counter overflowed on the way into the kernel: not a place in the program.
                if (pc >> 63 != 0) return;
                ev.kind = .sample;
                ev.a = .{ pc, stringId("cycles"), word(body, 48), addr, 0, 0 };
                var chain: []const u8 = "";
                if (s.options.callchain) {
                    // The first entry says "user space from here", not a pc.
                    const frames: usize = @intCast(word(rest, 0));
                    if (frames > 1) chain = rest[16..][0 .. (frames - 1) * 8];
                    // The recorder takes a payload the size of `Regs` for registers.
                    if (chain.len == @sizeOf(Regs)) chain.len -= 8;
                }
                try s.sink.emit(ev, chain);
            },
            .fault => |aux| {
                const regs = userRegs(rest);
                ev.kind = .fault;
                ev.aux = aux;
                ev.a = .{ addr, pc, 0, 0, 0, 0 };
                try s.sink.emit(ev, if (regs) |*r| std.mem.asBytes(r) else "");
            },
            .watch => |w| switch (w.mode) {
                .write => {
                    // The value now: the target has run on since it wrote.
                    var bytes: [8]u8 = undefined;
                    const n = (ptrace.Process{ .pid = s.pid }).read(addr, bytes[0..w.len]);
                    ev.kind = .mem_write;
                    ev.a = .{ addr, w.len, pc, 0, 0, 0 };
                    try s.sink.emit(ev, bytes[0..n]);
                },
                .access => {
                    ev.kind = .mem_read;
                    ev.a = .{ addr, w.len, pc, 0, 0, 0 };
                    try s.sink.emit(ev, "");
                },
                .exec => {
                    const regs = userRegs(rest) orelse Regs{};
                    ev.kind = .probe;
                    ev.a = .{ w.name, regs.rdi, regs.rsi, regs.rdx, regs.rcx, regs.r8 };
                    try s.sink.emit(ev, std.mem.asBytes(&regs));
                },
            },
            .syscall_enter, .syscall_exit => {
                // u32 size, then the trace event: 8 common bytes, the number, the arguments.
                const raw = rest[4..][0..half(rest, 0)];
                ev.kind = if (source == .syscall_enter) .syscall_enter else .syscall_exit;
                ev.aux = @truncate(half(raw, 8));
                for (&ev.a, 0..) |*a, i| {
                    if (16 + 8 * i + 8 <= raw.len) a.* = word(raw, 16 + 8 * i);
                }
                try s.sink.emit(ev, if (source == .syscall_enter) userData(raw) else "");
            },
        }
    }
};

fn word(b: []const u8, at: usize) u64 {
    return std.mem.readInt(u64, b[at..][0..8], .little);
}

fn half(b: []const u8, at: usize) u32 {
    return std.mem.readInt(u32, b[at..][0..4], .little);
}

/// When a record happened: at a fixed place in a sample, at the end of the rest.
fn time(rec: []const u8) u64 {
    return if (half(rec, 0) == @backingInt(PERF.RECORD.SAMPLE)) word(rec, 8 + 24) else word(rec, rec.len - trailer + 8);
}

/// The registers of a sample taken with `perf_regs_mask`: an ABI word (0
/// when the task had no user registers), then the values.
fn userRegs(b: []const u8) ?Regs {
    if (word(b, 0) == 0) return null;
    var regs: Regs = .{ .orig_rax = std.math.maxInt(u64) };
    inline for (perf_regs, 1..) |reg, i| @field(regs, reg) = word(b, 8 * i);
    return regs;
}

/// The first piece of user memory the kernel copied into a syscall's entry
/// record (a path, the bytes written; Linux 6.19 and later), else nothing.
/// After the arguments come up to three u32 (offset | length << 16) and the
/// data they point at; how many arguments there are is not in the record,
/// so each count is tried until one accounts for the record's exact size.
fn userData(raw: []const u8) []const u8 {
    for (0..7) |args| for (1..4) |pieces| {
        const metas = 16 + 8 * args;
        var at = metas + 4 * pieces;
        if (at > raw.len) continue;
        const first = half(raw, metas);
        for (0..pieces) |i| {
            const meta = half(raw, metas + 4 * i);
            if (meta & 0xffff != at) break;
            at += meta >> 16;
        } else if (std.mem.alignForward(usize, at + 4, 8) - 4 == raw.len) {
            const data = raw[first & 0xffff ..][0 .. first >> 16];
            return if (std.mem.endsWith(u8, data, "\x00")) data[0 .. data.len - 1] else data;
        }
    };
    return "";
}

/// Lets a target held by `ptrace.Process.spawn` run on its own: once the
/// events are attached, ptrace has done its part.
pub fn release(p: ptrace.Process) void {
    _ = linux.ptrace(linux.PTRACE.DETACH, p.pid, 0, 0, 0);
}

pub const Symbol = struct { addr: u64, size: u64 };

/// Where `pid` has the symbol `name` now, looking through every ELF file
/// it has mapped, in address order. Shared libraries are there once the
/// dynamic loader has run, not at the first instruction.
pub fn lookup(gpa: std.mem.Allocator, pid: linux.pid_t, name: []const u8) !?Symbol {
    const text = try readMaps(gpa, pid);
    defer gpa.free(text);
    var lines = std.mem.tokenizeScalar(u8, text, '\n');
    while (lines.next()) |line| {
        const m = parseMap(line) orelse continue;
        if (m.offset != 0 or m.path.len == 0 or m.path[0] != '/') continue;
        const path = try gpa.dupeSentinel(u8, m.path, 0);
        defer gpa.free(path);
        const elf = usdt.Elf.open(path) catch continue;
        defer elf.close();
        const sym = elf.symbol(name) orelse continue;
        return .{ .addr = sym.value +% elf.bias(m.start), .size = sym.size };
    }
    return null;
}

/// /proc/PID/maps, whole. Free with `gpa`.
pub fn readMaps(gpa: std.mem.Allocator, pid: linux.pid_t) ![]u8 {
    var path_buf: [64]u8 = undefined;
    const path = try std.fmt.bufPrintSentinel(&path_buf, "/proc/{d}/maps", .{pid}, 0);
    const fd = linux.open(path, .{}, 0);
    if (linux.errno(fd) != .SUCCESS) return error.NoSuchProcess;
    defer _ = linux.close(@intCast(fd));
    var text: std.ArrayList(u8) = .empty;
    errdefer text.deinit(gpa);
    while (true) {
        try text.ensureUnusedCapacity(gpa, 4096);
        const n = linux.read(@intCast(fd), text.unusedCapacitySlice().ptr, text.unusedCapacitySlice().len);
        if (linux.errno(n) != .SUCCESS or n == 0) break;
        text.items.len += n;
    }
    return text.toOwnedSlice(gpa);
}

pub const Map = struct { start: u64, end: u64, prot: u64, offset: u64, path: []const u8 };

/// A line of /proc/PID/maps.
pub fn parseMap(line: []const u8) ?Map {
    var fields = std.mem.tokenizeScalar(u8, line, ' ');
    const range = fields.next() orelse return null;
    const perms = fields.next() orelse return null;
    const offset = fields.next() orelse return null;
    _ = fields.next();
    _ = fields.next();
    const dash = std.mem.indexOfScalar(u8, range, '-') orelse return null;
    if (perms.len < 3) return null;
    return .{
        .start = std.fmt.parseInt(u64, range[0..dash], 16) catch return null,
        .end = std.fmt.parseInt(u64, range[dash + 1 ..], 16) catch return null,
        .prot = @as(u64, @intFromBool(perms[0] == 'r')) | @as(u64, @intFromBool(perms[1] == 'w')) << 1 | @as(u64, @intFromBool(perms[2] == 'x')) << 2,
        .offset = std.fmt.parseInt(u64, offset, 16) catch return null,
        .path = fields.rest(),
    };
}

fn perfOpen(attr: *linux.perf_event_attr, pid: linux.pid_t, cpu: i32) !linux.fd_t {
    // Paranoid 2 refuses anything that could see the kernel.
    attr.flags.exclude_kernel = true;
    attr.flags.exclude_hv = true;
    const rc = linux.perf_event_open(attr, pid, cpu, -1, PERF.FLAG.FD_CLOEXEC);
    return switch (linux.errno(rc)) {
        .SUCCESS => @intCast(rc),
        .ACCES, .PERM => error.PerfAccess,
        .NOENT, .INVAL, .NODEV, .OPNOTSUPP => error.PerfNoSuchEvent,
        .NOSPC, .BUSY => error.NoFreeBreakpoint,
        .SRCH => error.NoSuchProcess,
        .MFILE, .NFILE => error.ProcessFdQuotaExceeded,
        else => error.PerfEventOpen,
    };
}

test "a maps line; the user data of a syscall record" {
    const m = parseMap("7f80b4224000-7f80b439f000 r-xp 00024000 08:02 1234                    /usr/lib/libc.so.6").?;
    try std.testing.expectEqual(@as(u64, 0x7f80b4224000), m.start);
    try std.testing.expectEqual(@as(u64, 0x17b000), m.end - m.start);
    try std.testing.expectEqual(@as(u64, 5), m.prot);
    try std.testing.expectEqual(@as(u64, 0x24000), m.offset);
    try std.testing.expectEqualStrings("/usr/lib/libc.so.6", m.path);
    try std.testing.expectEqualStrings("", parseMap("7ffd1000-7ffd2000 rw-p 00000000 00:00 0").?.path);

    // openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC, 0) as Linux 7.2 records it.
    var raw: [76]u8 = @splat(0);
    std.mem.writeInt(u32, raw[8..12], 257, .little);
    std.mem.writeInt(u32, raw[48..52], 52 | 17 << 16, .little);
    @memcpy(raw[52..68], "/etc/ld.so.cache");
    try std.testing.expectEqualStrings("/etc/ld.so.cache", userData(&raw));
    // An argument that looks like a pointer into the record is not one: the sizes disagree.
    std.mem.writeInt(u32, raw[16..20], 20 | 4 << 16, .little);
    try std.testing.expectEqualStrings("/etc/ld.so.cache", userData(&raw));
    // An older kernel: arguments only.
    try std.testing.expectEqualStrings("", userData(raw[0..52]));
}