summaryrefslogtreecommitdiff
path: root/src/trace/bpf.zig
blob: 5dad4c6aca6fefbc4b0af1baf1b36cb06d6566a3 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
//! The kernel's view with privilege: BPF programs on the raw tracepoints
//! sys_enter, sys_exit and sched_switch, and on any kprobe, uprobe or USDT
//! probe. Needs CAP_BPF and CAP_PERFMON (root). Each program writes an
//! `Event`, and for a probe in user space the thread's registers after it,
//! into one BPF ring buffer; `Tracer.poll` moves them into a `Sink`.
//!
//! The programs are instruction arrays built here, for one process (its id
//! is a constant in them): no compiler, no object file, no libbpf. What the
//! kernel does not tell (where `pid` is in `task_struct`) comes from its BTF.
const std = @import("std");
const linux = std.os.linux;
const BPF = linux.BPF;
const Insn = BPF.Insn;
const event = @import("event.zig");
const usdt = @import("usdt.zig");
const Event = event.Event;
const Regs = event.Regs;
const Kind = event.Kind;

/// The kernel's `struct pt_regs` is `Regs` up to here.
const pt_regs_bytes = @offsetOf(Regs, "fs_base");
/// On a program's stack, from the frame pointer down: the key of the drop
/// counter, then the record. An event alone, or an event and registers.
const key_at = -8;
const small_at = key_at - @sizeOf(Event);
const large_at = small_at - pt_regs_bytes;

fn field(comptime name: []const u8) i16 {
    return @offsetOf(Event, name);
}

fn arg(i: usize) i16 {
    return @intCast(@offsetOf(Event, "a") + 8 * i);
}

fn reg(comptime name: []const u8) i16 {
    return @offsetOf(Regs, name);
}

pub const Program = struct {
    insns: [192]Insn = undefined,
    len: usize = 0,
    /// The jump that leaves early, to be pointed at the end.
    leave: ?usize = null,

    pub fn code(p: *const Program) []const Insn {
        return p.insns[0..p.len];
    }

    fn add(p: *Program, insn: Insn) void {
        p.insns[p.len] = insn;
        p.len += 1;
    }

    /// r6 = the context, r7 = thread and process id; goes on only in a
    /// thread of process `tgid`.
    fn enter(p: *Program, tgid: u32) void {
        p.add(Insn.mov(.r6, .r1));
        p.add(Insn.call(.get_current_pid_tgid));
        p.add(Insn.mov(.r7, .r0));
        p.add(Insn.rsh(.r0, 32));
        p.leave = p.len;
        p.add(Insn.jne(.r0, @as(i32, @intCast(tgid)), 0));
    }

    /// The time, the thread and the kind of the event at `at`.
    fn stamp(p: *Program, at: i16, kind: Kind) void {
        p.add(Insn.call(.ktime_get_ns));
        p.add(Insn.stx(.double_word, .r10, at + field("ts"), .r0));
        p.add(Insn.stx(.word, .r10, at + field("tid"), .r7));
        p.add(Insn.st(.half_word, .r10, at + field("kind"), @backingInt(kind)));
    }

    /// bpf_probe_read_kernel(frame + `to`, `size`, r3)
    fn read(p: *Program, to: i16, size: i32) void {
        p.add(Insn.mov(.r1, .r10));
        p.add(Insn.add(.r1, to));
        p.add(Insn.mov(.r2, size));
        p.add(Insn.call(.probe_read_kernel));
    }

    /// The low `bytes` of a register, extended to all of it.
    fn extend(p: *Program, r: Insn.Reg, bytes: u8, signed: bool) void {
        if (bytes >= 8) return;
        const shift: i32 = 64 - 8 * @as(i32, bytes);
        p.add(Insn.lsh(r, shift));
        p.add(if (signed) Insn.arsh(r, shift) else Insn.rsh(r, shift));
    }

    /// A register of the probed thread, from the context in r6.
    fn probed(p: *Program, to: Insn.Reg, r: anytype) void {
        p.add(Insn.ldx(.double_word, to, .r6, @as(i16, r.slot) * 8));
        p.extend(to, r.bytes, false);
    }

    /// One argument of a USDT probe into the u64 at `to`: what
    /// `usdt.Arg.fetch` computes, in the kernel, at the probe. Memory that
    /// cannot be read gives 0.
    fn usdtArg(p: *Program, a: usdt.Arg, to: i16) void {
        switch (a.at) {
            .constant => |c| {
                p.add(Insn.ld_dw1(.r1, @bitCast(c)));
                p.add(Insn.ld_dw2(@bitCast(c)));
            },
            .register => |r| p.probed(.r1, r),
            .memory => |m| {
                p.add(Insn.st(.double_word, .r10, to, 0));
                p.probed(.r3, m.base);
                if (m.index) |index| {
                    p.probed(.r1, index);
                    p.add(Insn.lsh(.r1, @as(i32, @ctz(m.scale))));
                    p.add(Insn.add(.r3, .r1));
                }
                p.add(Insn.ld_dw1(.r1, @bitCast(m.disp)));
                p.add(Insn.ld_dw2(@bitCast(m.disp)));
                p.add(Insn.add(.r3, .r1));
                p.add(Insn.mov(.r1, .r10));
                p.add(Insn.add(.r1, to));
                p.add(Insn.mov(.r2, @as(i32, a.size)));
                p.add(Insn.call(.probe_read_user));
                p.add(Insn.ldx(.double_word, .r1, .r10, to));
            },
        }
        p.extend(.r1, a.size, a.signed);
        p.add(Insn.stx(.double_word, .r10, to, .r1));
    }

    /// Sends the `size` bytes at `at`, counts the record in `drops` when the
    /// ring is full, and ends the program.
    fn send(p: *Program, maps: Maps, at: i16, size: i32) void {
        p.add(Insn.ld_map_fd1(.r1, maps.ring));
        p.add(Insn.ld_map_fd2(maps.ring));
        p.add(Insn.mov(.r2, .r10));
        p.add(Insn.add(.r2, at));
        p.add(Insn.mov(.r3, size));
        p.add(Insn.mov(.r4, 0));
        p.add(Insn.call(.ringbuf_output));
        p.add(Insn.jeq(.r0, 0, 9));
        p.add(Insn.st(.word, .r10, key_at, 0));
        p.add(Insn.ld_map_fd1(.r1, maps.drops));
        p.add(Insn.ld_map_fd2(maps.drops));
        p.add(Insn.mov(.r2, .r10));
        p.add(Insn.add(.r2, key_at));
        p.add(Insn.call(.map_lookup_elem));
        p.add(Insn.jeq(.r0, 0, 2));
        p.add(Insn.mov(.r1, 1));
        p.add(Insn.xadd(.r0, .r1));
        if (p.leave) |at_jump| p.insns[at_jump].off = @intCast(p.len - at_jump - 1);
        p.add(Insn.mov(.r0, 0));
        p.add(Insn.exit());
    }
};

/// The maps every program writes to: the ring of records, and an array of
/// one u64 counting the records that found it full.
pub const Maps = struct { ring: linux.fd_t, drops: linux.fd_t };

/// raw tracepoint sys_enter(struct pt_regs *regs, long id): a
/// `syscall_enter`.
pub fn sysEnter(maps: Maps, tgid: u32) Program {
    var p: Program = .{};
    // The registers are read to below the event, for the six that are arguments.
    p.enter(tgid);
    p.add(Insn.ldx(.double_word, .r3, .r6, 0));
    p.read(large_at, pt_regs_bytes);
    p.stamp(small_at, .syscall_enter);
    p.add(Insn.ldx(.double_word, .r1, .r6, 8));
    p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
    inline for (.{ "rdi", "rsi", "rdx", "r10", "r8", "r9" }, 0..) |name, i| {
        p.add(Insn.ldx(.double_word, .r1, .r10, large_at + reg(name)));
        p.add(Insn.stx(.double_word, .r10, small_at + arg(i), .r1));
    }
    p.send(maps, small_at, @sizeOf(Event));
    return p;
}

/// raw tracepoint sys_exit(struct pt_regs *regs, long ret): a `syscall_exit`.
pub fn sysExit(maps: Maps, tgid: u32) Program {
    var p: Program = .{};
    const number_at = small_at - 8;
    p.enter(tgid);
    p.add(Insn.ldx(.double_word, .r3, .r6, 0));
    p.add(Insn.add(.r3, reg("orig_rax")));
    p.read(number_at, 8);
    p.stamp(small_at, .syscall_exit);
    p.add(Insn.ldx(.double_word, .r1, .r10, number_at));
    p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
    p.add(Insn.ldx(.double_word, .r1, .r6, 8));
    p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r1));
    for (1..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0));
    p.send(maps, small_at, @sizeOf(Event));
    return p;
}

/// raw tracepoint sched_switch(bool preempt, struct task_struct *prev,
/// *next): a `sched` when either is a thread of the process. `pid_at` and
/// `tgid_at` are where those fields are in `task_struct`.
pub fn schedSwitch(maps: Maps, tgid: u32, pid_at: u32, tgid_at: u32) Program {
    var p: Program = .{};
    // prev's pid and tgid, next's pid and tgid, as u32 below the event.
    const ids_at = small_at - 16;
    p.add(Insn.mov(.r6, .r1));
    for ([_]i16{ 8, 16 }, 0..) |task, t| for ([_]u32{ pid_at, tgid_at }, 0..) |at, i| {
        p.add(Insn.ldx(.double_word, .r3, .r6, task));
        p.add(Insn.add(.r3, @as(i32, @intCast(at))));
        p.read(ids_at + @as(i16, @intCast(8 * t + 4 * i)), 4);
    };
    p.add(Insn.ldx(.word, .r1, .r10, ids_at + 4));
    p.add(Insn.jeq(.r1, @as(i32, @intCast(tgid)), 2));
    p.add(Insn.ldx(.word, .r1, .r10, ids_at + 12));
    p.leave = p.len;
    p.add(Insn.jne(.r1, @as(i32, @intCast(tgid)), 0));
    p.add(Insn.ldx(.word, .r7, .r10, ids_at));
    p.stamp(small_at, .sched);
    p.add(Insn.ldx(.double_word, .r1, .r6, 0));
    p.add(Insn.stx(.half_word, .r10, small_at + field("aux"), .r1));
    p.add(Insn.stx(.double_word, .r10, small_at + arg(0), .r7));
    p.add(Insn.ldx(.word, .r1, .r10, ids_at + 8));
    p.add(Insn.stx(.double_word, .r10, small_at + arg(1), .r1));
    p.add(Insn.call(.get_smp_processor_id));
    p.add(Insn.stx(.double_word, .r10, small_at + arg(2), .r0));
    for (3..6) |i| p.add(Insn.st(.double_word, .r10, small_at + arg(i), 0));
    p.send(maps, small_at, @sizeOf(Event));
    return p;
}

/// A kprobe or uprobe (the context is the registers there): a `probe`
/// named `id`. Its a1..a5 are the first five arguments of a function call,
/// or with `args` those of a USDT probe, read there and then. In `user`
/// space the registers follow the event: they are the thread's, which a
/// kernel function's are not.
pub fn probe(maps: Maps, tgid: u32, id: u64, args: ?[]const usdt.Arg, user: bool) Program {
    var p: Program = .{};
    const at: i16 = if (user) large_at else small_at;
    p.enter(tgid);
    p.stamp(at, .probe);
    p.add(Insn.st(.half_word, .r10, at + field("aux"), 0));
    p.add(Insn.ld_dw1(.r1, id));
    p.add(Insn.ld_dw2(id));
    p.add(Insn.stx(.double_word, .r10, at + arg(0), .r1));
    inline for (.{ "rdi", "rsi", "rdx", "rcx", "r8" }, 1..) |name, i| {
        if (args == null) {
            p.add(Insn.ldx(.double_word, .r1, .r6, reg(name)));
            p.add(Insn.stx(.double_word, .r10, at + arg(i), .r1));
        } else if (i <= args.?.len) {
            p.usdtArg(args.?[i - 1], at + arg(i));
        } else p.add(Insn.st(.double_word, .r10, at + arg(i), 0));
    }
    if (user) for (0..pt_regs_bytes / 8) |i| {
        p.add(Insn.ldx(.double_word, .r1, .r6, @intCast(8 * i)));
        p.add(Insn.stx(.double_word, .r10, at + @sizeOf(Event) + @as(i16, @intCast(8 * i)), .r1));
    };
    p.send(maps, at, if (user) @sizeOf(Event) + pt_regs_bytes else @sizeOf(Event));
    return p;
}

/// `probe` for a USDT probe with the arguments `specs` ("-4@%edi
/// 8@-24(%rbp)"): the first five become a1..a5.
pub fn usdtProgram(maps: Maps, tgid: u32, id: u64, specs: []const u8) !Program {
    var args: [5]usdt.Arg = undefined;
    var count: usize = 0;
    var each = std.mem.tokenizeScalar(u8, specs, ' ');
    while (each.next()) |spec| {
        if (count == args.len) break;
        args[count] = try .parse(spec);
        count += 1;
    }
    return probe(maps, tgid, id, args[0..count], true);
}

/// Where `member` is in the struct `name`, in bytes, from the kernel's
/// type information (the contents of /sys/kernel/btf/vmlinux).
pub fn fieldOffset(btf: []const u8, name: []const u8, member: []const u8) ?u32 {
    const T = BPF.btf;
    if (btf.len < @sizeOf(T.Header)) return null;
    const header = std.mem.bytesToValue(T.Header, btf[0..@sizeOf(T.Header)]);
    if (header.magic != T.magic or header.hdr_len + @as(u64, header.str_off) + header.str_len > btf.len) return null;
    const types = btf[header.hdr_len + header.type_off ..][0..header.type_len];
    const strings = btf[header.hdr_len + header.str_off ..][0..header.str_len];
    var at: usize = 0;
    while (at + @sizeOf(T.Type) <= types.len) {
        const t = std.mem.bytesToValue(T.Type, types[at..][0..@sizeOf(T.Type)]);
        at += @sizeOf(T.Type);
        const count: usize = t.info.vlen;
        // What follows the type, by kind.
        const extra: usize = switch (t.info.kind) {
            .int, .@"var", .decl_tag => 4,
            .array => 12,
            .@"struct", .@"union", .datasec, .enum64 => 12 * count,
            .@"enum", .func_proto => 8 * count,
            else => 0,
        };
        if (at + extra > types.len) return null;
        defer at += extra;
        if (t.info.kind != .@"struct" or !std.mem.eql(u8, std.mem.sliceTo(strings[@min(t.name_off, strings.len)..], 0), name)) continue;
        for (0..count) |i| {
            const m = std.mem.bytesToValue(T.Member, types[at + 12 * i ..][0..12]);
            if (std.mem.eql(u8, std.mem.sliceTo(strings[@min(m.name_off, strings.len)..], 0), member)) return m.offset.bit / 8;
        }
    }
    return null;
}

/// The reading end of a BPF ring buffer: a page with our position, a page
/// with the kernel's, and the data, which the kernel maps twice in a row so
/// that no record wraps. A record is a u32 length with two flag bits, four
/// bytes more, the bytes, padded to 8.
pub const RingBuf = struct {
    consumer: *u64,
    producer: *const u64,
    data: []const u8,
    /// Read so far; the kernel is told by `commit`, after the records were used.
    at: u64 = 0,

    const busy: u32 = 1 << 31;
    const discarded: u32 = 1 << 30;

    pub fn map(fd: linux.fd_t, size: usize) !RingBuf {
        const ours = linux.mmap(null, 4096, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED }, fd, 0);
        if (linux.errno(ours) != .SUCCESS) return error.BpfRingMap;
        const theirs = linux.mmap(null, 4096 + 2 * size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 4096);
        if (linux.errno(theirs) != .SUCCESS) return error.BpfRingMap;
        return .{ .consumer = @ptrFromInt(ours), .producer = @ptrFromInt(theirs), .data = @as([*]const u8, @ptrFromInt(theirs + 4096))[0..size] };
    }

    /// The next finished record, valid until `commit`.
    pub fn next(r: *RingBuf) ?[]const u8 {
        while (r.at < @atomicLoad(u64, r.producer, .acquire)) {
            const start: usize = @intCast(r.at & (r.data.len - 1));
            const header = @atomicLoad(u32, @as(*const u32, @ptrCast(@alignCast(r.data.ptr + start))), .acquire);
            if (header & busy != 0) return null;
            const len = header & ~(busy | discarded);
            r.at += std.mem.alignForward(u64, len + 8, 8);
            if (header & discarded == 0) return (r.data.ptr + start + 8)[0..len];
        }
        return null;
    }

    pub fn commit(r: *RingBuf) void {
        @atomicStore(u64, r.consumer, r.at, .release);
    }
};

/// The capability bits of this process (CapEff of /proc/self/status).
fn capabilities() u64 {
    var buf: [4096]u8 = undefined;
    const fd = linux.open("/proc/self/status", .{}, 0);
    if (linux.errno(fd) != .SUCCESS) return 0;
    defer _ = linux.close(@intCast(fd));
    const n = linux.read(@intCast(fd), &buf, buf.len);
    if (linux.errno(n) != .SUCCESS) return 0;
    const label = "CapEff:\t";
    const at = std.mem.indexOf(u8, buf[0..n], label) orelse return 0;
    return std.fmt.parseInt(u64, buf[at + label.len ..][0..16], 16) catch 0;
}

/// What this process lacks to trace with BPF, said into `buf`; null when
/// it has the capabilities (which inside a user namespace still is not
/// enough: the kernel wants them in the initial one).
pub fn missing(buf: []u8) ?[]const u8 {
    const caps = capabilities();
    const has = struct {
        fn has(all: u64, cap: u6) bool {
            return all >> cap & 1 != 0;
        }
    }.has;
    if (has(caps, linux.CAP.SYS_ADMIN) or (has(caps, linux.CAP.BPF) and has(caps, linux.CAP.PERFMON))) return null;
    return std.fmt.bufPrint(buf, "missing {s}{s}{s} (effective capabilities {x:0>16}, uid {d}). " ++
        "CAP_BPF is what bpf() wants to create maps and load programs, CAP_PERFMON to load tracing programs and open kprobes and uprobes; CAP_SYS_ADMIN stands for both. " ++
        "Run as root, or give the binary the two: setcap cap_bpf,cap_perfmon+ep ztrace", .{
        if (has(caps, linux.CAP.BPF)) "" else "CAP_BPF",
        if (has(caps, linux.CAP.BPF) or has(caps, linux.CAP.PERFMON)) "" else " and ",
        if (has(caps, linux.CAP.PERFMON)) "" else "CAP_PERFMON",
        caps,
        linux.getuid(),
    }) catch null;
}

pub const Tracer = struct {
    gpa: std.mem.Allocator,
    /// The process traced: its thread group id, in the initial pid namespace.
    /// shortcut: one process, and a tracer in a pid namespace of its own sees nothing; keep the ids in a BPF map to follow children.
    pid: linux.pid_t,
    sink: event.Sink,
    maps: Maps,
    ring: RingBuf,
    /// Programs and what attaches them.
    fds: std.ArrayList(linux.fd_t) = .empty,
    /// What the verifier said about the last program, when it refused one.
    log: []u8,

    pub fn open(gpa: std.mem.Allocator, pid: linux.pid_t, sink: event.Sink, ring_bytes: u32) !Tracer {
        const ring = try BPF.map_create(.ringbuf, 0, 0, ring_bytes);
        errdefer _ = linux.close(ring);
        const drops = try BPF.map_create(.array, 4, 8, 1);
        errdefer _ = linux.close(drops);
        const ring_map = try RingBuf.map(ring, ring_bytes);
        const log = try gpa.alloc(u8, 1 << 16);
        log[0] = 0;
        return .{ .gpa = gpa, .pid = pid, .sink = sink, .maps = .{ .ring = ring, .drops = drops }, .ring = ring_map, .log = log };
    }

    pub fn close(t: *Tracer) void {
        for (t.fds.items) |fd| _ = linux.close(fd);
        t.fds.deinit(t.gpa);
        _ = linux.munmap(@ptrCast(@alignCast(t.ring.consumer)), 4096);
        _ = linux.munmap(@ptrCast(@alignCast(t.ring.producer)), 4096 + 2 * t.ring.data.len);
        _ = linux.close(t.maps.ring);
        _ = linux.close(t.maps.drops);
        t.gpa.free(t.log);
    }

    fn load(t: *Tracer, kind: BPF.ProgType, program: Program) !linux.fd_t {
        // GPL: bpf_probe_read_kernel is for programs that say so.
        const fd = BPF.prog_load(kind, program.code(), null, "GPL", 0, 0) catch |err| {
            // Again for the verifier's reasons, which a good program need not pay for.
            var log: BPF.Log = .{ .level = 1, .buf = t.log };
            if (BPF.prog_load(kind, program.code(), &log, "GPL", 0, 0)) |again| _ = linux.close(again) else |_| {}
            return err;
        };
        try t.fds.append(t.gpa, fd);
        return fd;
    }

    fn rawTracepoint(t: *Tracer, name: [*:0]const u8, program: Program) !void {
        var attr = std.mem.zeroes(BPF.Attr);
        attr.raw_tracepoint = .{ .name = @intFromPtr(name), .prog_fd = try t.load(.raw_tracepoint, program) };
        const rc = linux.bpf(.raw_tracepoint_open, &attr, @sizeOf(BPF.RawTracepointAttr));
        if (linux.errno(rc) != .SUCCESS) return error.BpfAttach;
        try t.fds.append(t.gpa, @intCast(rc));
    }

    /// Every syscall of the process.
    pub fn syscalls(t: *Tracer) !void {
        try t.rawTracepoint("sys_enter", sysEnter(t.maps, @intCast(t.pid)));
        try t.rawTracepoint("sys_exit", sysExit(t.maps, @intCast(t.pid)));
    }

    /// Every time one of its threads leaves or gets a CPU. `btf` is the
    /// contents of /sys/kernel/btf/vmlinux.
    pub fn sched(t: *Tracer, btf: []const u8) !void {
        const pid_at = fieldOffset(btf, "task_struct", "pid") orelse return error.NoBtf;
        const tgid_at = fieldOffset(btf, "task_struct", "tgid") orelse return error.NoBtf;
        try t.rawTracepoint("sched_switch", schedSwitch(t.maps, @intCast(t.pid), pid_at, tgid_at));
    }

    /// A probe through the perf event of the kprobe or uprobe PMU: `target`
    /// is a kernel function or a file, `offset` is into it.
    fn attach(t: *Tracer, pmu: []const u8, target: [*:0]const u8, offset: u64, semaphore: u64, program: Program) !void {
        var path_buf: [64]u8 = undefined;
        var number: [16]u8 = undefined;
        const fd = linux.open(try std.fmt.bufPrintSentinel(&path_buf, "/sys/bus/event_source/devices/{s}/type", .{pmu}, 0), .{}, 0);
        if (linux.errno(fd) != .SUCCESS) return error.NoProbeSupport;
        defer _ = linux.close(@intCast(fd));
        const n = linux.read(@intCast(fd), &number, number.len);
        if (linux.errno(n) != .SUCCESS) return error.NoProbeSupport;
        const kind = std.fmt.parseInt(u32, std.mem.trim(u8, number[0..n], "\n "), 10) catch return error.NoProbeSupport;

        // A uprobe's semaphore (the offset in the file of its counter) rides in the upper half of config.
        // On the first thread alone, and not inherited: a program on a probe
        // runs for whoever hits it (hence the process id in ours), and the
        // kernel reads `target` again, from the target's memory, for an
        // event a clone would inherit, which fails the clone.
        var attr: linux.perf_event_attr = .{ .type = @fromBackingInt(@intCast(kind)), .config = semaphore << 32, .config1 = @intFromPtr(target), .config2 = offset };
        const event_fd = linux.perf_event_open(&attr, t.pid, -1, -1, linux.PERF.FLAG.FD_CLOEXEC);
        if (linux.errno(event_fd) != .SUCCESS) return if (linux.errno(event_fd) == .NOENT) error.NoSuchProbe else error.BpfAttach;
        try t.fds.append(t.gpa, @intCast(event_fd));
        const loaded = try t.load(.kprobe, program);
        if (linux.errno(linux.ioctl(@intCast(event_fd), linux.PERF.EVENT_IOC.SET_BPF, @intCast(loaded))) != .SUCCESS) return error.BpfAttach;
    }

    /// Every call of a kernel function by the process: a `probe` named
    /// `function` (define the string yourself: its id is `id`).
    pub fn kprobe(t: *Tracer, function: [*:0]const u8, id: u64) !void {
        try t.attach("kprobe", function, 0, 0, probe(t.maps, @intCast(t.pid), id, null, false));
    }

    /// Every time the process runs the instruction at `offset` of the
    /// file `path`.
    pub fn uprobe(t: *Tracer, path: [*:0]const u8, offset: u64, id: u64) !void {
        try t.attach("uprobe", path, offset, 0, probe(t.maps, @intCast(t.pid), id, null, true));
    }

    /// A static probe of the file `path` (`p` from `usdt.probes` with no
    /// bias). The kernel raises its semaphore while we are attached.
    pub fn usdtProbe(t: *Tracer, elf: usdt.Elf, path: [*:0]const u8, p: usdt.Probe, id: u64) !void {
        const semaphore = if (p.semaphore == 0) 0 else elf.offset(p.semaphore) orelse return error.NoSuchProbe;
        try t.attach("uprobe", path, elf.offset(p.pc) orelse return error.NoSuchProbe, semaphore, try usdtProgram(t.maps, @intCast(t.pid), id, p.args));
    }

    /// Waits up to the timeout for records and decodes them into the sink.
    /// Returns how many.
    pub fn poll(t: *Tracer, timeout_ms: i32) !usize {
        var fds = [1]linux.pollfd{.{ .fd = t.maps.ring, .events = linux.POLL.IN, .revents = 0 }};
        _ = linux.poll(&fds, 1, timeout_ms);
        var n: usize = 0;
        while (t.ring.next()) |record| : (n += 1) try decode(record, t.sink);
        t.ring.commit();
        return n;
    }

    /// Records that found the ring full.
    pub fn lost(t: *Tracer) u64 {
        var count: u64 = 0;
        BPF.map_lookup_elem(t.maps.drops, &std.mem.toBytes(@as(u32, 0)), std.mem.asBytes(&count)) catch return 0;
        return count;
    }
};

/// A record of the ring into the sink: the event, and the registers when
/// they follow it.
fn decode(record: []const u8, sink: event.Sink) !void {
    if (record.len < @sizeOf(Event)) return;
    const ev = std.mem.bytesToValue(Event, record[0..@sizeOf(Event)]);
    if (record.len < @sizeOf(Event) + pt_regs_bytes) return sink.emit(ev, "");
    var regs: Regs = .{};
    @memcpy(std.mem.asBytes(&regs)[0..pt_regs_bytes], record[@sizeOf(Event)..][0..pt_regs_bytes]);
    try sink.emit(ev, std.mem.asBytes(&regs));
}

// What follows runs the programs without a kernel: enough of a BPF machine
// for what they use, on this process's own memory.
const Machine = struct {
    pid_tgid: u64 = 0,
    now: u64 = 0,
    cpu: u64 = 0,
    /// What bpf_ringbuf_output returns; not 0 is a full ring.
    ring_full: bool = false,
    sent: [512]u8 = undefined,
    sent_len: ?usize = null,
    drops: u64 = 0,

    fn run(m: *Machine, code: []const Insn, context: *const anyopaque) !void {
        var stack: [512]u8 align(8) = undefined;
        var r: [11]u64 = @splat(0);
        r[1] = @intFromPtr(context);
        r[10] = @intFromPtr(&stack) + stack.len;
        var pc: usize = 0;
        var steps: usize = 0;
        while (true) : (steps += 1) {
            if (steps > 10_000 or pc >= code.len) return error.RanAway;
            const insn = code[pc];
            pc += 1;
            const imm: u64 = @bitCast(@as(i64, insn.imm));
            const source = if (insn.code & BPF.X != 0) r[insn.src] else imm;
            const address = (if (insn.code & 7 == BPF.LDX) r[insn.src] else r[insn.dst]) +% @as(u64, @bitCast(@as(i64, insn.off)));
            switch (insn.code) {
                BPF.ALU64 | BPF.MOV | BPF.K, BPF.ALU64 | BPF.MOV | BPF.X => r[insn.dst] = source,
                BPF.ALU64 | BPF.ADD | BPF.K, BPF.ALU64 | BPF.ADD | BPF.X => r[insn.dst] +%= source,
                BPF.ALU64 | BPF.RSH | BPF.K => r[insn.dst] >>= @intCast(insn.imm),
                BPF.ALU64 | BPF.LSH | BPF.K => r[insn.dst] <<= @intCast(insn.imm),
                BPF.ALU64 | BPF.ARSH | BPF.K => r[insn.dst] = @bitCast(@as(i64, @bitCast(r[insn.dst])) >> @intCast(insn.imm)),
                BPF.JMP | BPF.JEQ | BPF.K => if (r[insn.dst] == imm) {
                    pc = @intCast(@as(i64, @intCast(pc)) + insn.off);
                },
                BPF.JMP | BPF.JNE | BPF.K => if (r[insn.dst] != imm) {
                    pc = @intCast(@as(i64, @intCast(pc)) + insn.off);
                },
                BPF.LD | BPF.DW | BPF.IMM => {
                    r[insn.dst] = @as(u32, @bitCast(insn.imm)) | @as(u64, @as(u32, @bitCast(code[pc].imm))) << 32;
                    pc += 1;
                },
                inline BPF.LDX | BPF.MEM | BPF.B, BPF.LDX | BPF.MEM | BPF.H, BPF.LDX | BPF.MEM | BPF.W, BPF.LDX | BPF.MEM | BPF.DW => |code_now| {
                    r[insn.dst] = @as(*align(1) const Width(code_now), @ptrFromInt(address)).*;
                },
                inline BPF.STX | BPF.MEM | BPF.H, BPF.STX | BPF.MEM | BPF.W, BPF.STX | BPF.MEM | BPF.DW => |code_now| {
                    @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(r[insn.src]);
                },
                inline BPF.ST | BPF.MEM | BPF.H, BPF.ST | BPF.MEM | BPF.W, BPF.ST | BPF.MEM | BPF.DW => |code_now| {
                    @as(*align(1) Width(code_now), @ptrFromInt(address)).* = @truncate(imm);
                },
                BPF.STX | BPF.XADD | BPF.DW => @as(*u64, @ptrFromInt(address)).* += r[insn.src],
                BPF.JMP | BPF.CALL => r[0] = switch (@as(BPF.Helper, @fromBackingInt(@intCast(insn.imm)))) {
                    .get_current_pid_tgid => m.pid_tgid,
                    .ktime_get_ns => m.now,
                    .get_smp_processor_id => m.cpu,
                    .probe_read_kernel, .probe_read_user => read: {
                        @memcpy(@as([*]u8, @ptrFromInt(r[1]))[0..r[2]], @as([*]const u8, @ptrFromInt(r[3]))[0..r[2]]);
                        break :read 0;
                    },
                    .ringbuf_output => out: {
                        if (r[1] != 3 or m.ring_full) break :out @as(u64, @bitCast(@as(i64, -11)));
                        m.sent_len = r[3];
                        @memcpy(m.sent[0..r[3]], @as([*]const u8, @ptrFromInt(r[2]))[0..r[3]]);
                        break :out 0;
                    },
                    .map_lookup_elem => if (r[1] == 4 and @as(*const u32, @ptrFromInt(r[2])).* == 0) @intFromPtr(&m.drops) else 0,
                    else => return error.UnknownHelper,
                },
                BPF.JMP | BPF.EXIT => return,
                else => return error.UnknownInstruction,
            }
        }
    }

    fn Width(comptime code: u8) type {
        return switch (code & 0x18) {
            BPF.B => u8,
            BPF.H => u16,
            BPF.W => u32,
            else => u64,
        };
    }

    fn sentEvent(m: *const Machine) Event {
        return std.mem.bytesToValue(Event, m.sent[0..@sizeOf(Event)]);
    }
};

const test_maps: Maps = .{ .ring = 3, .drops = 4 };

test "the syscall programs: the right process only, the event the tracer expects" {
    var regs: Regs = .{ .rdi = 1, .rsi = 2, .rdx = 3, .r10 = 4, .r8 = 5, .r9 = 6, .orig_rax = 257, .rip = 0x401000 };
    const enter_context = [2]u64{ @intFromPtr(&regs), 257 };
    var m: Machine = .{ .pid_tgid = 1000 << 32 | 1001, .now = 77 };
    try m.run(sysEnter(test_maps, 1000).code(), &enter_context);
    try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len);
    try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_enter, .aux = 257, .a = .{ 1, 2, 3, 4, 5, 6 } }, m.sentEvent());

    const exit_context = [2]u64{ @intFromPtr(&regs), @bitCast(@as(i64, -2)) };
    try m.run(sysExit(test_maps, 1000).code(), &exit_context);
    try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len);
    try std.testing.expectEqual(Event{ .ts = 77, .tid = 1001, .kind = .syscall_exit, .aux = 257, .a = .{ @bitCast(@as(i64, -2)), 0, 0, 0, 0, 0 } }, m.sentEvent());

    // Another process: nothing. A full ring: counted.
    m.sent_len = null;
    try m.run(sysEnter(test_maps, 999).code(), &enter_context);
    try std.testing.expectEqual(@as(?usize, null), m.sent_len);
    m.ring_full = true;
    try m.run(sysExit(test_maps, 1000).code(), &exit_context);
    try m.run(sysEnter(test_maps, 1000).code(), &enter_context);
    try std.testing.expectEqual(@as(u64, 2), m.drops);
}

test "the sched_switch program: either side of the switch, ids from where BTF says" {
    // Two task_structs with the ids at offsets of our choosing.
    var prev: [64]u32 = @splat(0);
    var next: [64]u32 = @splat(0);
    prev[10] = 1001;
    prev[11] = 1000;
    next[10] = 55;
    next[11] = 50;
    const context = [3]u64{ 1, @intFromPtr(&prev), @intFromPtr(&next) };
    var m: Machine = .{ .now = 9, .cpu = 3 };
    try m.run(schedSwitch(test_maps, 1000, 40, 44).code(), &context);
    try std.testing.expectEqual(Event{ .ts = 9, .tid = 1001, .kind = .sched, .aux = 1, .a = .{ 1001, 55, 3, 0, 0, 0 } }, m.sentEvent());
    m.sent_len = null;
    try m.run(schedSwitch(test_maps, 50, 40, 44).code(), &context);
    try std.testing.expectEqual(@as(u64, 55), m.sentEvent().a[1]);
    m.sent_len = null;
    try m.run(schedSwitch(test_maps, 7, 40, 44).code(), &context);
    try std.testing.expectEqual(@as(?usize, null), m.sent_len);
}

test "the probe program and its record decoded; a USDT probe's arguments as usdt.Arg.fetch has them" {
    // The probed thread's memory is ours: rbp and rdx point into this.
    var memory: [64]u8 = undefined;
    for (&memory, 0..) |*b, i| b.* = @truncate(0xf0 + i);
    const regs: Regs = .{ .rdi = 11, .rsi = 0xffff_fffe, .rdx = @intFromPtr(&memory), .rcx = 3, .r8 = 15, .rbp = @intFromPtr(&memory) + 32, .rip = 0x401234, .rsp = 0x7000, .r10 = 0x1_0000_00ff };
    var m: Machine = .{ .pid_tgid = 1000 << 32 | 1000, .now = 5 };
    // In the kernel: the event alone. In user space: the registers too.
    try m.run(probe(test_maps, 1000, 0xfeed, null, false).code(), &regs);
    try std.testing.expectEqual(@as(?usize, @sizeOf(Event)), m.sent_len);
    try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent());
    try m.run(probe(test_maps, 1000, 0xfeed, null, true).code(), &regs);
    try std.testing.expectEqual(@as(?usize, @sizeOf(Event) + pt_regs_bytes), m.sent_len);
    try std.testing.expectEqual(Event{ .ts = 5, .tid = 1000, .kind = .probe, .a = .{ 0xfeed, 11, regs.rsi, regs.rdx, 3, 15 } }, m.sentEvent());

    // A sink that keeps the last event.
    const Last = struct {
        ev: Event = undefined,
        regs: Regs = undefined,
        fn emit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void {
            const l: *@This() = @ptrCast(@alignCast(ctx));
            l.ev = ev.*;
            l.regs = std.mem.bytesToValue(Regs, payload);
        }
    };
    var last: Last = .{};
    try decode(m.sent[0..m.sent_len.?], .{ .ctx = &last, .emitFn = Last.emit });
    try std.testing.expectEqual(@as(u64, 11), last.ev.a[1]);
    try std.testing.expectEqual(regs.rip, last.regs.rip);
    try std.testing.expectEqual(@as(u64, 0), last.regs.fs_base);

    const here = struct {
        pub fn read(_: @This(), addr: u64, buf: []u8) usize {
            @memcpy(buf, @as([*]const u8, @ptrFromInt(addr)));
            return buf.len;
        }
    }{};
    const specs = [_][5][]const u8{
        .{ "-4@%esi", "4@%esi", "1@-16(%rbp)", "8@$9", "-8@$-9" },
        .{ "-1@8(%rdx,%rcx,8)", "2@(%rdx)", "-2@-32(%rbp)", "1@%r10b", "8@%r10" },
    };
    for (specs) |five| {
        var args: [5]usdt.Arg = undefined;
        for (&args, five) |*a, spec| a.* = try .parse(spec);
        try m.run(probe(test_maps, 1000, 0xfeed, &args, true).code(), &regs);
        for (args, m.sentEvent().a[1..]) |a, got| try std.testing.expectEqual(a.fetch(&regs, here).?, got);
    }
    // Fewer than five: the rest is 0.
    try m.run(probe(test_maps, 1000, 0xfeed, &.{try .parse("4@%edi")}, true).code(), &regs);
    try std.testing.expectEqual([6]u64{ 0xfeed, 11, 0, 0, 0, 0 }, m.sentEvent().a);
}

test "a ring buffer as the kernel lays it out: finished records only, discarded ones skipped" {
    var consumer: u64 = 0;
    var producer: u64 = 0;
    var data: [64]u8 align(8) = @splat(0);
    var ring: RingBuf = .{ .consumer = &consumer, .producer = &producer, .data = &data };
    const records = [_]struct { u32, []const u8 }{ .{ 5, "hello" }, .{ 3 | RingBuf.discarded, "no!" }, .{ 2, "ok" }, .{ 4 | RingBuf.busy, "wait" } };
    for (records) |record| {
        std.mem.writeInt(u32, data[producer..][0..4], record[0], .little);
        @memcpy(data[producer + 8 ..][0..record[1].len], record[1]);
        producer += std.mem.alignForward(u64, record[1].len + 8, 8);
    }
    try std.testing.expectEqualStrings("hello", ring.next().?);
    try std.testing.expectEqualStrings("ok", ring.next().?);
    try std.testing.expect(ring.next() == null);
    ring.commit();
    try std.testing.expectEqual(@as(u64, 48), consumer);
    // The producer finishes it.
    std.mem.writeInt(u32, data[48..52], 4, .little);
    try std.testing.expectEqualStrings("wait", ring.next().?);
}

test "task_struct's pid and tgid in this kernel's BTF; privilege is told apart" {
    var buf: [512]u8 = undefined;
    if (missing(&buf)) |why| try std.testing.expect(std.mem.indexOf(u8, why, "CAP_") != null);

    const btf = std.Io.Dir.cwd().readFileAlloc(std.testing.io, "/sys/kernel/btf/vmlinux", std.testing.allocator, .unlimited) catch return error.SkipZigTest;
    defer std.testing.allocator.free(btf);
    const pid_at = fieldOffset(btf, "task_struct", "pid").?;
    // Next to each other, some way into a struct of a few kilobytes.
    try std.testing.expectEqual(pid_at + 4, fieldOffset(btf, "task_struct", "tgid").?);
    try std.testing.expect(pid_at > 256 and pid_at < 16384);
    try std.testing.expect(fieldOffset(btf, "task_struct", "no_such_field") == null);
    try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "rdi")), fieldOffset(btf, "pt_regs", "di"));
    try std.testing.expectEqual(@as(?u32, @offsetOf(Regs, "orig_rax")), fieldOffset(btf, "pt_regs", "orig_ax"));
}