summaryrefslogtreecommitdiff
path: root/src/trace/Ring.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/trace/Ring.zig')
-rw-r--r--src/trace/Ring.zig277
1 files changed, 277 insertions, 0 deletions
diff --git a/src/trace/Ring.zig b/src/trace/Ring.zig
new file mode 100644
index 0000000..47b490b
--- /dev/null
+++ b/src/trace/Ring.zig
@@ -0,0 +1,277 @@
+//! Events from a traced process to its tracer through shared memory: many
+//! producers (the target's threads), one consumer (the tracer). Lossless by
+//! default: a producer waits when the ring is full.
+//!
+//! The file is a header page and `capacity` data bytes; the data is mapped
+//! twice in a row, so a record never needs splitting at the end. A record is
+//! a u64 word (payload length, and the commit bit set last), an `Event`, and
+//! the payload padded to 8 bytes. Free space is always zero (the consumer
+//! clears what it reads), so a reserved record not yet written reads as
+//! uncommitted.
+//!
+//! Only raw syscalls and atomics: the agent calls `emit` from inside malloc
+//! and signal handlers.
+const Ring = @This();
+
+const std = @import("std");
+const linux = std.os.linux;
+const event = @import("event.zig");
+const Event = event.Event;
+
+pub const magic: u64 = 0x31474e4952545a; // "ZTRING1"
+const header_size = 4096;
+const commit_bit: u64 = 1 << 63;
+const record_overhead = 8 + @sizeOf(Event);
+
+pub const Header = extern struct {
+ magic: u64,
+ /// Data bytes, a power of two.
+ capacity: u64,
+ /// Bytes ever reserved by producers.
+ head: std.atomic.Value(u64) align(64),
+ /// Bytes ever consumed.
+ tail: std.atomic.Value(u64) align(64),
+ /// Events dropped because the ring was full and `lossy` was set.
+ dropped: std.atomic.Value(u64) align(64),
+ /// A set of `Kind.bit`s: a producer that emits one of these waits, after
+ /// committing it, until the consumer calls `resumeAll`. This is how the
+ /// tracer stops the target at an event without ptrace.
+ stop_mask: std.atomic.Value(u64),
+ /// Stopped producers whose record lies below this may continue.
+ acked: std.atomic.Value(u64),
+ resume_seq: std.atomic.Value(u32),
+ /// Producers now waiting in a stop.
+ stopped: std.atomic.Value(u32),
+ /// Drop instead of waiting when full.
+ lossy: std.atomic.Value(u32),
+ space_seq: std.atomic.Value(u32),
+ space_waiters: std.atomic.Value(u32),
+ data_seq: std.atomic.Value(u32),
+ consumer_sleeping: std.atomic.Value(u32),
+};
+
+pub const Record = struct { ev: *const Event, payload: []const u8 };
+
+fd: linux.fd_t,
+hdr: *Header,
+data: [*]u8,
+
+/// A new ring in an anonymous memory file. The fd is inheritable: a child
+/// maps it with `attach`.
+pub fn create(capacity: usize) !Ring {
+ std.debug.assert(std.math.isPowerOfTwo(capacity) and capacity >= header_size);
+ const fd = try check(linux.memfd_create("ztrace-ring", 0));
+ errdefer _ = linux.close(@intCast(fd));
+ _ = try check(linux.ftruncate(@intCast(fd), @intCast(header_size + capacity)));
+ var ring = try map(@intCast(fd), capacity);
+ ring.hdr.magic = magic;
+ ring.hdr.capacity = capacity;
+ return ring;
+}
+
+/// Maps a ring another process created.
+pub fn attach(fd: linux.fd_t) !Ring {
+ const page = try check(linux.mmap(null, header_size, .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 0));
+ const hdr: *const Header = @ptrFromInt(page);
+ const capacity = hdr.capacity;
+ const ok = hdr.magic == magic;
+ _ = linux.munmap(@ptrFromInt(page), header_size);
+ if (!ok) return error.NotARing;
+ return map(fd, capacity);
+}
+
+fn map(fd: linux.fd_t, capacity: usize) !Ring {
+ const rw: linux.PROT = .{ .READ = true, .WRITE = true };
+ const base = try check(linux.mmap(null, header_size + 2 * capacity, .{}, .{ .TYPE = .PRIVATE, .ANONYMOUS = true }, -1, 0));
+ const ptr: [*]u8 = @ptrFromInt(base);
+ _ = try check(linux.mmap(ptr, header_size + capacity, rw, .{ .TYPE = .SHARED, .FIXED = true }, fd, 0));
+ _ = try check(linux.mmap(ptr + header_size + capacity, capacity, rw, .{ .TYPE = .SHARED, .FIXED = true }, fd, header_size));
+ return .{ .fd = fd, .hdr = @ptrCast(@alignCast(ptr)), .data = ptr + header_size };
+}
+
+pub fn close(r: *Ring) void {
+ _ = linux.munmap(@ptrCast(r.hdr), header_size + 2 * r.hdr.capacity);
+ _ = linux.close(r.fd);
+}
+
+fn check(rc: usize) !usize {
+ return if (linux.errno(rc) == .SUCCESS) rc else error.RingSyscall;
+}
+
+fn futexWait(word: *const std.atomic.Value(u32), expect: u32, timeout_ns: ?u64) void {
+ const ts: ?linux.timespec = if (timeout_ns) |ns| .{ .sec = @intCast(ns / std.time.ns_per_s), .nsec = @intCast(ns % std.time.ns_per_s) } else null;
+ _ = linux.futex_4arg(word, .{ .cmd = .WAIT, .private = false }, expect, if (ts) |*t| t else null);
+}
+
+fn futexWake(word: *const std.atomic.Value(u32)) void {
+ _ = linux.futex_3arg(word, .{ .cmd = .WAKE, .private = false }, std.math.maxInt(i32));
+}
+
+// Producer side.
+
+pub fn emit(r: *Ring, ev: Event, payload: []const u8) void {
+ const h = r.hdr;
+ const need = record_overhead + std.mem.alignForward(usize, payload.len, 8);
+ std.debug.assert(need <= h.capacity);
+ var at = h.head.load(.monotonic);
+ while (true) {
+ if (at + need - h.tail.load(.acquire) > h.capacity) {
+ if (h.lossy.load(.monotonic) != 0) {
+ _ = h.dropped.fetchAdd(1, .monotonic);
+ return;
+ }
+ // Full: wait for the consumer. The timeout covers a wake lost
+ // between the check and the wait.
+ const seq = h.space_seq.load(.acquire);
+ _ = h.space_waiters.fetchAdd(1, .seq_cst);
+ if (at + need - h.tail.load(.seq_cst) > h.capacity) futexWait(&h.space_seq, seq, std.time.ns_per_ms);
+ _ = h.space_waiters.fetchSub(1, .monotonic);
+ at = h.head.load(.monotonic);
+ continue;
+ }
+ at = h.head.cmpxchgWeak(at, at + need, .acquire, .monotonic) orelse break;
+ }
+ const rec = r.data + (at & (h.capacity - 1));
+ @memcpy(rec[8..][0..@sizeOf(Event)], std.mem.asBytes(&ev));
+ @memcpy(rec[record_overhead..][0..payload.len], payload);
+ @atomicStore(u64, @as(*u64, @ptrCast(@alignCast(rec))), payload.len | commit_bit, .release);
+
+ if (h.consumer_sleeping.load(.seq_cst) != 0) {
+ _ = h.data_seq.fetchAdd(1, .release);
+ futexWake(&h.data_seq);
+ }
+ if (h.stop_mask.load(.monotonic) & ev.kind.bit() != 0) {
+ _ = h.stopped.fetchAdd(1, .release);
+ while (true) {
+ const seq = h.resume_seq.load(.seq_cst);
+ if (h.acked.load(.seq_cst) > at) break;
+ futexWait(&h.resume_seq, seq, null);
+ }
+ _ = h.stopped.fetchSub(1, .release);
+ }
+}
+
+pub fn sink(r: *Ring) event.Sink {
+ return .{ .ctx = r, .emitFn = sinkEmit };
+}
+
+fn sinkEmit(ctx: *anyopaque, ev: *const Event, payload: []const u8) anyerror!void {
+ const r: *Ring = @ptrCast(@alignCast(ctx));
+ r.emit(ev.*, payload);
+}
+
+// Consumer side.
+
+/// The oldest record, valid until `advance`. Null when there is none, or it
+/// is still being written.
+pub fn peek(r: *Ring) ?Record {
+ const h = r.hdr;
+ const t = h.tail.load(.monotonic);
+ if (t == h.head.load(.acquire)) return null;
+ const rec = r.data + (t & (h.capacity - 1));
+ const word = @atomicLoad(u64, @as(*const u64, @ptrCast(@alignCast(rec))), .acquire);
+ if (word & commit_bit == 0) return null;
+ const len: usize = @intCast(word & ~commit_bit);
+ return .{ .ev = @ptrCast(@alignCast(rec + 8)), .payload = rec[record_overhead..][0..len] };
+}
+
+/// Consumes the record `peek` returned.
+pub fn advance(r: *Ring, rec: Record) void {
+ const h = r.hdr;
+ const size = record_overhead + std.mem.alignForward(usize, rec.payload.len, 8);
+ const start: [*]u8 = @ptrFromInt(@intFromPtr(rec.ev) - 8);
+ @memset(start[0..size], 0);
+ _ = h.tail.fetchAdd(size, .seq_cst);
+ if (h.space_waiters.load(.seq_cst) != 0) {
+ _ = h.space_seq.fetchAdd(1, .release);
+ futexWake(&h.space_seq);
+ }
+}
+
+/// Everything committed so far, into `to`. Returns the number of events.
+pub fn drain(r: *Ring, to: event.Sink) !usize {
+ var n: usize = 0;
+ while (r.peek()) |rec| : (n += 1) {
+ try to.emit(rec.ev.*, rec.payload);
+ r.advance(rec);
+ }
+ return n;
+}
+
+/// Sleeps until a producer commits something, or the timeout.
+pub fn wait(r: *Ring, timeout_ns: u64) void {
+ const h = r.hdr;
+ const seq = h.data_seq.load(.acquire);
+ h.consumer_sleeping.store(1, .seq_cst);
+ if (r.peek() == null) futexWait(&h.data_seq, seq, timeout_ns);
+ h.consumer_sleeping.store(0, .release);
+}
+
+/// Lets every producer stopped at a record already consumed continue.
+pub fn resumeAll(r: *Ring) void {
+ r.hdr.acked.store(r.hdr.tail.load(.seq_cst), .seq_cst);
+ _ = r.hdr.resume_seq.fetchAdd(1, .seq_cst);
+ futexWake(&r.hdr.resume_seq);
+}
+
+test "records survive the wrap, payloads included" {
+ var ring = try Ring.create(4096);
+ defer ring.close();
+ var payload: [200]u8 = undefined;
+ // 264 bytes a record and a 4096-byte ring: the wrap falls inside one.
+ for (0..100) |i| {
+ @memset(&payload, @intCast(i));
+ ring.emit(.{ .kind = .mem_write, .a = .{ i, 0, 0, 0, 0, 0 } }, payload[0 .. i + 1]);
+ const rec = ring.peek().?;
+ try std.testing.expectEqual(i, rec.ev.a[0]);
+ try std.testing.expectEqual(i + 1, rec.payload.len);
+ try std.testing.expectEqual(@as(u8, @intCast(i)), rec.payload[i]);
+ ring.advance(rec);
+ }
+ try std.testing.expect(ring.peek() == null);
+}
+
+test "a second mapping sees the first's events; a full lossy ring drops" {
+ var a = try Ring.create(4096);
+ defer a.close();
+ var b = try Ring.attach(a.fd);
+ defer _ = linux.munmap(@ptrCast(b.hdr), header_size + 2 * b.hdr.capacity);
+ a.hdr.lossy.store(1, .monotonic);
+ for (0..100) |i| a.emit(.{ .kind = .step, .a = .{ i, 0, 0, 0, 0, 0 } }, "");
+ try std.testing.expectEqual(4096 / record_overhead, 100 - a.hdr.dropped.load(.monotonic));
+ try std.testing.expectEqual(@as(u64, 0), b.peek().?.ev.a[0]);
+}
+
+test "producers on other threads, a stop and a resume" {
+ var ring = try Ring.create(1 << 16);
+ defer ring.close();
+ ring.hdr.stop_mask.store(event.Kind.mark.bit(), .monotonic);
+ const per = 20_000;
+ const producer = struct {
+ fn run(r: *Ring, id: u64) void {
+ for (0..per) |i| r.emit(.{ .kind = .step, .a = .{ id, i, 0, 0, 0, 0 } }, "");
+ r.emit(.{ .kind = .mark, .a = .{ id, 0, 0, 0, 0, 0 } }, "done");
+ }
+ };
+ var threads: [4]std.Thread = undefined;
+ for (&threads, 0..) |*t, id| t.* = try std.Thread.spawn(.{}, producer.run, .{ &ring, id });
+ var next: [4]u64 = @splat(0);
+ var marks: usize = 0;
+ while (marks < 4) {
+ const rec = ring.peek() orelse {
+ ring.wait(std.time.ns_per_ms);
+ continue;
+ };
+ const id = rec.ev.a[0];
+ if (rec.ev.kind == .mark) marks += 1 else {
+ try std.testing.expectEqual(next[id], rec.ev.a[1]);
+ next[id] += 1;
+ }
+ ring.advance(rec);
+ }
+ // Every producer is held at its mark until resumed.
+ while (ring.hdr.stopped.load(.acquire) != 4) _ = linux.sched_yield();
+ ring.resumeAll();
+ for (threads) |t| t.join();
+ try std.testing.expectEqual(@as(u64, 0), ring.hdr.dropped.load(.monotonic));
+}