//! The agent as a library for programs that have a libc: the C side of the //! target's API (include/ztrace.h), and the libc functions it stands in //! front of: the allocator (`alloc` and `free` events with the caller's pc), //! pthread_create (`thread_start`, `thread_exit`) and fork. //! //! Preloaded, the library's `malloc` shadows libc's and finds it with //! dlsym(RTLD_NEXT). Linked in statically nothing can shadow libc, so the //! target is linked with `-Wl,--wrap=malloc` (and so on): its calls then go //! to `__wrap_malloc` here, which calls `__real_malloc`. const std = @import("std"); const builtin = @import("builtin"); const zt = @import("ztrace"); const agent = @import("agent.zig"); const preloaded = builtin.link_mode == .dynamic; /// A failed check says so and ends the process: no stack traces, no std I/O, /// from inside somebody else's program. pub const panic = std.debug.FullPanic(struct { fn panic(message: []const u8, _: ?usize) noreturn { @branchHint(.cold); const linux = std.os.linux; _ = linux.write(2, "ztrace agent: ", 14); _ = linux.write(2, message.ptr, message.len); _ = linux.write(2, "\n", 1); linux.exit_group(134); } }.panic); /// Runs when the program is loaded, before main. export const ztrace_agent_init linksection(".init_array") = [1]*const fn () callconv(.c) void{loaded}; fn loaded() callconv(.c) void { agent.init(); } export fn ztrace_mark(label: [*:0]const u8, value: u64) void { agent.mark(std.mem.span(label), value); } export fn ztrace_watch(ptr: [*]const u8, len: usize, name: [*:0]const u8) void { agent.watch(ptr[0..len], std.mem.span(name)); } export fn ztrace_snapshot(ptr: [*]const u8, len: usize) void { agent.snapshot(ptr[0..len]); } const Start = *const fn (?*anyopaque) callconv(.c) ?*anyopaque; /// libc's own, by name. const Real = struct { malloc: ?*const fn (usize) callconv(.c) ?*anyopaque, calloc: ?*const fn (usize, usize) callconv(.c) ?*anyopaque, realloc: ?*const fn (?*anyopaque, usize) callconv(.c) ?*anyopaque, free: ?*const fn (?*anyopaque) callconv(.c) void, posix_memalign: ?*const fn (*?*anyopaque, usize, usize) callconv(.c) c_int, aligned_alloc: ?*const fn (usize, usize) callconv(.c) ?*anyopaque, memalign: ?*const fn (usize, usize) callconv(.c) ?*anyopaque, pthread_create: ?*const fn (*anyopaque, ?*const anyopaque, Start, ?*anyopaque) callconv(.c) c_int, fork: ?*const fn () callconv(.c) c_int, }; var real: Real = undefined; /// 0 not looked up, 1 looking up, 2 found. var found: std.atomic.Value(u8) = .init(0); /// False while the lookup itself is running: dlsym allocates, and lands here. fn ready() bool { if (found.load(.acquire) == 2) return true; if (found.cmpxchgStrong(0, 1, .acquire, .monotonic) != null) return false; inline for (@typeInfo(Real).@"struct".field_names) |name| { const T = @FieldType(Real, name); @field(real, name) = if (preloaded) real: { const dlsym = @extern(?*const fn (?*anyopaque, [*:0]const u8) callconv(.c) ?*anyopaque, .{ .name = "dlsym", .linkage = .weak }) orelse break :real null; const RTLD_NEXT: ?*anyopaque = @ptrFromInt(std.math.maxInt(usize)); break :real @ptrCast(dlsym(RTLD_NEXT, name)); } else @extern(T, .{ .name = "__real_" ++ name, .linkage = .weak }); } found.store(2, .release); // The first call into libc that comes through us may come before the constructor. agent.init(); return true; } /// libc's own `name`. A fully static link leaves out what nothing refers /// to, and with --wrap nothing refers to libc's malloc any more: said, with /// the way out, instead of a jump to nowhere. fn libc(comptime name: []const u8) @typeInfo(@FieldType(Real, name)).optional.child { return @field(real, name) orelse { const message = "ztrace agent: this link has no " ++ name ++ " of libc's; add -Wl,-u," ++ name ++ "\n"; _ = std.os.linux.write(2, message, message.len); std.os.linux.exit_group(134); }; } /// Memory for the allocations made while the lookup runs. Never reused, so /// always zero; each block is preceded by its size. var early: [1 << 16]u8 align(16) = @splat(0); var early_used: std.atomic.Value(usize) = .init(0); fn earlyAlloc(size: usize, alignment: usize) ?*anyopaque { const step = @max(alignment, 16); const need = step + 16 + std.mem.alignForward(usize, size, 16); const at = early_used.fetchAdd(need, .monotonic); if (at + need > early.len) return null; const block = std.mem.alignForward(usize, @intFromPtr(&early) + at + 16, step); @as(*usize, @ptrFromInt(block - 16)).* = size; return @ptrFromInt(block); } fn isEarly(p: ?*anyopaque) bool { return @intFromPtr(p) -% @intFromPtr(&early) < early.len; } fn allocated(how: zt.event.Alloc, p: ?*anyopaque, size: usize, pc: usize) void { if (agent.allocations and p != null) agent.emit(.alloc, @intFromEnum(how), .{ @intFromPtr(p), size, pc, 0, 0, 0 }, ""); } fn freed(p: ?*anyopaque, pc: usize) void { if (agent.allocations) agent.emit(.free, 0, .{ @intFromPtr(p), pc, 0, 0, 0, 0 }, ""); } const interposed = struct { fn malloc(size: usize) callconv(.c) ?*anyopaque { if (!ready()) return earlyAlloc(size, 16); const p = libc("malloc")(size); allocated(.malloc, p, size, @returnAddress()); return p; } fn calloc(n: usize, size: usize) callconv(.c) ?*anyopaque { if (!ready()) return earlyAlloc(n *| size, 16); const p = libc("calloc")(n, size); allocated(.calloc, p, n *| size, @returnAddress()); return p; } fn realloc(old: ?*anyopaque, size: usize) callconv(.c) ?*anyopaque { if (!ready() or isEarly(old)) { // An early block cannot grow in place: a new one, with its bytes. const p: [*]u8 = @ptrCast((if (ready()) libc("malloc")(size) else earlyAlloc(size, 16)) orelse return null); if (old) |o| { const had = @as(*const usize, @ptrFromInt(@intFromPtr(o) - 16)).*; @memcpy(p[0..@min(had, size)], @as([*]const u8, @ptrCast(o))); } return p; } const p = libc("realloc")(old, size); // One that moved is a free and an alloc; one in place, an alloc at the same pointer. if (old != null and p != old and (p != null or size == 0)) freed(old, @returnAddress()); allocated(.realloc, p, size, @returnAddress()); return p; } fn free(p: ?*anyopaque) callconv(.c) void { if (p == null or isEarly(p) or !ready()) return; // Before it is free: once it is, another thread's alloc may be given the same pointer. freed(p, @returnAddress()); libc("free")(p); } fn posix_memalign(out: *?*anyopaque, alignment: usize, size: usize) callconv(.c) c_int { if (!ready()) { out.* = earlyAlloc(size, alignment) orelse return @intFromEnum(std.os.linux.E.NOMEM); return 0; } const rc = libc("posix_memalign")(out, alignment, size); if (rc == 0) allocated(.memalign, out.*, size, @returnAddress()); return rc; } fn aligned_alloc(alignment: usize, size: usize) callconv(.c) ?*anyopaque { if (!ready()) return earlyAlloc(size, alignment); const p = libc("aligned_alloc")(alignment, size); allocated(.memalign, p, size, @returnAddress()); return p; } fn memalign(alignment: usize, size: usize) callconv(.c) ?*anyopaque { if (!ready()) return earlyAlloc(size, alignment); const p = libc("memalign")(alignment, size); allocated(.memalign, p, size, @returnAddress()); return p; } fn pthread_create(thread: *anyopaque, attr: ?*const anyopaque, start: Start, arg: ?*anyopaque) callconv(.c) c_int { _ = ready(); if (agent.ring == null) return libc("pthread_create")(thread, attr, start, arg); // The new thread runs `threadMain`, which needs to know what it was meant to run. const box: *Box = @ptrCast(@alignCast(libc("malloc")(@sizeOf(Box)) orelse return @intFromEnum(std.os.linux.E.AGAIN))); box.* = .{ .start = start, .arg = arg }; const rc = libc("pthread_create")(thread, attr, threadMain, box); if (rc != 0) libc("free")(box); return rc; } fn fork() callconv(.c) c_int { _ = ready(); const pid = libc("fork")(); if (pid == 0 and agent.ring != null) agent.forked(); return pid; } }; const Box = struct { start: Start, arg: ?*anyopaque }; fn threadMain(arg: ?*anyopaque) callconv(.c) ?*anyopaque { const box: *Box = @ptrCast(@alignCast(arg)); const run = box.*; libc("free")(box); agent.threadStart(); agent.sud.selector = agent.sud.resting; const result = run.start(run.arg); // shortcut: a thread that ends in pthread_exit or is cancelled does not come back here; hook the exit path if a target's threads end that way. agent.emit(.thread_exit, 0, @splat(0), ""); return result; } comptime { for (@typeInfo(Real).@"struct".field_names) |name| { @export(&@field(interposed, name), .{ .name = (if (preloaded) "" else "__wrap_") ++ name }); } }