From c892cf0c3246b6c41f8f67810d360c5ea0ff66ef Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 02:34:25 -0300 Subject: a replica client's join costs the session's step its mutable state alone: the dump leaves the blobs out (walk.zig `Dump.deferred`: a `.blob`'s texts of 4 KiB and up, noted where they go) and its terminals' emulators (a size: the first delta brings each), the client's writer thread writes the welcome and the dump with the blobs put back in place, and each blob is pinned until written (memory.zig `pin`: its free put off); the texts the session never digested are hashed first on a thread of their own (walk.zig `unhashed`, `makeWords`), the join waiting a boundary for them The client is unchanged: what it reads is the dump `dumpBytes` makes, byte for byte (a test splices one and compares). ReleaseFast, the session's worst boundary while three clients join one after another (`t2/join2.py`), before and after: typical (4.1 MB of files, 1 MB of scrollback, an 8.1 MB dump) 12.4 to 1.6 ms; big (38.6 MB, 8 MB, 76 MB) 51.3 to 1.5 ms; a 100 MB file open beside typical's (254 MB) 172.5 to 1.6 ms. Of the dump the step makes 113 to 122 KB. PARDES_REPLICA_STATS says each join's time and the state's size. Co-Authored-By: Claude Opus 5.5 --- src/walk.zig | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'src/walk.zig') diff --git a/src/walk.zig b/src/walk.zig index 2dc0a721..d0d20e8a 100644 --- a/src/walk.zig +++ b/src/walk.zig @@ -112,6 +112,10 @@ fn hashBlob(comptime E: type, items: []const E, h: *Hasher) void { break :kept if (b.len == items.len) b.word else null; } else null; if (!rehash) if (kept) |w| break :word w; + if (E == u8) if (unhashed_out) |list| if (kept == null) { + list.append(std.heap.c_allocator, items) catch {}; + break :word 0; + }; in_blob = false; defer in_blob = true; var one = Hasher.init(0); @@ -130,6 +134,37 @@ fn hashBlob(comptime E: type, items: []const E, h: *Hasher) void { h.update(std.mem.asBytes(&word)); } +/// Where `unhashed` notes the texts a digest would hash for want of a word. +threadlocal var unhashed_out: ?*std.ArrayListUnmanaged([]const u8) = null; + +/// The blob texts a digest of `v` would hash now, having no word yet, onto +/// `out` (the C heap's): a walk with no blob hashed, which `makeWords` then +/// makes on another thread. A replica's join (src/replica.zig) does that +/// first: the session's step never hashes a text it never digested before. +pub fn unhashed(comptime T: type, v: *const T, out: *std.ArrayListUnmanaged([]const u8)) void { + unhashed_out = out; + defer unhashed_out = null; + _ = digest(T, v); +} + +/// The words of `texts` (`unhashed`'s), made on any thread: they must stay +/// as they are meanwhile (pinned, memory.zig), and the next digest takes +/// them. +pub fn makeWords(texts: []const []const u8) void { + for (texts) |items| { + // As hashBlob makes it (hashSlice's length, then the bytes). + var one = Hasher.init(0); + one.update(std.mem.asBytes(&items.len)); + one.update(items); + const w = one.final(); + lockBlobs(); + defer unlockBlobs(); + const got = blob_words.getOrPut(std.heap.page_allocator, @intFromPtr(items.ptr)) catch continue; + if (!got.found_existing) got.value_ptr.* = .{ .len = items.len, .word = w }; + blob_count.store(blob_words.count(), .release); + } +} + pub fn digest(comptime T: type, v: *const T) u64 { var h = Hasher.init(0); hash(T, v, &h); @@ -801,6 +836,9 @@ fn tableOf(comptime types: []const type) []const u8 { return Once.bytes; } +/// A blob's bytes a dump left out (`Dump.deferred`): they go at `at`. +pub const Deferred = struct { at: usize, bytes: []const u8 }; + /// Writes a dump of `roots`' values, one `put` each in order. pub fn Dump(comptime roots: []const type) type { return struct { @@ -811,6 +849,13 @@ pub fn Dump(comptime roots: []const type) type { out: std.Io.Writer.Allocating, memo: std.AutoHashMapUnmanaged(Key, u32) = .empty, shared: u32 = 0, + /// Set: a `.blob`'s bytes (of `blob_min` and up) are not written but + /// noted, where they go and what they are; the dump with them put + /// back in place is the dump. A replica's join streams them from + /// its writer thread (src/replica.zig), the session's step paying + /// for the rest alone. + deferred: ?*std.ArrayListUnmanaged(Deferred) = null, + in_blob: bool = false, const Key = struct { addr: usize, len: usize, ty: u32 }; @@ -893,6 +938,13 @@ pub fn Dump(comptime roots: []const type) type { .slice => { const E = @typeInfo(T).pointer.child; if (try d.seen(T, @intFromPtr(v.ptr), v.len)) return; + if (comptime flat(E)) if (d.in_blob) if (d.deferred) |later| { + const bytes = std.mem.sliceAsBytes(v.*); + if (bytes.len >= blob_min) { + try w.writeAll(std.mem.asBytes(&@as(u64, v.len))); + return later.append(d.gpa, .{ .at = d.out.writer.end, .bytes = bytes }); + } + }; try d.items(E, v.*); }, .array => { @@ -905,6 +957,9 @@ pub fn Dump(comptime roots: []const type) type { const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed"; inline for (comptime dumpFields(T)) |name| { const F = @FieldType(T, name); + const was = d.in_blob; + d.in_blob = was or comptime blob(T, name); + defer d.in_blob = was; if (comptime cutArray(T, name)) { const all = &@field(v, name); try d.items(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))]); -- cgit v1.3