summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-10-30 02:34:25 -0300
committerGabriel Schneider <[email protected]>2026-10-30 03:13:15 -0300
commitc892cf0c3246b6c41f8f67810d360c5ea0ff66ef (patch)
treeeed27b055334a7c55ce0b9c203ea4b74b371f5e8 /src
parent44587446c2478de276b7c5c5a12063e6c7946b4e (diff)
downloadpardes-c892cf0c3246b6c41f8f67810d360c5ea0ff66ef.tar.gz
pardes-c892cf0c3246b6c41f8f67810d360c5ea0ff66ef.zip
a replica client's join costs the session's step its mutable state alone: the dump leaves the blobs out (walk.zig `Dump.deferred`: a `.blob`'s texts of 4 KiB and up, noted where they go) and its terminals' emulators (a size: the first delta brings each), the client's writer thread writes the welcome and the dump with the blobs put back in place, and each blob is pinned until written (memory.zig `pin`: its free put off); the texts the session never digested are hashed first on a thread of their own (walk.zig `unhashed`, `makeWords`), the join waiting a boundary for them
The client is unchanged: what it reads is the dump `dumpBytes` makes, byte for byte (a test splices one and compares). ReleaseFast, the session's worst boundary while three clients join one after another (`t2/join2.py`), before and after: typical (4.1 MB of files, 1 MB of scrollback, an 8.1 MB dump) 12.4 to 1.6 ms; big (38.6 MB, 8 MB, 76 MB) 51.3 to 1.5 ms; a 100 MB file open beside typical's (254 MB) 172.5 to 1.6 ms. Of the dump the step makes 113 to 122 KB. PARDES_REPLICA_STATS says each join's time and the state's size. Co-Authored-By: Claude Opus 5.5 <[email protected]>
Diffstat (limited to 'src')
-rw-r--r--src/dump.zig19
-rw-r--r--src/memory.zig79
-rw-r--r--src/replica.zig205
-rw-r--r--src/terminal.zig49
-rw-r--r--src/walk.zig55
5 files changed, 364 insertions, 43 deletions
diff --git a/src/dump.zig b/src/dump.zig
index 152495e2..c64f83c4 100644
--- a/src/dump.zig
+++ b/src/dump.zig
@@ -174,8 +174,27 @@ pub fn dumpState(p: *Pardes) !void {
/// The session as a dump, gpa-owned.
pub fn dumpBytes(p: *Pardes) ![]u8 {
+ return dumpWith(p, null);
+}
+
+/// A dump for a screen-state replica's join (src/replica.zig): its blobs'
+/// bytes left out and noted in `deferred` (they are the session's, alive
+/// while pinned: memory.zig `pin`), its terminals' emulators left out (the
+/// first delta brings each afresh). What is left is the session's mutable
+/// state, made in its step.
+pub fn dumpJoin(p: *Pardes, deferred: *std.ArrayListUnmanaged(walk.Deferred)) ![]u8 {
+ slim_terminals = true;
+ defer slim_terminals = false;
+ return dumpWith(p, deferred);
+}
+
+/// `dumpJoin`'s: a terminal dumps its size, not its emulator.
+pub var slim_terminals = false;
+
+fn dumpWith(p: *Pardes, deferred: ?*std.ArrayListUnmanaged(walk.Deferred)) ![]u8 {
var d = try Session.init(p.gpa);
errdefer d.deinit();
+ d.deferred = deferred;
try d.put(Pardes, p);
const mounts: []const filesystem.Mount = p.fs.mounts.items;
try d.put([]const filesystem.Mount, &mounts);
diff --git a/src/memory.zig b/src/memory.zig
index 7adcf914..cd29308f 100644
--- a/src/memory.zig
+++ b/src/memory.zig
@@ -422,6 +422,7 @@ pub fn freeUnfilled(allocator: Allocator, bytes: []const u8) void {
if (bytes.len == 0) return;
// A blob's digest word goes with it (walk.zig).
walk.forget(@intFromPtr(bytes.ptr));
+ if (pinned(allocator, bytes, .of(u8))) return;
allocator.rawFree(@constCast(bytes), .of(u8), @returnAddress());
}
@@ -433,9 +434,72 @@ pub fn freeRaw(allocator: Allocator, items: anytype) void {
const bytes = std.mem.sliceAsBytes(items);
if (bytes.len == 0) return;
walk.forget(@intFromPtr(bytes.ptr));
+ if (pinned(allocator, bytes, .of(E))) return;
allocator.rawFree(@constCast(bytes), .of(E), @returnAddress());
}
+// ---- a join's pins (src/replica.zig) ----
+//
+// A replica client's join dumps the session's mutable state in its step and
+// leaves its blobs out (walk.Deferred); its writer thread streams them after.
+// A blob is never written in place and is freed raw (above), so until the
+// writer has sent it, its free is put off: `pin` it, `unpin` it after.
+var pins_lock: std.atomic.Value(bool) = .init(false);
+var pins: std.AutoHashMapUnmanaged(usize, u32) = .empty;
+var pin_count: std.atomic.Value(usize) = .init(0);
+const PutOff = struct { allocator: Allocator, bytes: []const u8, alignment: std.mem.Alignment };
+var put_off: std.ArrayListUnmanaged(PutOff) = .empty;
+
+fn lockPins() void {
+ while (pins_lock.cmpxchgWeak(false, true, .acquire, .monotonic) != null) std.atomic.spinLoopHint();
+}
+
+/// The blob at `bytes.ptr` kept alive until it is unpinned as many times.
+pub fn pin(bytes: []const u8) void {
+ lockPins();
+ defer pins_lock.store(false, .release);
+ const got = pins.getOrPut(std.heap.page_allocator, @intFromPtr(bytes.ptr)) catch return;
+ got.value_ptr.* = if (got.found_existing) got.value_ptr.* + 1 else 1;
+ pin_count.store(pins.count(), .release);
+}
+
+/// One pin of the blob at `bytes.ptr` let go: its free, put off, made now
+/// the last one goes.
+pub fn unpin(bytes: []const u8) void {
+ const addr = @intFromPtr(bytes.ptr);
+ lockPins();
+ const n = pins.getPtr(addr) orelse {
+ pins_lock.store(false, .release);
+ return;
+ };
+ n.* -= 1;
+ var due: ?PutOff = null;
+ if (n.* == 0) {
+ _ = pins.remove(addr);
+ pin_count.store(pins.count(), .release);
+ for (put_off.items, 0..) |f, i| if (@intFromPtr(f.bytes.ptr) == addr) {
+ due = put_off.swapRemove(i);
+ break;
+ };
+ }
+ pins_lock.store(false, .release);
+ if (due) |f| {
+ // A word made for it while it was pinned (walk.makeWords) goes too.
+ walk.forget(addr);
+ f.allocator.rawFree(@constCast(f.bytes), f.alignment, @returnAddress());
+ }
+}
+
+/// Whether the free of `bytes` is put off (it is pinned).
+fn pinned(allocator: Allocator, bytes: []const u8, alignment: std.mem.Alignment) bool {
+ if (pin_count.load(.acquire) == 0) return false;
+ lockPins();
+ defer pins_lock.store(false, .release);
+ if (!pins.contains(@intFromPtr(bytes.ptr))) return false;
+ put_off.append(std.heap.page_allocator, .{ .allocator = allocator, .bytes = bytes, .alignment = alignment }) catch return false;
+ return true;
+}
+
/// Gives free memory back to the system, the arenas' and libc's, inside its
/// heap too: after a big result is formatted or a big pane torn down, the
/// tree-sitter arena kept 73 MiB of emptied buckets resident and glibc what
@@ -568,3 +632,18 @@ test "memory limits preserve desktop capacities" {
try std.testing.expectEqual(256 * MiB, limits.arena.image);
try std.testing.expectEqual(if (config.mupdf) 512 * MiB else 64 * KiB, limits.arena.pdf);
}
+
+test "a pinned blob's free is put off until its last pin goes" {
+ const gpa = std.testing.allocator;
+ const b = try gpa.alloc(u8, 8192);
+ @memset(b, 7);
+ pin(b);
+ pin(b);
+ freeUnfilled(gpa, b);
+ // Still the blob's: a join's writer reads it.
+ try std.testing.expectEqual(@as(u8, 7), b[100]);
+ unpin(b);
+ try std.testing.expectEqual(@as(u8, 7), b[8000]);
+ // The last pin: freed now (the testing allocator says if not).
+ unpin(b);
+}
diff --git a/src/replica.zig b/src/replica.zig
index 8d8d47c5..2e14cb6f 100644
--- a/src/replica.zig
+++ b/src/replica.zig
@@ -1071,6 +1071,17 @@ fn writeAll(fd: c_int, bytes: []const u8) bool {
return true;
}
+/// `bytes` as dump pieces (1 MiB at most each).
+fn writePieces(fd: c_int, bytes: []const u8) bool {
+ var at: usize = 0;
+ while (at < bytes.len) : (at += dump_piece) {
+ const piece = bytes[at..@min(bytes.len, at + dump_piece)];
+ const head = wire.header(@backingInt(wire.ServerTag.dump), piece.len);
+ if (!writeAll(fd, &head) or !writeAll(fd, piece)) return false;
+ }
+ return true;
+}
+
fn readSome(fd: c_int, buf: []u8) usize {
while (true) {
const n = libc.read(fd, buf.ptr, buf.len);
@@ -1144,6 +1155,24 @@ pub const Peer = struct {
cut: bool = false,
/// Its writer has ended (what it was sent, written).
done: std.atomic.Value(bool) = .init(false),
+ /// Its join, which its writer writes before what it is sent: the
+ /// welcome, then the dump with the session's blobs put back where they
+ /// go, pinned until written (memory.zig `pin`). The core's allocator's.
+ welcome: [wire.hello_max + 64]u8 = undefined,
+ welcome_len: usize = 0,
+ dump: []u8 = &.{},
+ blobs: std.ArrayListUnmanaged(walk.Deferred) = .empty,
+ dump_gpa: Allocator = undefined,
+
+ fn writeJoin(peer: *Peer) bool {
+ if (!writeAll(peer.fd, peer.welcome[0..peer.welcome_len])) return false;
+ var pos: usize = 0;
+ for (peer.blobs.items) |b| {
+ if (!writePieces(peer.fd, peer.dump[pos..b.at]) or !writePieces(peer.fd, b.bytes)) return false;
+ pos = b.at;
+ }
+ return writePieces(peer.fd, peer.dump[pos..]);
+ }
/// `parts` queued for it, in order; never waits. A client whose queue
/// passes its limit is cut off: its link is shut down, and its reader
@@ -1174,6 +1203,12 @@ pub const Peer = struct {
fn writeLoop(peer: *Peer) void {
const io = peer.peers.io;
defer peer.done.store(true, .release);
+ if (!peer.writeJoin()) {
+ _ = libc.shutdown(peer.fd, 2);
+ return;
+ }
+ // Its pins let go at the next boundary.
+ peer.peers.post(.{ .joined = peer });
while (true) {
peer.lock.lockUncancelable(io);
while (peer.out.items.len == 0 and !peer.closing and !peer.cut) peer.cond.waitUncancelable(io, &peer.lock);
@@ -1261,7 +1296,10 @@ pub const Peers = struct {
alone_unattended: bool = false,
/// The worst a join cost the session's step, and the last one's dump.
join_ns: u64 = 0,
+ each_join: std.ArrayListUnmanaged(u64) = .empty,
dump_bytes: usize = 0,
+ /// ...of it the state the step dumped (the rest, blobs, the writer's).
+ state_bytes: usize = 0,
/// What each client input cost the session (its step, its unit sent),
/// and each boundary (its pump's unit sent, what came taken in), ns:
/// PARDES_REPLICA_STATS writes them at the session's end.
@@ -1273,8 +1311,14 @@ pub const Peers = struct {
/// it records: what each was sent.
screens: Screens = undefined,
msgs: std.ArrayListUnmanaged([]u8) = .empty,
+ /// Clients waiting to join while the session's texts are hashed
+ /// (`joins`), the texts and the thread hashing them.
+ held: std.ArrayListUnmanaged(c_int) = .empty,
+ texts: std.ArrayListUnmanaged([]const u8) = .empty,
+ hasher: ?std.Thread = null,
+ rounds: u8 = 0,
- pub const Note = union(enum) { join: c_int, input: Msg, screen: Msg, gone: *Peer, check: *Peer };
+ pub const Note = union(enum) { join: c_int, input: Msg, screen: Msg, gone: *Peer, check: *Peer, joined: *Peer, hashed };
pub const Msg = struct { peer: *Peer, msg: []u8 };
pub fn count(ps: *const Peers) usize {
@@ -1290,7 +1334,7 @@ pub const Peers = struct {
switch (note) {
.join => |fd| _ = libc.close(fd),
.input, .screen => |in| ps.gpa.free(in.msg),
- .gone, .check => {},
+ .gone, .check, .joined, .hashed => {},
}
return;
};
@@ -1344,7 +1388,17 @@ pub const Peers = struct {
ps.mail.store(false, .release);
ps.lock.unlock(ps.io);
for (ps.taken.items) |note| switch (note) {
- .join => |fd| ps.join(core, fd),
+ .join => |fd| {
+ ps.held.append(ps.gpa, fd) catch {
+ refuse(fd, "out of memory");
+ continue;
+ };
+ ps.joins(core);
+ },
+ .hashed => {
+ ps.unhashedDone();
+ ps.joins(core);
+ },
.input => |in| {
ps.input(core, in.peer, in.msg);
ps.gpa.free(in.msg);
@@ -1357,6 +1411,7 @@ pub const Peers = struct {
in.peer.send(&.{ &head, reply });
},
.gone => |peer| ps.leave(core, peer),
+ .joined => |peer| released(peer),
.check => |peer| {
var out: [16]u8 = undefined;
peer.send(&.{wire.encodeServer(&out, .{ .check = tape.digestOf(core) }) catch unreachable});
@@ -1410,9 +1465,48 @@ pub const Peers = struct {
_ = libc.close(fd);
}
+ /// The clients waiting to join, joined, once the texts the session
+ /// never digested have their words (`walk.unhashed`): those are hashed
+ /// on a thread of their own, pinned meanwhile, and the joins wait a
+ /// boundary or two for them, the session's step never. A second round
+ /// joins whatever is left.
+ fn joins(ps: *Peers, core: *Pardes) void {
+ if (ps.hasher != null or ps.held.items.len == 0) return;
+ if (ps.rounds < 2) {
+ walk.unhashed(Pardes, core, &ps.texts);
+ if (ps.texts.items.len > 0) {
+ for (ps.texts.items) |b| pardes.memory.pin(b);
+ if (std.Thread.spawn(.{}, hashTexts, .{ps})) |t| {
+ ps.hasher = t;
+ ps.rounds += 1;
+ return;
+ } else |_| ps.unhashedDone();
+ }
+ }
+ ps.rounds = 0;
+ for (ps.held.items) |fd| ps.join(core, fd);
+ ps.held.clearRetainingCapacity();
+ }
+
+ fn hashTexts(ps: *Peers) void {
+ walk.makeWords(ps.texts.items);
+ ps.post(.hashed);
+ }
+
+ /// The hashing thread joined, its texts unpinned.
+ fn unhashedDone(ps: *Peers) void {
+ if (ps.hasher) |t| t.join();
+ ps.hasher = null;
+ for (ps.texts.items) |b| pardes.memory.unpin(b);
+ ps.texts.clearRetainingCapacity();
+ }
+
fn join(ps: *Peers, core: *Pardes, fd: c_int) void {
const t0 = nowNs();
- defer ps.join_ns = @max(ps.join_ns, nowNs() - t0);
+ defer {
+ ps.join_ns = @max(ps.join_ns, nowNs() - t0);
+ ps.each_join.append(ps.gpa, nowNs() - t0) catch {};
+ }
const slot = for (&ps.list) |*s| {
if (s.* == null) break s;
} else return refuse(fd, "that session has as many clients as it takes");
@@ -1440,7 +1534,12 @@ pub const Peers = struct {
core.undrawn |= @as(u8, 1) << @intCast(k);
ps.seq += 1;
ps.unit(0, ps.seq);
- const bytes = pardes.dump.dumpBytes(core) catch |err| {
+ // The session's mutable state, dumped now; its blobs (file texts,
+ // undo, registers) left out, pinned, and streamed by the client's
+ // writer thread: the step pays for the state, not for the text.
+ var blobs: std.ArrayListUnmanaged(walk.Deferred) = .empty;
+ const bytes = pardes.dump.dumpJoin(core, &blobs) catch |err| {
+ blobs.deinit(core.gpa);
core.undrawn &= ~(@as(u8, 1) << @intCast(k));
core.closeView(k);
ps.unit(0, ps.seq);
@@ -1448,28 +1547,29 @@ pub const Peers = struct {
var why: [96]u8 = undefined;
return refuse(fd, std.fmt.bufPrint(&why, "that session could not be dumped: {t}", .{err}) catch "that session could not be dumped");
};
- // The core's, as a Dump's is.
- defer core.gpa.free(bytes);
- ps.dump_bytes = bytes.len;
+ var whole = bytes.len;
+ for (blobs.items) |b| {
+ pardes.memory.pin(b.bytes);
+ whole += b.bytes.len;
+ }
+ ps.dump_bytes = whole;
+ ps.state_bytes = bytes.len;
// The joiner's core is the dump's: the host this tape wraps, and
// what the host wrote in the core, are said again at the next input.
tape.resync(&ps.t);
- const peer = ps.gpa.create(Peer) catch return refuse(fd, "out of memory");
+ const peer = ps.gpa.create(Peer) catch {
+ for (blobs.items) |b| pardes.memory.unpin(b.bytes);
+ blobs.deinit(core.gpa);
+ core.gpa.free(bytes);
+ return refuse(fd, "out of memory");
+ };
if (std.c.getenv("PARDES_REPLICA_LAG_KB")) |kb| lag_max = (std.fmt.parseInt(usize, std.mem.span(kb), 10) catch 64 << 10) << 10;
- peer.* = .{ .peers = ps, .fd = fd, .id = ps.next_id, .view = k, .limit = bytes.len + lag_max };
+ peer.* = .{ .peers = ps, .fd = fd, .id = ps.next_id, .view = k, .limit = lag_max, .dump = bytes, .blobs = blobs, .dump_gpa = core.gpa };
ps.next_id +%= 1;
if (ps.next_id == 0) ps.next_id = 1;
- var out: [wire.hello_max + 64]u8 = undefined;
// The session's digest as dumped, for the joiner to check its load by.
const digest = tape.digestOf(core);
- const welcome = wire.encodeServer(&out, .{ .welcome = .{ .pid = @intCast(libc.getpid()), .session = ps.name[0..@min(ps.name.len, wire.max_name)], .client = peer.id, .view = k, .dump_len = bytes.len, .gmtoff = gmtoff(), .digest = digest, .semantics = wire.semantics } }) catch unreachable;
- peer.send(&.{welcome});
- var at: usize = 0;
- while (at < bytes.len) : (at += dump_piece) {
- const piece = bytes[at..@min(bytes.len, at + dump_piece)];
- const head = wire.header(@backingInt(wire.ServerTag.dump), piece.len);
- peer.send(&.{ &head, piece });
- }
+ peer.welcome_len = (wire.encodeServer(&peer.welcome, .{ .welcome = .{ .pid = @intCast(libc.getpid()), .session = ps.name[0..@min(ps.name.len, wire.max_name)], .client = peer.id, .view = k, .dump_len = whole, .gmtoff = gmtoff(), .digest = digest, .semantics = wire.semantics } }) catch unreachable).len;
peer.writer = std.Thread.spawn(.{}, Peer.writeLoop, .{peer}) catch null;
peer.reader = std.Thread.spawn(.{}, Peer.readLoop, .{peer}) catch null;
if (peer.writer == null or peer.reader == null) {
@@ -1485,7 +1585,7 @@ pub const Peers = struct {
peer.sid = sid;
ps.screensTo(core, peer, .flush);
} else |_| _ = libc.shutdown(fd, 2);
- log.info("client {d} joined '{s}' as view {d}: a {d} KB dump in {d} us", .{ peer.id, ps.name, k, bytes.len >> 10, (nowNs() - t0) / 1000 });
+ log.info("client {d} joined '{s}' as view {d}: a {d} KB dump ({d} KB of state) in {d} us", .{ peer.id, ps.name, k, whole >> 10, bytes.len >> 10, (nowNs() - t0) / 1000 });
if (std.c.getenv("PARDES_REPLICA_TRACE") != null) std.debug.print("session: client {d} joined after unit {d}\n", .{ peer.id, ps.seq });
}
@@ -1514,6 +1614,7 @@ pub const Peers = struct {
_ = libc.shutdown(peer.fd, 2);
if (peer.writer) |w| w.join();
if (peer.reader) |r| r.join();
+ released(peer);
_ = libc.close(peer.fd);
peer.out.deinit(ps.gpa);
peer.sending.deinit(ps.gpa);
@@ -1523,8 +1624,8 @@ pub const Peers = struct {
while (i < ps.inbox.items.len) {
const mine = switch (ps.inbox.items[i]) {
.input, .screen => |in| in.peer == peer,
- .gone, .check => |p| p == peer,
- .join => false,
+ .gone, .check, .joined => |p| p == peer,
+ .join, .hashed => false,
};
if (!mine) {
i += 1;
@@ -1567,6 +1668,15 @@ pub const Peers = struct {
ps.dropAll(core);
}
+ /// A join's dump and pins let go: written, or its client gone.
+ fn released(peer: *Peer) void {
+ for (peer.blobs.items) |b| pardes.memory.unpin(b.bytes);
+ peer.blobs.deinit(peer.dump_gpa);
+ peer.blobs = .empty;
+ if (peer.dump.len > 0) peer.dump_gpa.free(peer.dump);
+ peer.dump = &.{};
+ }
+
/// Detach typed in client view `k`'s: its link ends (its reader says
/// it is gone, and the next boundary lets it go).
pub fn detach(ps: *Peers, k: u8) bool {
@@ -1589,6 +1699,10 @@ pub const Peers = struct {
/// Every client let go (the session's end, a Restore): each sees its
/// link end.
pub fn dropAll(ps: *Peers, core: *Pardes) void {
+ ps.unhashedDone();
+ for (ps.held.items) |fd| _ = libc.close(fd);
+ ps.held.clearRetainingCapacity();
+ ps.rounds = 0;
for (&ps.list) |*slot| if (slot.*) |peer| {
slot.* = null;
core.undrawn &= ~(@as(u8, 1) << @intCast(peer.view));
@@ -1601,7 +1715,7 @@ pub const Peers = struct {
for (ps.inbox.items) |note| switch (note) {
.join => |fd| _ = libc.close(fd),
.input, .screen => |in| ps.gpa.free(in.msg),
- .gone, .check => {},
+ .gone, .check, .joined, .hashed => {},
};
ps.inbox.clearRetainingCapacity();
ps.lock.unlock(ps.io);
@@ -1615,20 +1729,29 @@ pub const Peers = struct {
ps.input_ns.deinit(ps.gpa);
ps.boundary_ns.deinit(ps.gpa);
ps.msgs.deinit(ps.gpa);
+ ps.held.deinit(ps.gpa);
+ ps.texts.deinit(std.heap.c_allocator);
+ ps.each_join.deinit(ps.gpa);
}
fn writeStats(ps: *Peers, path: []const u8) void {
var buf: [400]u8 = undefined;
- const line = std.fmt.bufPrint(&buf, "session inputs {d} input_us {d}/{d}/{d} boundaries {d} boundary_us {d}/{d}/{d} join_us {d} dump_kb {d} cut {d}\n", .{
+ const line = std.fmt.bufPrint(&buf, "session inputs {d} input_us {d}/{d}/{d} boundaries {d} boundary_us {d}/{d}/{d} join_us {d} dump_kb {d} state_kb {d} cut {d}\n", .{
ps.input_ns.items.len, pctOf(ps.input_ns.items, 50) / 1000, pctOf(ps.input_ns.items, 99) / 1000, pctOf(ps.input_ns.items, 100) / 1000,
ps.boundary_ns.items.len, pctOf(ps.boundary_ns.items, 50) / 1000, pctOf(ps.boundary_ns.items, 99) / 1000, pctOf(ps.boundary_ns.items, 100) / 1000,
- ps.join_ns / 1000, ps.dump_bytes >> 10, ps.cut,
+ ps.join_ns / 1000, ps.dump_bytes >> 10, ps.state_bytes >> 10, ps.cut,
}) catch return;
var z: [4096]u8 = undefined;
const pz = std.fmt.bufPrintSentinel(&z, "{s}", .{path}, 0) catch return;
const fd = libc.open(pz, .{ .ACCMODE = .WRONLY, .CREAT = true, .APPEND = true, .CLOEXEC = true }, @as(libc.mode_t, 0o644));
if (fd < 0) return;
_ = libc.write(fd, line.ptr, line.len);
+ var each: [400]u8 = undefined;
+ var w: std.Io.Writer = .fixed(&each);
+ w.writeAll("session joins_us") catch {};
+ for (ps.each_join.items) |ns| w.print(" {d}", .{ns / 1000}) catch {};
+ w.writeAll("\n") catch {};
+ _ = libc.write(fd, w.buffered().ptr, w.buffered().len);
_ = libc.close(fd);
}
};
@@ -2052,3 +2175,35 @@ test "a replica joined from the session's dump replays the units after it" {
try std.testing.expect(r.quiet());
try std.testing.expectEqual(digestOf(a), r.digest());
}
+
+test "a join's dump, its blobs put back where they go, is the session's dump" {
+ const gpa = std.testing.allocator;
+ const a = try Authority.init(gpa, .{ .cols = 80, .rows = 24, .file = "mise.toml" }, TestHost.host(), false);
+ defer a.deinit();
+ const text = try gpa.alloc(u8, 20000);
+ defer gpa.free(text);
+ @memset(text, 'x');
+ _ = try a.local(.{ .key = .{ .cp = 'i', .text = "i" } });
+ _ = try a.local(.{ .paste = text });
+ _ = try a.local(.{ .key = .{ .cp = pardes.Key.escape, .text = "" } });
+ const was = Stepping.begin(&a.t, null);
+ defer was.end();
+ const whole = try pardes.dump.dumpBytes(a.core);
+ defer a.core.gpa.free(whole);
+ var blobs: std.ArrayListUnmanaged(walk.Deferred) = .empty;
+ defer blobs.deinit(a.core.gpa);
+ const state = try pardes.dump.dumpJoin(a.core, &blobs);
+ defer a.core.gpa.free(state);
+ try std.testing.expect(blobs.items.len > 0);
+ try std.testing.expect(state.len + 20000 <= whole.len);
+ var joined: std.ArrayListUnmanaged(u8) = .empty;
+ defer joined.deinit(gpa);
+ var pos: usize = 0;
+ for (blobs.items) |b| {
+ try joined.appendSlice(gpa, state[pos..b.at]);
+ try joined.appendSlice(gpa, b.bytes);
+ pos = b.at;
+ }
+ try joined.appendSlice(gpa, state[pos..]);
+ try std.testing.expectEqualSlices(u8, whole, joined.items);
+}
diff --git a/src/terminal.zig b/src/terminal.zig
index 2bc96a6e..42268f0a 100644
--- a/src/terminal.zig
+++ b/src/terminal.zig
@@ -259,6 +259,9 @@ pub const State = struct {
defer gpa.free(bytes);
try w.writeAll(std.mem.asBytes(&@as(u64, bytes.len)));
try w.writeAll(bytes);
+ // A join's (dump.zig `dumpJoin`): its size alone, an emulator made
+ // afresh there, which its first delta replaces.
+ if (pardes.dump.slim_terminals) return w.writeAll(std.mem.asBytes(&[2]u16{ s.vt.cols, s.vt.rows }));
try ghostty_vt.snapshot.encode(gpa, w, &s.vt, .{ .continuation = .ground });
}
@@ -283,12 +286,17 @@ pub const State = struct {
defer arena.deinit();
const parts = try pardes.walk.load(Parts, arena.allocator(), bytes[8..][0..n], null);
var reader: std.Io.Reader = .fixed(bytes[8 + n ..]);
- var decoded = try ghostty_vt.snapshot.decodeExact(gpa, terminalIo(), &reader, .{
- .max_continuation_bytes = 0,
- .kitty_image_storage_limit = if (images_enabled) TerminalImages.storage_limit else 0,
- });
- defer decoded.deinit(gpa);
- out.* = .{ .gpa = gpa, .vt = decoded.toOwned(), .stream = undefined };
+ if (reader.buffer.len == 4) {
+ const size = std.mem.bytesToValue([2]u16, bytes[8 + n ..][0..4]);
+ out.* = .{ .gpa = gpa, .vt = try emulator(gpa, @max(1, size[0]), @max(1, size[1])), .stream = undefined };
+ } else {
+ var decoded = try ghostty_vt.snapshot.decodeExact(gpa, terminalIo(), &reader, .{
+ .max_continuation_bytes = 0,
+ .kitty_image_storage_limit = if (images_enabled) TerminalImages.storage_limit else 0,
+ });
+ defer decoded.deinit(gpa);
+ out.* = .{ .gpa = gpa, .vt = decoded.toOwned(), .stream = undefined };
+ }
if (comptime images_enabled) inline for (.{ .primary, .alternate }) |key| {
if (out.vt.screens.get(key)) |screen| screen.kitty_images.image_limits = .allWithTempDir(tempDir());
};
@@ -846,18 +854,9 @@ pub fn deinitEmulator(pane: *Pane, gpa: std.mem.Allocator) void {
pane.terminal = null;
}
-/// Allocate the live emulator half of a terminal pane. Slot ownership, serial
-/// assignment, and spawn effects remain core lifecycle invariants.
-pub fn create(gpa: std.mem.Allocator, cols: u16, rows: u16) !*Pane {
- if (authorityOnly()) return error.OutOfMemory;
- const pane = try createDoc(gpa, cols, rows);
- errdefer gpa.destroy(pane);
- if (comptime !enabled) return pane;
- const own = if (pardes.tape.screenReplica()) emulator_gpa else gpa;
- const state = try own.create(State);
- errdefer own.destroy(state);
- state.gpa = own;
- state.vt = try ghostty_vt.Terminal.init(terminalIo(), own, .{
+/// A terminal pane's emulator, `cols` by `rows`, as `create` makes one.
+fn emulator(gpa: std.mem.Allocator, cols: u16, rows: u16) !ghostty_vt.Terminal {
+ return ghostty_vt.Terminal.init(terminalIo(), gpa, .{
.cols = cols,
.rows = rows,
.max_scrollback_bytes = 16 * 1024 * 1024,
@@ -870,6 +869,20 @@ pub fn create(gpa: std.mem.Allocator, cols: u16, rows: u16) !*Pane {
// it could send itself: it runs as the user, under pardes.
.kitty_image_loading_limits = if (images_enabled) .allWithTempDir(tempDir()) else if (@FieldType(ghostty_vt.Terminal.Options, "kitty_image_loading_limits") == void) {} else .direct,
});
+}
+
+/// Allocate the live emulator half of a terminal pane. Slot ownership, serial
+/// assignment, and spawn effects remain core lifecycle invariants.
+pub fn create(gpa: std.mem.Allocator, cols: u16, rows: u16) !*Pane {
+ if (authorityOnly()) return error.OutOfMemory;
+ const pane = try createDoc(gpa, cols, rows);
+ errdefer gpa.destroy(pane);
+ if (comptime !enabled) return pane;
+ const own = if (pardes.tape.screenReplica()) emulator_gpa else gpa;
+ const state = try own.create(State);
+ errdefer own.destroy(state);
+ state.gpa = own;
+ state.vt = try emulator(own, cols, rows);
state.stream = .init(.{ .handler = .{ .inner = state.vt.vtHandler() }, .allocator = own });
state.reply_len = 0;
state.changes = 0;
diff --git a/src/walk.zig b/src/walk.zig
index 2dc0a721..d0d20e8a 100644
--- a/src/walk.zig
+++ b/src/walk.zig
@@ -112,6 +112,10 @@ fn hashBlob(comptime E: type, items: []const E, h: *Hasher) void {
break :kept if (b.len == items.len) b.word else null;
} else null;
if (!rehash) if (kept) |w| break :word w;
+ if (E == u8) if (unhashed_out) |list| if (kept == null) {
+ list.append(std.heap.c_allocator, items) catch {};
+ break :word 0;
+ };
in_blob = false;
defer in_blob = true;
var one = Hasher.init(0);
@@ -130,6 +134,37 @@ fn hashBlob(comptime E: type, items: []const E, h: *Hasher) void {
h.update(std.mem.asBytes(&word));
}
+/// Where `unhashed` notes the texts a digest would hash for want of a word.
+threadlocal var unhashed_out: ?*std.ArrayListUnmanaged([]const u8) = null;
+
+/// The blob texts a digest of `v` would hash now, having no word yet, onto
+/// `out` (the C heap's): a walk with no blob hashed, which `makeWords` then
+/// makes on another thread. A replica's join (src/replica.zig) does that
+/// first: the session's step never hashes a text it never digested before.
+pub fn unhashed(comptime T: type, v: *const T, out: *std.ArrayListUnmanaged([]const u8)) void {
+ unhashed_out = out;
+ defer unhashed_out = null;
+ _ = digest(T, v);
+}
+
+/// The words of `texts` (`unhashed`'s), made on any thread: they must stay
+/// as they are meanwhile (pinned, memory.zig), and the next digest takes
+/// them.
+pub fn makeWords(texts: []const []const u8) void {
+ for (texts) |items| {
+ // As hashBlob makes it (hashSlice's length, then the bytes).
+ var one = Hasher.init(0);
+ one.update(std.mem.asBytes(&items.len));
+ one.update(items);
+ const w = one.final();
+ lockBlobs();
+ defer unlockBlobs();
+ const got = blob_words.getOrPut(std.heap.page_allocator, @intFromPtr(items.ptr)) catch continue;
+ if (!got.found_existing) got.value_ptr.* = .{ .len = items.len, .word = w };
+ blob_count.store(blob_words.count(), .release);
+ }
+}
+
pub fn digest(comptime T: type, v: *const T) u64 {
var h = Hasher.init(0);
hash(T, v, &h);
@@ -801,6 +836,9 @@ fn tableOf(comptime types: []const type) []const u8 {
return Once.bytes;
}
+/// A blob's bytes a dump left out (`Dump.deferred`): they go at `at`.
+pub const Deferred = struct { at: usize, bytes: []const u8 };
+
/// Writes a dump of `roots`' values, one `put` each in order.
pub fn Dump(comptime roots: []const type) type {
return struct {
@@ -811,6 +849,13 @@ pub fn Dump(comptime roots: []const type) type {
out: std.Io.Writer.Allocating,
memo: std.AutoHashMapUnmanaged(Key, u32) = .empty,
shared: u32 = 0,
+ /// Set: a `.blob`'s bytes (of `blob_min` and up) are not written but
+ /// noted, where they go and what they are; the dump with them put
+ /// back in place is the dump. A replica's join streams them from
+ /// its writer thread (src/replica.zig), the session's step paying
+ /// for the rest alone.
+ deferred: ?*std.ArrayListUnmanaged(Deferred) = null,
+ in_blob: bool = false,
const Key = struct { addr: usize, len: usize, ty: u32 };
@@ -893,6 +938,13 @@ pub fn Dump(comptime roots: []const type) type {
.slice => {
const E = @typeInfo(T).pointer.child;
if (try d.seen(T, @intFromPtr(v.ptr), v.len)) return;
+ if (comptime flat(E)) if (d.in_blob) if (d.deferred) |later| {
+ const bytes = std.mem.sliceAsBytes(v.*);
+ if (bytes.len >= blob_min) {
+ try w.writeAll(std.mem.asBytes(&@as(u64, v.len)));
+ return later.append(d.gpa, .{ .at = d.out.writer.end, .bytes = bytes });
+ }
+ };
try d.items(E, v.*);
},
.array => {
@@ -905,6 +957,9 @@ pub fn Dump(comptime roots: []const type) type {
const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed";
inline for (comptime dumpFields(T)) |name| {
const F = @FieldType(T, name);
+ const was = d.in_blob;
+ d.in_blob = was or comptime blob(T, name);
+ defer d.in_blob = was;
if (comptime cutArray(T, name)) {
const all = &@field(v, name);
try d.items(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))]);