summaryrefslogtreecommitdiff
path: root/src/walk.zig
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-11-05 14:42:45 -0300
committerGabriel Schneider <[email protected]>2026-11-05 19:16:37 -0300
commit15a1b94eaa73747afaa299262a2d6e4fd6af42fd (patch)
tree46bff4f35b642521eead62bc3b5657902e5fee0b /src/walk.zig
parent4e096f6dc6db043caa630514fa75bd18438cb15b (diff)
downloadpardes-15a1b94eaa73747afaa299262a2d6e4fd6af42fd.tar.gz
pardes-15a1b94eaa73747afaa299262a2d6e4fd6af42fd.zip
dumps: a Dump is a labelled entry of the state file (src/Db.zig, architecture §14.3), one fsynced transaction: `dumps/<label>` holds the session as a database of its own, which the walk writes by field name into xitdb's maps (walk.Put: a struct a map keyed by its fields' names, a union a map of its one field, records a list, plain numbers one bytes value tagged with their kind and width, an enum its name, a number a uint or an int, an optional an absent key, a type with dumpWrite its bytes, a .blob text of 4 KiB and up its BLAKE3 hash) and another build reads by name (walk.Get: defaults, notes, a pane it cannot read left out, as the byte loader did), with the hashes of the texts it keeps once under `blobs/`, so an unchanged text is written once across dumps; labels are `pardes-<date>-<time>` (`-2` and on in one second), `last` names the newest, the newest 32 are kept and the texts no kept one names are dropped in the same transaction, and at a session's end the file is compacted into `pardes.xitdb.new` and renamed over once it is past twice what it holds plus 64 MiB (another session notices the new file by its inode); a dump travels as an image, a database in memory holding it alone, deterministic (std.Io.failing's zero ids, so the replay's fixpoint holds): what Dump hands its host, what `Dump export <path>` and $PARDES_DUMP write whole (the snapshot goldens' `Restore <path>` unchanged), what a Restore of a label is read into, what a handover carries (`handover-<seq>`), and what `-l` and `Restore <path>` read; Restore takes a label or a path (bare: this session's last dump, else the newest), a label the file keeps none of refused in words (`Restore: no dump labelled X`, asked of the authority as the closed panes are); `/dumps` lists them, `<label> <date> <size> <panes>`, the newest first; Upgrade commits `upgrade-<pid>` before the exec, its record naming the state file and the label (a record without one, an older build's, names its `.dump` file, read as before), and the label stays; the byte dump stays for the join, the closed panes and the old `.dump` files (dump.byteDump), and dump.outPath, uniqueIn and upgradePath go for dump.label; measured (Debug, a session with a 3.2 MB file and three scratch panes, 20 dumps): the image on the loop 64-66 ms (59 ms of it BLAKE3 of the file, 0.65 ms in ReleaseFast) where the byte dump was 0.7 ms, the commit 9-16 ms with its fsyncs where the file was written in 2, a Restore 1.8 ms to read and 11.4 to load, and the file 3.55 MB after the first dump and 379 KB a dump after it with the big text unchanged (3.58 MB a dump with it edited each time, as twenty `.dump` files would be); tests: dump.zig's 'two dumps in one second get names of their own' and 'a dump makes every directory missing on the way to its DumpDir' become Db.zig's 'two dumps with one label get labels of their own' and 'a Dump makes every directory missing on the way to its DumpDir, and the file there', 'a session read by name, field by field as another build reads it, is the same session' becomes 'a session's byte dump read by name...' beside the new 'a session's image, read by name as every load reads it, is the same session, and dumps to the same bytes again', two replica tests compare the join's stream with dump.byteDump, the Handover tests read the carried image, upgrade's record test carries a label, the root listing tests and the monkey list `dumps`, the behaviour suite gains five root-ctl tests (a Dump and Restore round trip through 9P with `/dumps` and the tag's word, a label kept none of refused in words, the 33rd dump dropping the first, an unchanged text written once across two dumps by the file's size, a killed Dump leaving the one before it whole), the test host performs a Dump's write and a Restore as a host does, the monkey lets go of its log follower at a Restore, and daemon.py's upgrade case finds the kept dump in `/dumps`
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge
Diffstat (limited to 'src/walk.zig')
-rw-r--r--src/walk.zig808
1 files changed, 785 insertions, 23 deletions
diff --git a/src/walk.zig b/src/walk.zig
index 426408c8..8360a8b6 100644
--- a/src/walk.zig
+++ b/src/walk.zig
@@ -1,9 +1,11 @@
//! The reflective walk over the core's state: one comptime `@typeInfo` walk
//! that hashes a session into a digest (the replay harness compares two),
//! encodes values for the tape, and dumps a session for another build to
-//! load by name (`Dump`, `Load`: Restore, -l, the store, a handover, and
-//! the join of phase 5). The rollback copy is to walk the same way
-//! (docs/next-daemon.md).
+//! load by name: into a xitdb database's maps (`Put`, `Get`: a Dump,
+//! Restore, -l, an Upgrade, a handover; Db.zig), or as bytes with a table
+//! of its types (`Dump`, `Load`: a closed pane, the join of phase 5, and
+//! for one release the `.dump` files before). The rollback copy is to walk
+//! the same way (docs/next-daemon.md).
//!
//! Values, never addresses: a pointer is followed to what it points at, a
//! slice is its elements, an ArrayList its items, a hash map its entries in
@@ -33,6 +35,7 @@
//! digest(*const T, *Hasher) void` of its own hashes itself; one with
//! `dumpWrite`, `dumpRead` (and `dumpFree`) dumps itself as bytes.
const std = @import("std");
+const xitdb = @import("xitdb");
pub const Hasher = std.hash.Wyhash;
const builtin = @import("builtin");
@@ -1331,27 +1334,8 @@ pub fn Load(comptime roots: []const type) type {
try l.fields(T, s, out, true);
}
- /// `out` as a new `T` is before its fields are read: each field's
- /// default, an allocator the dump's.
fn fresh(l: *Self, comptime T: type, out: *T) DecodeError!void {
- if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom) return;
- if (comptime kindOf(T) == .skip) return;
- const s = @typeInfo(T).@"struct";
- inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| {
- if (attr.@"comptime") continue;
- if (comptime attr.default_value_ptr) |dp| {
- @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*;
- } else if (comptime F == std.mem.Allocator) {
- @field(out, field) = l.gpa;
- } else if (comptime !walkedField(T, field)) {
- @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to");
- }
- }
- }
-
- fn walkedField(comptime T: type, comptime field: []const u8) bool {
- inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true;
- return false;
+ freshOf(T, out, l.gpa);
}
/// Whether the source's `ty` and `T` are of one kind, else the
@@ -1731,6 +1715,28 @@ pub fn Load(comptime roots: []const type) type {
};
}
+/// `out` as a new `T` is before a load reads its fields: each field's
+/// default, an allocator `gpa`.
+fn freshOf(comptime T: type, out: *T, gpa: std.mem.Allocator) void {
+ if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom or kindOf(T) == .skip) return;
+ const s = @typeInfo(T).@"struct";
+ inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| {
+ if (attr.@"comptime") continue;
+ if (comptime attr.default_value_ptr) |dp| {
+ @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*;
+ } else if (comptime F == std.mem.Allocator) {
+ @field(out, field) = gpa;
+ } else if (comptime !walkedField(T, field)) {
+ @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to");
+ }
+ }
+}
+
+fn walkedField(comptime T: type, comptime field: []const u8) bool {
+ inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true;
+ return false;
+}
+
/// A value `save` made, `gpa`'s; what was let go is said in `notes` when
/// it is given.
pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes: ?*std.ArrayList(u8)) DecodeError!T {
@@ -1742,6 +1748,762 @@ pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes:
return v;
}
+// ---- a dump as a database: the same walk, by name, into xitdb's maps ----
+//
+// A dump (Db.zig) is a xitdb database in memory, its bytes one value of the
+// state file, written by the walk by name: a struct is a map whose keys are
+// its fields' names (`putKey`: a reader goes through them by name, never by
+// a hash of its own), a union a map of its one field, a slice, an array or
+// a list of records a list, of plain numbers one bytes value (its format
+// tag the numbers' kind and width), an enum the bytes of its name, a number
+// a uint or an int (a wider one its bytes), an optional an absent key (an
+// explicit none where the field's default is not null), a type with
+// `dumpWrite` its bytes, and a `.blob`'s text of `blob_min` bytes and up
+// its BLAKE3 hash (tag "b3"): the text is kept once beside the dump
+// (`Blob`). A pointer or a slice met again is the first one's value again:
+// what two fields share is shared again.
+//
+// Another build reads it by name: a field the dump has not takes its
+// default, a key this build has not is passed over, a value of another kind
+// takes its default and is said (`notes`); with no default the value it is
+// part of is refused, and an element of an array of optional pointers (a
+// session's panes) is then left out, nothing above it.
+
+pub const Mem = xitdb.Database(.memory, u256);
+
+/// A text a dump keeps beside it, once: its BLAKE3 hash and its bytes (the
+/// session's, while the dump is made).
+pub const Blob = struct { hash: [32]u8, bytes: []const u8 };
+
+/// A field's key: its name's word, unique within its struct (a reader
+/// matches names, so another build's words need not be these).
+fn nameKey(comptime name: []const u8) u256 {
+ return comptime std.hash.Wyhash.hash(0, name);
+}
+
+/// Whether a run of `E` is one bytes value: plain numbers.
+fn numbers(comptime E: type) bool {
+ return switch (@typeInfo(E)) {
+ .int => |i| @sizeOf(E) * 8 == i.bits and @sizeOf(E) <= 8,
+ .bool => true,
+ .float => |f| f.bits == 32 or f.bits == 64,
+ else => false,
+ };
+}
+
+/// A run of numbers' format tag: their kind and byte width ("u4", "i8", "b1").
+fn numbersTag(comptime E: type) [2]u8 {
+ const kind: u8 = switch (@typeInfo(E)) {
+ .int => |i| if (i.signedness == .signed) 'i' else 'u',
+ .bool => 'b',
+ .float => 'f',
+ else => unreachable,
+ };
+ return .{ kind, '0' + @sizeOf(E) };
+}
+
+const blob_tag = "b3".*;
+const wide_tag = "w8".*;
+const custom_tag = "cu".*;
+const bitset_tag = "bs".*;
+
+fn typeWord(comptime T: type) u64 {
+ return comptime std.hash.Wyhash.hash(0, @typeName(T));
+}
+
+/// Writes values into a database in memory, by name (see above).
+pub const Put = struct {
+ gpa: std.mem.Allocator,
+ /// Each pointer and slice written, by address, length and type, and
+ /// where its value is: met again, that value is written again.
+ memo: std.AutoHashMapUnmanaged(Key, xitdb.Slot) = .empty,
+ /// The blobs met, each once.
+ blobs: std.ArrayListUnmanaged(Blob) = .empty,
+ in_blob: bool = false,
+
+ const Key = struct { addr: usize, len: usize, ty: u64 };
+ const Cursor = Mem.Cursor(.read_write);
+
+ pub fn deinit(w: *Put) void {
+ w.memo.deinit(w.gpa);
+ w.blobs.deinit(w.gpa);
+ }
+
+ /// `v` written as `name`'s value in `map` (a dump's roots).
+ pub fn member(w: *Put, map: Mem.HashMap(.read_write), comptime name: []const u8, comptime T: type, v: *const T) anyerror!void {
+ try map.putKey(comptime nameKey(name), .{ .bytes = name });
+ var at = try map.putCursor(comptime nameKey(name));
+ _ = try w.value(T, v, &at);
+ }
+
+ /// `v` written at `c`; where its value is.
+ pub fn value(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot {
+ if (comptime isAtomic(T)) return w.value(@FieldType(T, "raw"), &v.raw, c);
+ if (comptime isList(T)) return w.many(@typeInfo(@FieldType(T, "items")).pointer.child, v.items, c, true);
+ switch (comptime kindOf(T)) {
+ .skip, .void => try c.write(.{ .slot = null }),
+ .bool => try c.write(.{ .uint = @intFromBool(v.*) }),
+ .int => if (@typeInfo(T).int.bits <= 64) {
+ try c.write(if (@typeInfo(T).int.signedness == .signed) .{ .int = v.* } else .{ .uint = v.* });
+ } else {
+ const wide: i128 = @intCast(v.*);
+ try c.write(.{ .bytes_object = .{ .value = std.mem.asBytes(&wide), .format_tag = wide_tag } });
+ },
+ .float => try c.write(.{ .float = v.* }),
+ .@"enum" => if (std.enums.tagName(T, v.*)) |name| {
+ try c.write(.{ .bytes = name });
+ } else try c.write(.{ .int = @intCast(@intFromEnum(v.*)) }),
+ .optional => {
+ const C = @typeInfo(T).optional.child;
+ if (v.*) |*payload| {
+ // An allocator, say, is no state: there or not, which a
+ // load answers with its own.
+ if (comptime kindOf(Canon(C)) == .skip) try c.write(.{ .uint = 1 }) else return w.value(C, payload, c);
+ } else try c.write(.{ .slot = null });
+ },
+ .one => {
+ const C = @typeInfo(T).pointer.child;
+ const key: Key = .{ .addr = @intFromPtr(v.*), .len = @max(1, @sizeOf(C)), .ty = typeWord(T) };
+ if (w.memo.get(key)) |slot| {
+ try c.write(.{ .slot = slot });
+ return slot;
+ }
+ const slot = try w.value(C, v.*, c);
+ try w.memo.put(w.gpa, key, slot);
+ return slot;
+ },
+ .slice => return w.many(@typeInfo(T).pointer.child, v.*, c, true),
+ .array => return w.many(@typeInfo(T).array.child, v, c, false),
+ .prefix => unreachable,
+ .@"struct" => return w.fields(T, v, c),
+ .@"union" => {
+ const map = try Mem.HashMap(.read_write).init(c.*);
+ switch (v.*) {
+ inline else => |*payload, tag| {
+ const name = @tagName(tag);
+ try map.putKey(comptime nameKey(name), .{ .bytes = name });
+ var at = try map.putCursor(comptime nameKey(name));
+ _ = try w.value(@TypeOf(payload.*), payload, &at);
+ },
+ }
+ return map.slot();
+ },
+ // No type a dump walks has them.
+ .err, .error_union => unreachable,
+ .custom => {
+ var buf: [4096]u8 = undefined;
+ var out = try c.writer(&buf);
+ try T.dumpWrite(v, w.gpa, &out.interface);
+ out.format_tag = custom_tag;
+ try out.finish();
+ },
+ .bitset => {
+ const words = (v.bit_length + @bitSizeOf(usize) - 1) / @bitSizeOf(usize);
+ var buf: [64]u8 = undefined;
+ var out = try c.writer(&buf);
+ try out.interface.writeAll(std.mem.asBytes(&@as(u64, v.bit_length)));
+ if (words > 0) try out.interface.writeAll(std.mem.sliceAsBytes(v.masks[0..words]));
+ out.format_tag = bitset_tag;
+ try out.finish();
+ },
+ }
+ return c.slot();
+ }
+
+ /// A run of `E`: numbers as one bytes value (a blob's text by its
+ /// hash), records as a list; a slice met again (`shared`) its first
+ /// value.
+ fn many(w: *Put, comptime E: type, items: []const E, c: *Cursor, comptime shared: bool) anyerror!xitdb.Slot {
+ const key: Key = .{ .addr = @intFromPtr(items.ptr), .len = items.len, .ty = typeWord([]const E) };
+ if (shared and items.len > 0) if (w.memo.get(key)) |slot| {
+ try c.write(.{ .slot = slot });
+ return slot;
+ };
+ const slot = if (comptime numbers(E)) slot: {
+ const bytes = std.mem.sliceAsBytes(items);
+ if (E == u8 and w.in_blob and bytes.len >= blob_min) {
+ var sum: [32]u8 = undefined;
+ std.crypto.hash.Blake3.hash(bytes, &sum, .{});
+ const known = for (w.blobs.items) |b| {
+ if (std.mem.eql(u8, &b.hash, &sum)) break true;
+ } else false;
+ if (!known) try w.blobs.append(w.gpa, .{ .hash = sum, .bytes = bytes });
+ try c.write(.{ .bytes_object = .{ .value = &sum, .format_tag = blob_tag } });
+ } else try c.write(.{ .bytes_object = .{ .value = bytes, .format_tag = numbersTag(E) } });
+ break :slot try c.slot();
+ } else slot: {
+ const list = try Mem.ArrayList(.read_write).init(c.*);
+ for (items) |*e| {
+ var at = try list.appendCursor();
+ _ = try w.value(E, e, &at);
+ }
+ break :slot try list.slot();
+ };
+ if (shared and items.len > 0) try w.memo.put(w.gpa, key, slot);
+ return slot;
+ }
+
+ fn fields(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot {
+ const map = try Mem.HashMap(.read_write).init(c.*);
+ const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed";
+ inline for (comptime dumpFields(T)) |name| {
+ const F = @FieldType(T, name);
+ if (comptime kindOf(Canon(F)) != .void) put: {
+ // Null where the field's default is: no key at all.
+ if (comptime @typeInfo(F) == .optional and defaultIsNull(T, name)) if (@field(v, name) == null) break :put;
+ const was = w.in_blob;
+ w.in_blob = was or comptime blob(T, name);
+ defer w.in_blob = was;
+ try map.putKey(comptime nameKey(name), .{ .bytes = name });
+ var at = try map.putCursor(comptime nameKey(name));
+ if (comptime cutArray(T, name)) {
+ const all = &@field(v, name);
+ _ = try w.many(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))], &at, false);
+ } else if (comptime packed_struct) {
+ const copy: F = @field(v, name);
+ _ = try w.value(F, &copy, &at);
+ } else _ = try w.value(F, &@field(v, name), &at);
+ }
+ }
+ return map.slot();
+ }
+};
+
+fn defaultIsNull(comptime T: type, comptime name: []const u8) bool {
+ const F = @FieldType(T, name);
+ const attr = @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, name).?];
+ const dp = attr.default_value_ptr orelse return false;
+ return @as(*const F, @ptrCast(@alignCast(dp))).* == null;
+}
+
+/// Reads values a `Put` wrote, by name, as another build may have written
+/// them (see above).
+pub const Get = struct {
+ /// Where what is read is made (a `replica.alloc` field's own allocator
+ /// while it is read).
+ gpa: std.mem.Allocator,
+ /// The reader's own lists'.
+ own: std.mem.Allocator,
+ /// The texts its blobs' hashes name (Db.zig's `blobs`), by hash.
+ blobs: ?Mem.HashMap(.read_only) = null,
+ /// Each pointer and slice made so far: what a value met again is, and
+ /// what `rollback` frees.
+ made: std.ArrayList(Made) = .empty,
+ /// Where each value made is: met again, the same one.
+ memo: std.AutoHashMapUnmanaged(u128, usize) = .empty,
+ /// What a type's `dumpRead` made, freed by its `dumpFree` too.
+ customs: std.ArrayList(struct { ptr: *anyopaque, gpa: std.mem.Allocator, free: *const fn (*anyopaque, std.mem.Allocator) void }) = .empty,
+ /// What was let go, for the person whose session it is.
+ notes: std.ArrayList(u8) = .empty,
+ /// The field being read, for a note.
+ field: []const u8 = "",
+ allocs: Allocators = .{},
+
+ const Cursor = Mem.Cursor(.read_only);
+ pub const Error = error{ BadDump, Incompatible, OutOfMemory };
+
+ pub fn init(gpa: std.mem.Allocator) Get {
+ return .{ .gpa = gpa, .own = gpa };
+ }
+
+ pub fn deinit(l: *Get) void {
+ l.made.deinit(l.own);
+ l.memo.deinit(l.own);
+ l.customs.deinit(l.own);
+ l.notes.deinit(l.own);
+ }
+
+ /// Everything read so far freed: a load that fails leaves nothing.
+ /// What it was read into must not be used after (nor freed).
+ pub fn rollback(l: *Get) void {
+ for (l.customs.items) |c| c.free(c.ptr, c.gpa);
+ for (l.made.items) |m| if (m.bytes.len > 0) m.gpa.rawFree(m.bytes, m.alignment, @returnAddress());
+ l.customs.clearRetainingCapacity();
+ l.made.clearRetainingCapacity();
+ l.memo.clearRetainingCapacity();
+ }
+
+ pub fn note(l: *Get, comptime fmt: []const u8, args: anytype) void {
+ if (l.notes.items.len > 4096) return;
+ l.notes.print(l.own, fmt ++ "\n", args) catch {};
+ }
+
+ /// The value at `c`, made: what it points at is `gpa`'s.
+ pub fn get(l: *Get, comptime T: type, c: Cursor) Error!T {
+ var out: T = undefined;
+ try l.fresh(T, &out);
+ try l.value(T, c, &out);
+ return out;
+ }
+
+ /// The struct at `c` read into `out` as it stands: a field the dump
+ /// holds is replaced (what it held is not freed), every other one is
+ /// left as it is, and one another build cannot give is left too, and
+ /// noted.
+ pub fn into(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void {
+ try l.fields(T, c, out, true);
+ }
+
+ /// Where the value of `name` is in the map at `c` (a dump's roots), by
+ /// its name; null when it is not there.
+ pub fn member(c: Cursor, name: []const u8) Error!?Cursor {
+ const map = Mem.HashMap(.read_only).init(c) catch return error.BadDump;
+ var it = map.iterator() catch return error.BadDump;
+ while (it.next() catch return error.BadDump) |kv_at| {
+ const kv = kv_at.readKeyValuePair() catch return error.BadDump;
+ var buf: [64]u8 = undefined;
+ const got = kv.key_cursor.readBytes(&buf) catch continue;
+ if (std.mem.eql(u8, got, name)) return kv.value_cursor;
+ }
+ return null;
+ }
+
+ fn fresh(l: *Get, comptime T: type, out: *T) Error!void {
+ freshOf(T, out, l.gpa);
+ }
+
+ fn slotWord(slot: xitdb.Slot) u128 {
+ return @as(u72, @bitCast(slot));
+ }
+
+ fn slotOf(c: Cursor) Error!xitdb.Slot {
+ return c.slot() catch error.BadDump;
+ }
+
+ /// Whether the value at `slot` has a place of its own (an inline number
+ /// or a short text is its slot alone: no identity to share).
+ fn placed(slot: xitdb.Slot) bool {
+ return switch (slot.tag) {
+ .none, .uint, .int, .float, .short_bytes => false,
+ else => true,
+ };
+ }
+
+ /// An allocation made for the value at `slot`, kept: what a value met
+ /// again is, and what `rollback` frees.
+ fn keep(l: *Get, bytes: []u8, alignment: std.mem.Alignment, slot: xitdb.Slot) Error!usize {
+ try l.made.append(l.own, .{ .bytes = bytes, .alignment = alignment, .gpa = l.gpa });
+ const at = l.made.items.len - 1;
+ // Not kept after all: the caller frees it.
+ if (placed(slot)) l.memo.put(l.own, slotWord(slot), at) catch |err| {
+ _ = l.made.pop();
+ return err;
+ };
+ return at;
+ }
+
+ /// The value made before for the same place, when there was one.
+ fn again(l: *Get, slot: xitdb.Slot) Error!?[]u8 {
+ if (!placed(slot)) return null;
+ const at = l.memo.get(slotWord(slot)) orelse return null;
+ // Made, then let go of by the reader's caller (a dump's mounts).
+ if (at >= l.made.items.len) return null;
+ const m = l.made.items[at];
+ if (m.dead) return error.Incompatible;
+ return m.bytes;
+ }
+
+ /// `T` read from `c` into `out`, which `fresh` has made ready.
+ fn value(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void {
+ if (comptime isAtomic(T)) {
+ const R = @FieldType(T, "raw");
+ var raw: R = undefined;
+ try l.fresh(R, &raw);
+ try l.value(R, c, &raw);
+ out.* = .{ .raw = raw };
+ return;
+ }
+ const slot = try slotOf(c);
+ if (comptime isList(T)) {
+ const got = try l.slice(@typeInfo(@FieldType(T, "items")).pointer.child, c, slot);
+ out.* = .empty;
+ out.items = got;
+ out.capacity = got.len;
+ return;
+ }
+ switch (comptime kindOf(T)) {
+ .skip, .void => {},
+ .bool => out.* = switch (slot.tag) {
+ .uint => slot.value != 0,
+ else => return error.Incompatible,
+ },
+ .int => out.* = try l.int(T, c, slot),
+ .float => out.* = switch (slot.tag) {
+ .float => @floatCast(@as(f64, @bitCast(slot.value))),
+ else => return error.Incompatible,
+ },
+ .@"enum" => {
+ const e = @typeInfo(T).@"enum";
+ switch (slot.tag) {
+ .bytes, .short_bytes => {
+ var buf: [256]u8 = undefined;
+ const name = c.readBytes(&buf) catch return error.Incompatible;
+ inline for (e.field_names) |field| if (std.mem.eql(u8, name, field)) {
+ out.* = @field(T, field);
+ return;
+ };
+ l.note("{s}: no {s} here", .{ l.field, name });
+ return error.Incompatible;
+ },
+ .int, .uint => if (e.mode == .nonexhaustive) {
+ const n: i64 = @bitCast(slot.value);
+ out.* = @enumFromInt(std.math.cast(e.tag_type, n) orelse return error.Incompatible);
+ } else return error.Incompatible,
+ else => return error.Incompatible,
+ }
+ },
+ .optional => {
+ const C = @typeInfo(T).optional.child;
+ if (slot.tag == .none) {
+ out.* = null;
+ return;
+ }
+ if (comptime kindOf(Canon(C)) == .skip) {
+ out.* = if (comptime C == std.mem.Allocator) l.gpa else null;
+ return;
+ }
+ // Through a local, as the byte loader does: a pointer's
+ // null is its address.
+ if (comptime @typeInfo(C) == .pointer) {
+ var payload: C = undefined;
+ try l.value(C, c, &payload);
+ out.* = payload;
+ return;
+ }
+ out.* = @as(C, undefined);
+ try l.fresh(C, &out.*.?);
+ l.value(C, c, &out.*.?) catch |err| {
+ out.* = null;
+ return err;
+ };
+ },
+ .one => {
+ const C = @typeInfo(T).pointer.child;
+ if (try l.again(slot)) |bytes| {
+ out.* = @ptrCast(@alignCast(bytes.ptr));
+ return;
+ }
+ const one = try l.gpa.create(C);
+ const at = l.keep(std.mem.asBytes(one), .of(C), slot) catch |err| {
+ l.gpa.destroy(one);
+ return err;
+ };
+ try l.fresh(C, one);
+ l.value(C, c, one) catch |err| {
+ // Left half made: no later value may be it.
+ l.made.items[at].dead = true;
+ return err;
+ };
+ out.* = one;
+ },
+ .slice => {
+ if (comptime @typeInfo(T).pointer.sentinel_ptr != null) @compileError("walk: a sentinel slice: " ++ @typeName(T));
+ out.* = try l.slice(@typeInfo(T).pointer.child, c, slot);
+ },
+ .array => try l.elements(@typeInfo(T).array.child, c, slot, out),
+ .prefix => unreachable,
+ .@"struct" => try l.fields(T, c, out, false),
+ .@"union" => {
+ const u = @typeInfo(T).@"union";
+ const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible;
+ var it = map.iterator() catch return error.BadDump;
+ const kv_at = (it.next() catch return error.BadDump) orelse return error.Incompatible;
+ const kv = kv_at.readKeyValuePair() catch return error.BadDump;
+ var buf: [256]u8 = undefined;
+ const name = kv.key_cursor.readBytes(&buf) catch return error.Incompatible;
+ inline for (u.field_names, u.field_types) |field, F| if (std.mem.eql(u8, name, field)) {
+ out.* = @unionInit(T, field, undefined);
+ try l.fresh(F, &@field(out.*, field));
+ return l.value(F, kv.value_cursor, &@field(out.*, field));
+ };
+ l.note("{s}: no {s} here", .{ l.field, name });
+ return error.Incompatible;
+ },
+ .err, .error_union => unreachable,
+ .custom => {
+ const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible;
+ defer l.own.free(got.value);
+ T.dumpRead(out, l.gpa, got.value) catch |err| {
+ if (err == error.OutOfMemory) return error.OutOfMemory;
+ l.note("{s}: {t}", .{ l.field, err });
+ return error.Incompatible;
+ };
+ if (comptime @hasDecl(T, "dumpFree")) try l.customs.append(l.own, .{ .ptr = out, .gpa = l.gpa, .free = @ptrCast(&T.dumpFree) });
+ },
+ .bitset => {
+ const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible;
+ defer l.own.free(got.value);
+ if (got.value.len < 8) return error.BadDump;
+ const bits = countOf(std.mem.bytesToValue(u64, got.value[0..8])) catch return error.BadDump;
+ const words = (bits + @bitSizeOf(usize) - 1) / @bitSizeOf(usize);
+ if (got.value.len != 8 + words * @sizeOf(usize)) return error.BadDump;
+ out.* = try .initEmpty(l.gpa, bits);
+ if (words > 0) @memcpy(std.mem.sliceAsBytes(out.masks[0..words]), got.value[8..]);
+ },
+ }
+ }
+
+ fn int(l: *Get, comptime I: type, c: Cursor, slot: xitdb.Slot) Error!I {
+ _ = l;
+ return switch (slot.tag) {
+ .uint => std.math.cast(I, slot.value) orelse error.Incompatible,
+ .int => std.math.cast(I, @as(i64, @bitCast(slot.value))) orelse error.Incompatible,
+ .bytes, .short_bytes => wide: {
+ var buf: [16]u8 = undefined;
+ const got = c.readBytesObject(&buf) catch return error.Incompatible;
+ if (got.value.len != 16 or !std.mem.eql(u8, &(got.format_tag orelse return error.Incompatible), &wide_tag)) return error.Incompatible;
+ break :wide std.math.cast(I, std.mem.bytesToValue(i128, got.value[0..16])) orelse error.Incompatible;
+ },
+ else => error.Incompatible,
+ };
+ }
+
+ /// Numbers' `bytes` (format `tag`) as `E`s into `out`, as many as fit:
+ /// another width of the same kind (another build's) converted one by
+ /// one. How many the bytes hold.
+ fn convert(comptime E: type, bytes: []const u8, tag: [2]u8, out: []E) Error!usize {
+ const width: usize = if (tag[1] >= '1' and tag[1] <= '8') tag[1] - '0' else return error.BadDump;
+ if (bytes.len % width != 0) return error.BadDump;
+ const n = bytes.len / width;
+ const fits = @min(n, out.len);
+ if (std.mem.eql(u8, &tag, &numbersTag(E))) {
+ @memcpy(std.mem.sliceAsBytes(out[0..fits]), bytes[0 .. fits * width]);
+ return n;
+ }
+ if (tag[0] != numbersTag(E)[0]) return error.Incompatible;
+ for (out[0..fits], 0..) |*e, i| {
+ const raw = bytes[i * width ..][0..width];
+ e.* = switch (@typeInfo(E)) {
+ .int => |info| if (info.signedness == .signed) cast: {
+ var wide: i64 = 0;
+ @memcpy(std.mem.asBytes(&wide)[0..width], raw);
+ if (width < 8) {
+ const shift: u6 = @intCast(64 - width * 8);
+ wide = (wide << shift) >> shift;
+ }
+ break :cast std.math.cast(E, wide) orelse return error.Incompatible;
+ } else cast: {
+ var wide: u64 = 0;
+ @memcpy(std.mem.asBytes(&wide)[0..width], raw);
+ break :cast std.math.cast(E, wide) orelse return error.Incompatible;
+ },
+ .float => if (width == 8) @floatCast(std.mem.bytesToValue(f64, raw[0..8])) else if (width == 4) @floatCast(std.mem.bytesToValue(f32, raw[0..4])) else return error.Incompatible,
+ .bool => raw[0] != 0,
+ else => unreachable,
+ };
+ }
+ return n;
+ }
+
+ /// The numbers' bytes at `c` and their format tag, `alloc`'s.
+ fn numbersAt(c: Cursor, alloc: std.mem.Allocator) Error!struct { bytes: []u8, tag: [2]u8 } {
+ const got = c.readBytesObjectAlloc(alloc, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.Incompatible;
+ const bytes: []u8 = @constCast(got.value);
+ const tag = got.format_tag orelse {
+ alloc.free(bytes);
+ return error.Incompatible;
+ };
+ return .{ .bytes = bytes, .tag = tag };
+ }
+
+ /// A slice of `E` at `c`: numbers' bytes, a blob's text by its hash, or
+ /// a list; one met again is the same.
+ fn slice(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot) Error![]E {
+ if (try l.again(slot)) |bytes| return @as([*]E, @ptrCast(@alignCast(bytes.ptr)))[0 .. bytes.len / @max(1, @sizeOf(E))];
+ if (comptime numbers(E)) {
+ switch (slot.tag) {
+ .bytes, .short_bytes => {},
+ else => return error.Incompatible,
+ }
+ if (E == u8) if (try l.blob(c, slot)) |text| return text;
+ // A text, as it is written: its bytes are the slice.
+ if (E == u8) {
+ const got = try numbersAt(c, l.gpa);
+ if (!std.mem.eql(u8, &got.tag, &numbersTag(u8))) {
+ l.gpa.free(got.bytes);
+ return error.Incompatible;
+ }
+ if (got.bytes.len == 0) {
+ l.gpa.free(got.bytes);
+ return &.{};
+ }
+ _ = l.keep(got.bytes, .@"1", slot) catch |err| {
+ l.gpa.free(got.bytes);
+ return err;
+ };
+ return got.bytes;
+ }
+ const got = try numbersAt(c, l.own);
+ defer l.own.free(got.bytes);
+ const width: usize = if (got.tag[1] >= '1' and got.tag[1] <= '8') got.tag[1] - '0' else return error.BadDump;
+ const n = got.bytes.len / width;
+ if (n == 0) return &.{};
+ const many = try l.gpa.alloc(E, n);
+ const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| {
+ l.gpa.free(many);
+ return err;
+ };
+ _ = convert(E, got.bytes, got.tag, many) catch |err| {
+ l.made.items[at].dead = true;
+ return err;
+ };
+ return many;
+ }
+ const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible;
+ const n = countOf(list.count() catch return error.BadDump) catch return error.BadDump;
+ if (n == 0) return &.{};
+ const many = try l.gpa.alloc(E, n);
+ const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| {
+ l.gpa.free(many);
+ return err;
+ };
+ l.listInto(E, list, many) catch |err| {
+ l.made.items[at].dead = true;
+ return err;
+ };
+ return many;
+ }
+
+ /// A blob's text, when `c` holds a blob's hash: made in `gpa`'s.
+ fn blob(l: *Get, c: Cursor, slot: xitdb.Slot) Error!?[]u8 {
+ var buf: [32]u8 = undefined;
+ const got = c.readBytesObject(&buf) catch return null;
+ const tag = got.format_tag orelse return null;
+ if (!std.mem.eql(u8, &tag, &blob_tag) or got.value.len != 32) return null;
+ const texts = l.blobs orelse return error.BadDump;
+ const at = (texts.getCursor(std.mem.readInt(u256, got.value[0..32], .big)) catch return error.BadDump) orelse return error.BadDump;
+ const text = at.readBytesAlloc(l.gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.BadDump;
+ const bytes: []u8 = @constCast(text);
+ _ = l.keep(bytes, .@"1", slot) catch |err| {
+ l.gpa.free(bytes);
+ return err;
+ };
+ return bytes;
+ }
+
+ /// An array of `E` at `c`: as many as it holds, the rest as they are.
+ fn elements(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot, out: []E) Error!void {
+ if (comptime numbers(E)) {
+ switch (slot.tag) {
+ .bytes, .short_bytes => {},
+ else => return error.Incompatible,
+ }
+ const got = try numbersAt(c, l.own);
+ defer l.own.free(got.bytes);
+ _ = try convert(E, got.bytes, got.tag, out);
+ return;
+ }
+ const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible;
+ try l.listInto(E, list, out);
+ }
+
+ /// The list's elements into `out`, as many as fit; an element of an
+ /// array of optional pointers (a session's panes) that cannot be read is
+ /// left out, not the array.
+ fn listInto(l: *Get, comptime E: type, list: Mem.ArrayList(.read_only), out: []E) Error!void {
+ const droppable = comptime @typeInfo(E) == .optional and @typeInfo(@typeInfo(E).optional.child) == .pointer;
+ var failed = false;
+ var it = list.iterator() catch return error.BadDump;
+ var i: usize = 0;
+ while (it.next() catch return error.BadDump) |at| : (i += 1) {
+ if (i >= out.len) break;
+ try l.fresh(E, &out[i]);
+ l.value(E, at, &out[i]) catch |err| {
+ if (err != error.Incompatible) return err;
+ if (droppable) {
+ out[i] = null;
+ l.note("{s}[{d}] left out", .{ l.field, i });
+ } else failed = true;
+ };
+ }
+ if (failed) return error.Incompatible;
+ }
+
+ fn fields(l: *Get, comptime T: type, c: Cursor, out: *T, existing: bool) Error!void {
+ const names = comptime dumpFields(T);
+ const outer = l.field;
+ defer l.field = outer;
+ const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible;
+ var found: std.StaticBitSet(@max(1, names.len)) = .empty;
+ var failed = false;
+ var it = map.iterator() catch return error.BadDump;
+ while (it.next() catch return error.BadDump) |kv_at| {
+ const kv = kv_at.readKeyValuePair() catch return error.BadDump;
+ var buf: [256]u8 = undefined;
+ // A key that is no field's name of this build: passed over.
+ const name = kv.key_cursor.readBytes(&buf) catch continue;
+ const k: ?usize = inline for (names, 0..) |field, k| {
+ if (std.mem.eql(u8, name, field)) break k;
+ } else null;
+ const at = k orelse continue;
+ found.set(at);
+ inline for (names, 0..) |field, i| if (i == at) {
+ l.field = field;
+ l.oneField(T, field, kv.value_cursor, out, existing) catch |err| {
+ if (err != error.Incompatible) return err;
+ const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?];
+ if (comptime attr.default_value_ptr) |dp| {
+ @field(out, field) = @as(*const @FieldType(T, field), @ptrCast(@alignCast(dp))).*;
+ l.note("{s}.{s}: its kind changed; its default taken", .{ @typeName(T), field });
+ } else if (existing) {
+ l.note("{s}.{s}: its kind changed; left as it was", .{ @typeName(T), field });
+ } else failed = true;
+ };
+ };
+ }
+ // A field the writer did not have (or held null at a null default):
+ // its default (`fresh`), or the value cannot be made.
+ inline for (names, 0..) |field, i| if (!found.isSet(i)) {
+ const F = @FieldType(T, field);
+ const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?];
+ if (comptime attr.default_value_ptr == null and kindOf(Canon(F)) != .void) {
+ if (!existing) {
+ l.note("{s}.{s}: not in the dump", .{ @typeName(T), field });
+ failed = true;
+ }
+ } else if (comptime @typeInfo(F) == .optional and defaultIsNull(T, field)) if (existing) {
+ // A null left out: null here too.
+ @field(out, field) = null;
+ };
+ };
+ if (failed) return error.Incompatible;
+ }
+
+ fn oneField(l: *Get, comptime T: type, comptime field: []const u8, c: Cursor, out: *T, existing: bool) Error!void {
+ const F = @FieldType(T, field);
+ const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed";
+ const gpa = l.gpa;
+ defer l.gpa = gpa;
+ if (comptime allocOf(T, field)) |which| if (@field(l.allocs, which)) |a| {
+ l.gpa = a;
+ };
+ // A field this build compiles out (a PDF slot where there is no
+ // MuPDF): passed over, said when it held something.
+ if (comptime kindOf(Canon(F)) == .void) {
+ if ((try slotOf(c)).tag != .none) l.note("{s}: not in this build", .{l.field});
+ return;
+ }
+ if (comptime cutArray(T, field)) {
+ const slot = try slotOf(c);
+ return l.elements(@typeInfo(F).array.child, c, slot, &@field(out, field));
+ }
+ // A struct held in place in one read as it stands (the root's):
+ // what it does not hold stays as it is too.
+ if (comptime kindOf(Canon(F)) == .@"struct" and Canon(F) == F and @typeInfo(F).@"struct".layout != .@"packed") if (existing) {
+ return l.fields(F, c, &@field(out, field), true);
+ };
+ if (packed_struct) {
+ var copy: F = undefined;
+ try l.fresh(F, &copy);
+ try l.value(F, c, &copy);
+ @field(out, field) = copy;
+ return;
+ }
+ try l.fresh(F, &@field(out, field));
+ try l.value(F, c, &@field(out, field));
+ }
+};
+
test "an optional slice comes back, and the fields after it" {
const S = struct { before: u32 = 0, command: ?[]u8 = null, after: u32 = 0 };
var text = "sleep 30".*;