summaryrefslogtreecommitdiff
path: root/src/walk.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/walk.zig')
-rw-r--r--src/walk.zig808
1 files changed, 785 insertions, 23 deletions
diff --git a/src/walk.zig b/src/walk.zig
index 426408c8..8360a8b6 100644
--- a/src/walk.zig
+++ b/src/walk.zig
@@ -1,9 +1,11 @@
//! The reflective walk over the core's state: one comptime `@typeInfo` walk
//! that hashes a session into a digest (the replay harness compares two),
//! encodes values for the tape, and dumps a session for another build to
-//! load by name (`Dump`, `Load`: Restore, -l, the store, a handover, and
-//! the join of phase 5). The rollback copy is to walk the same way
-//! (docs/next-daemon.md).
+//! load by name: into a xitdb database's maps (`Put`, `Get`: a Dump,
+//! Restore, -l, an Upgrade, a handover; Db.zig), or as bytes with a table
+//! of its types (`Dump`, `Load`: a closed pane, the join of phase 5, and
+//! for one release the `.dump` files before). The rollback copy is to walk
+//! the same way (docs/next-daemon.md).
//!
//! Values, never addresses: a pointer is followed to what it points at, a
//! slice is its elements, an ArrayList its items, a hash map its entries in
@@ -33,6 +35,7 @@
//! digest(*const T, *Hasher) void` of its own hashes itself; one with
//! `dumpWrite`, `dumpRead` (and `dumpFree`) dumps itself as bytes.
const std = @import("std");
+const xitdb = @import("xitdb");
pub const Hasher = std.hash.Wyhash;
const builtin = @import("builtin");
@@ -1331,27 +1334,8 @@ pub fn Load(comptime roots: []const type) type {
try l.fields(T, s, out, true);
}
- /// `out` as a new `T` is before its fields are read: each field's
- /// default, an allocator the dump's.
fn fresh(l: *Self, comptime T: type, out: *T) DecodeError!void {
- if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom) return;
- if (comptime kindOf(T) == .skip) return;
- const s = @typeInfo(T).@"struct";
- inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| {
- if (attr.@"comptime") continue;
- if (comptime attr.default_value_ptr) |dp| {
- @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*;
- } else if (comptime F == std.mem.Allocator) {
- @field(out, field) = l.gpa;
- } else if (comptime !walkedField(T, field)) {
- @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to");
- }
- }
- }
-
- fn walkedField(comptime T: type, comptime field: []const u8) bool {
- inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true;
- return false;
+ freshOf(T, out, l.gpa);
}
/// Whether the source's `ty` and `T` are of one kind, else the
@@ -1731,6 +1715,28 @@ pub fn Load(comptime roots: []const type) type {
};
}
+/// `out` as a new `T` is before a load reads its fields: each field's
+/// default, an allocator `gpa`.
+fn freshOf(comptime T: type, out: *T, gpa: std.mem.Allocator) void {
+ if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom or kindOf(T) == .skip) return;
+ const s = @typeInfo(T).@"struct";
+ inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| {
+ if (attr.@"comptime") continue;
+ if (comptime attr.default_value_ptr) |dp| {
+ @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*;
+ } else if (comptime F == std.mem.Allocator) {
+ @field(out, field) = gpa;
+ } else if (comptime !walkedField(T, field)) {
+ @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to");
+ }
+ }
+}
+
+fn walkedField(comptime T: type, comptime field: []const u8) bool {
+ inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true;
+ return false;
+}
+
/// A value `save` made, `gpa`'s; what was let go is said in `notes` when
/// it is given.
pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes: ?*std.ArrayList(u8)) DecodeError!T {
@@ -1742,6 +1748,762 @@ pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes:
return v;
}
+// ---- a dump as a database: the same walk, by name, into xitdb's maps ----
+//
+// A dump (Db.zig) is a xitdb database in memory, its bytes one value of the
+// state file, written by the walk by name: a struct is a map whose keys are
+// its fields' names (`putKey`: a reader goes through them by name, never by
+// a hash of its own), a union a map of its one field, a slice, an array or
+// a list of records a list, of plain numbers one bytes value (its format
+// tag the numbers' kind and width), an enum the bytes of its name, a number
+// a uint or an int (a wider one its bytes), an optional an absent key (an
+// explicit none where the field's default is not null), a type with
+// `dumpWrite` its bytes, and a `.blob`'s text of `blob_min` bytes and up
+// its BLAKE3 hash (tag "b3"): the text is kept once beside the dump
+// (`Blob`). A pointer or a slice met again is the first one's value again:
+// what two fields share is shared again.
+//
+// Another build reads it by name: a field the dump has not takes its
+// default, a key this build has not is passed over, a value of another kind
+// takes its default and is said (`notes`); with no default the value it is
+// part of is refused, and an element of an array of optional pointers (a
+// session's panes) is then left out, nothing above it.
+
+pub const Mem = xitdb.Database(.memory, u256);
+
+/// A text a dump keeps beside it, once: its BLAKE3 hash and its bytes (the
+/// session's, while the dump is made).
+pub const Blob = struct { hash: [32]u8, bytes: []const u8 };
+
+/// A field's key: its name's word, unique within its struct (a reader
+/// matches names, so another build's words need not be these).
+fn nameKey(comptime name: []const u8) u256 {
+ return comptime std.hash.Wyhash.hash(0, name);
+}
+
+/// Whether a run of `E` is one bytes value: plain numbers.
+fn numbers(comptime E: type) bool {
+ return switch (@typeInfo(E)) {
+ .int => |i| @sizeOf(E) * 8 == i.bits and @sizeOf(E) <= 8,
+ .bool => true,
+ .float => |f| f.bits == 32 or f.bits == 64,
+ else => false,
+ };
+}
+
+/// A run of numbers' format tag: their kind and byte width ("u4", "i8", "b1").
+fn numbersTag(comptime E: type) [2]u8 {
+ const kind: u8 = switch (@typeInfo(E)) {
+ .int => |i| if (i.signedness == .signed) 'i' else 'u',
+ .bool => 'b',
+ .float => 'f',
+ else => unreachable,
+ };
+ return .{ kind, '0' + @sizeOf(E) };
+}
+
+const blob_tag = "b3".*;
+const wide_tag = "w8".*;
+const custom_tag = "cu".*;
+const bitset_tag = "bs".*;
+
+fn typeWord(comptime T: type) u64 {
+ return comptime std.hash.Wyhash.hash(0, @typeName(T));
+}
+
+/// Writes values into a database in memory, by name (see above).
+pub const Put = struct {
+ gpa: std.mem.Allocator,
+ /// Each pointer and slice written, by address, length and type, and
+ /// where its value is: met again, that value is written again.
+ memo: std.AutoHashMapUnmanaged(Key, xitdb.Slot) = .empty,
+ /// The blobs met, each once.
+ blobs: std.ArrayListUnmanaged(Blob) = .empty,
+ in_blob: bool = false,
+
+ const Key = struct { addr: usize, len: usize, ty: u64 };
+ const Cursor = Mem.Cursor(.read_write);
+
+ pub fn deinit(w: *Put) void {
+ w.memo.deinit(w.gpa);
+ w.blobs.deinit(w.gpa);
+ }
+
+ /// `v` written as `name`'s value in `map` (a dump's roots).
+ pub fn member(w: *Put, map: Mem.HashMap(.read_write), comptime name: []const u8, comptime T: type, v: *const T) anyerror!void {
+ try map.putKey(comptime nameKey(name), .{ .bytes = name });
+ var at = try map.putCursor(comptime nameKey(name));
+ _ = try w.value(T, v, &at);
+ }
+
+ /// `v` written at `c`; where its value is.
+ pub fn value(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot {
+ if (comptime isAtomic(T)) return w.value(@FieldType(T, "raw"), &v.raw, c);
+ if (comptime isList(T)) return w.many(@typeInfo(@FieldType(T, "items")).pointer.child, v.items, c, true);
+ switch (comptime kindOf(T)) {
+ .skip, .void => try c.write(.{ .slot = null }),
+ .bool => try c.write(.{ .uint = @intFromBool(v.*) }),
+ .int => if (@typeInfo(T).int.bits <= 64) {
+ try c.write(if (@typeInfo(T).int.signedness == .signed) .{ .int = v.* } else .{ .uint = v.* });
+ } else {
+ const wide: i128 = @intCast(v.*);
+ try c.write(.{ .bytes_object = .{ .value = std.mem.asBytes(&wide), .format_tag = wide_tag } });
+ },
+ .float => try c.write(.{ .float = v.* }),
+ .@"enum" => if (std.enums.tagName(T, v.*)) |name| {
+ try c.write(.{ .bytes = name });
+ } else try c.write(.{ .int = @intCast(@intFromEnum(v.*)) }),
+ .optional => {
+ const C = @typeInfo(T).optional.child;
+ if (v.*) |*payload| {
+ // An allocator, say, is no state: there or not, which a
+ // load answers with its own.
+ if (comptime kindOf(Canon(C)) == .skip) try c.write(.{ .uint = 1 }) else return w.value(C, payload, c);
+ } else try c.write(.{ .slot = null });
+ },
+ .one => {
+ const C = @typeInfo(T).pointer.child;
+ const key: Key = .{ .addr = @intFromPtr(v.*), .len = @max(1, @sizeOf(C)), .ty = typeWord(T) };
+ if (w.memo.get(key)) |slot| {
+ try c.write(.{ .slot = slot });
+ return slot;
+ }
+ const slot = try w.value(C, v.*, c);
+ try w.memo.put(w.gpa, key, slot);
+ return slot;
+ },
+ .slice => return w.many(@typeInfo(T).pointer.child, v.*, c, true),
+ .array => return w.many(@typeInfo(T).array.child, v, c, false),
+ .prefix => unreachable,
+ .@"struct" => return w.fields(T, v, c),
+ .@"union" => {
+ const map = try Mem.HashMap(.read_write).init(c.*);
+ switch (v.*) {
+ inline else => |*payload, tag| {
+ const name = @tagName(tag);
+ try map.putKey(comptime nameKey(name), .{ .bytes = name });
+ var at = try map.putCursor(comptime nameKey(name));
+ _ = try w.value(@TypeOf(payload.*), payload, &at);
+ },
+ }
+ return map.slot();
+ },
+ // No type a dump walks has them.
+ .err, .error_union => unreachable,
+ .custom => {
+ var buf: [4096]u8 = undefined;
+ var out = try c.writer(&buf);
+ try T.dumpWrite(v, w.gpa, &out.interface);
+ out.format_tag = custom_tag;
+ try out.finish();
+ },
+ .bitset => {
+ const words = (v.bit_length + @bitSizeOf(usize) - 1) / @bitSizeOf(usize);
+ var buf: [64]u8 = undefined;
+ var out = try c.writer(&buf);
+ try out.interface.writeAll(std.mem.asBytes(&@as(u64, v.bit_length)));
+ if (words > 0) try out.interface.writeAll(std.mem.sliceAsBytes(v.masks[0..words]));
+ out.format_tag = bitset_tag;
+ try out.finish();
+ },
+ }
+ return c.slot();
+ }
+
+ /// A run of `E`: numbers as one bytes value (a blob's text by its
+ /// hash), records as a list; a slice met again (`shared`) its first
+ /// value.
+ fn many(w: *Put, comptime E: type, items: []const E, c: *Cursor, comptime shared: bool) anyerror!xitdb.Slot {
+ const key: Key = .{ .addr = @intFromPtr(items.ptr), .len = items.len, .ty = typeWord([]const E) };
+ if (shared and items.len > 0) if (w.memo.get(key)) |slot| {
+ try c.write(.{ .slot = slot });
+ return slot;
+ };
+ const slot = if (comptime numbers(E)) slot: {
+ const bytes = std.mem.sliceAsBytes(items);
+ if (E == u8 and w.in_blob and bytes.len >= blob_min) {
+ var sum: [32]u8 = undefined;
+ std.crypto.hash.Blake3.hash(bytes, &sum, .{});
+ const known = for (w.blobs.items) |b| {
+ if (std.mem.eql(u8, &b.hash, &sum)) break true;
+ } else false;
+ if (!known) try w.blobs.append(w.gpa, .{ .hash = sum, .bytes = bytes });
+ try c.write(.{ .bytes_object = .{ .value = &sum, .format_tag = blob_tag } });
+ } else try c.write(.{ .bytes_object = .{ .value = bytes, .format_tag = numbersTag(E) } });
+ break :slot try c.slot();
+ } else slot: {
+ const list = try Mem.ArrayList(.read_write).init(c.*);
+ for (items) |*e| {
+ var at = try list.appendCursor();
+ _ = try w.value(E, e, &at);
+ }
+ break :slot try list.slot();
+ };
+ if (shared and items.len > 0) try w.memo.put(w.gpa, key, slot);
+ return slot;
+ }
+
+ fn fields(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot {
+ const map = try Mem.HashMap(.read_write).init(c.*);
+ const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed";
+ inline for (comptime dumpFields(T)) |name| {
+ const F = @FieldType(T, name);
+ if (comptime kindOf(Canon(F)) != .void) put: {
+ // Null where the field's default is: no key at all.
+ if (comptime @typeInfo(F) == .optional and defaultIsNull(T, name)) if (@field(v, name) == null) break :put;
+ const was = w.in_blob;
+ w.in_blob = was or comptime blob(T, name);
+ defer w.in_blob = was;
+ try map.putKey(comptime nameKey(name), .{ .bytes = name });
+ var at = try map.putCursor(comptime nameKey(name));
+ if (comptime cutArray(T, name)) {
+ const all = &@field(v, name);
+ _ = try w.many(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))], &at, false);
+ } else if (comptime packed_struct) {
+ const copy: F = @field(v, name);
+ _ = try w.value(F, &copy, &at);
+ } else _ = try w.value(F, &@field(v, name), &at);
+ }
+ }
+ return map.slot();
+ }
+};
+
+fn defaultIsNull(comptime T: type, comptime name: []const u8) bool {
+ const F = @FieldType(T, name);
+ const attr = @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, name).?];
+ const dp = attr.default_value_ptr orelse return false;
+ return @as(*const F, @ptrCast(@alignCast(dp))).* == null;
+}
+
+/// Reads values a `Put` wrote, by name, as another build may have written
+/// them (see above).
+pub const Get = struct {
+ /// Where what is read is made (a `replica.alloc` field's own allocator
+ /// while it is read).
+ gpa: std.mem.Allocator,
+ /// The reader's own lists'.
+ own: std.mem.Allocator,
+ /// The texts its blobs' hashes name (Db.zig's `blobs`), by hash.
+ blobs: ?Mem.HashMap(.read_only) = null,
+ /// Each pointer and slice made so far: what a value met again is, and
+ /// what `rollback` frees.
+ made: std.ArrayList(Made) = .empty,
+ /// Where each value made is: met again, the same one.
+ memo: std.AutoHashMapUnmanaged(u128, usize) = .empty,
+ /// What a type's `dumpRead` made, freed by its `dumpFree` too.
+ customs: std.ArrayList(struct { ptr: *anyopaque, gpa: std.mem.Allocator, free: *const fn (*anyopaque, std.mem.Allocator) void }) = .empty,
+ /// What was let go, for the person whose session it is.
+ notes: std.ArrayList(u8) = .empty,
+ /// The field being read, for a note.
+ field: []const u8 = "",
+ allocs: Allocators = .{},
+
+ const Cursor = Mem.Cursor(.read_only);
+ pub const Error = error{ BadDump, Incompatible, OutOfMemory };
+
+ pub fn init(gpa: std.mem.Allocator) Get {
+ return .{ .gpa = gpa, .own = gpa };
+ }
+
+ pub fn deinit(l: *Get) void {
+ l.made.deinit(l.own);
+ l.memo.deinit(l.own);
+ l.customs.deinit(l.own);
+ l.notes.deinit(l.own);
+ }
+
+ /// Everything read so far freed: a load that fails leaves nothing.
+ /// What it was read into must not be used after (nor freed).
+ pub fn rollback(l: *Get) void {
+ for (l.customs.items) |c| c.free(c.ptr, c.gpa);
+ for (l.made.items) |m| if (m.bytes.len > 0) m.gpa.rawFree(m.bytes, m.alignment, @returnAddress());
+ l.customs.clearRetainingCapacity();
+ l.made.clearRetainingCapacity();
+ l.memo.clearRetainingCapacity();
+ }
+
+ pub fn note(l: *Get, comptime fmt: []const u8, args: anytype) void {
+ if (l.notes.items.len > 4096) return;
+ l.notes.print(l.own, fmt ++ "\n", args) catch {};
+ }
+
+ /// The value at `c`, made: what it points at is `gpa`'s.
+ pub fn get(l: *Get, comptime T: type, c: Cursor) Error!T {
+ var out: T = undefined;
+ try l.fresh(T, &out);
+ try l.value(T, c, &out);
+ return out;
+ }
+
+ /// The struct at `c` read into `out` as it stands: a field the dump
+ /// holds is replaced (what it held is not freed), every other one is
+ /// left as it is, and one another build cannot give is left too, and
+ /// noted.
+ pub fn into(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void {
+ try l.fields(T, c, out, true);
+ }
+
+ /// Where the value of `name` is in the map at `c` (a dump's roots), by
+ /// its name; null when it is not there.
+ pub fn member(c: Cursor, name: []const u8) Error!?Cursor {
+ const map = Mem.HashMap(.read_only).init(c) catch return error.BadDump;
+ var it = map.iterator() catch return error.BadDump;
+ while (it.next() catch return error.BadDump) |kv_at| {
+ const kv = kv_at.readKeyValuePair() catch return error.BadDump;
+ var buf: [64]u8 = undefined;
+ const got = kv.key_cursor.readBytes(&buf) catch continue;
+ if (std.mem.eql(u8, got, name)) return kv.value_cursor;
+ }
+ return null;
+ }
+
+ fn fresh(l: *Get, comptime T: type, out: *T) Error!void {
+ freshOf(T, out, l.gpa);
+ }
+
+ fn slotWord(slot: xitdb.Slot) u128 {
+ return @as(u72, @bitCast(slot));
+ }
+
+ fn slotOf(c: Cursor) Error!xitdb.Slot {
+ return c.slot() catch error.BadDump;
+ }
+
+ /// Whether the value at `slot` has a place of its own (an inline number
+ /// or a short text is its slot alone: no identity to share).
+ fn placed(slot: xitdb.Slot) bool {
+ return switch (slot.tag) {
+ .none, .uint, .int, .float, .short_bytes => false,
+ else => true,
+ };
+ }
+
+ /// An allocation made for the value at `slot`, kept: what a value met
+ /// again is, and what `rollback` frees.
+ fn keep(l: *Get, bytes: []u8, alignment: std.mem.Alignment, slot: xitdb.Slot) Error!usize {
+ try l.made.append(l.own, .{ .bytes = bytes, .alignment = alignment, .gpa = l.gpa });
+ const at = l.made.items.len - 1;
+ // Not kept after all: the caller frees it.
+ if (placed(slot)) l.memo.put(l.own, slotWord(slot), at) catch |err| {
+ _ = l.made.pop();
+ return err;
+ };
+ return at;
+ }
+
+ /// The value made before for the same place, when there was one.
+ fn again(l: *Get, slot: xitdb.Slot) Error!?[]u8 {
+ if (!placed(slot)) return null;
+ const at = l.memo.get(slotWord(slot)) orelse return null;
+ // Made, then let go of by the reader's caller (a dump's mounts).
+ if (at >= l.made.items.len) return null;
+ const m = l.made.items[at];
+ if (m.dead) return error.Incompatible;
+ return m.bytes;
+ }
+
+ /// `T` read from `c` into `out`, which `fresh` has made ready.
+ fn value(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void {
+ if (comptime isAtomic(T)) {
+ const R = @FieldType(T, "raw");
+ var raw: R = undefined;
+ try l.fresh(R, &raw);
+ try l.value(R, c, &raw);
+ out.* = .{ .raw = raw };
+ return;
+ }
+ const slot = try slotOf(c);
+ if (comptime isList(T)) {
+ const got = try l.slice(@typeInfo(@FieldType(T, "items")).pointer.child, c, slot);
+ out.* = .empty;
+ out.items = got;
+ out.capacity = got.len;
+ return;
+ }
+ switch (comptime kindOf(T)) {
+ .skip, .void => {},
+ .bool => out.* = switch (slot.tag) {
+ .uint => slot.value != 0,
+ else => return error.Incompatible,
+ },
+ .int => out.* = try l.int(T, c, slot),
+ .float => out.* = switch (slot.tag) {
+ .float => @floatCast(@as(f64, @bitCast(slot.value))),
+ else => return error.Incompatible,
+ },
+ .@"enum" => {
+ const e = @typeInfo(T).@"enum";
+ switch (slot.tag) {
+ .bytes, .short_bytes => {
+ var buf: [256]u8 = undefined;
+ const name = c.readBytes(&buf) catch return error.Incompatible;
+ inline for (e.field_names) |field| if (std.mem.eql(u8, name, field)) {
+ out.* = @field(T, field);
+ return;
+ };
+ l.note("{s}: no {s} here", .{ l.field, name });
+ return error.Incompatible;
+ },
+ .int, .uint => if (e.mode == .nonexhaustive) {
+ const n: i64 = @bitCast(slot.value);
+ out.* = @enumFromInt(std.math.cast(e.tag_type, n) orelse return error.Incompatible);
+ } else return error.Incompatible,
+ else => return error.Incompatible,
+ }
+ },
+ .optional => {
+ const C = @typeInfo(T).optional.child;
+ if (slot.tag == .none) {
+ out.* = null;
+ return;
+ }
+ if (comptime kindOf(Canon(C)) == .skip) {
+ out.* = if (comptime C == std.mem.Allocator) l.gpa else null;
+ return;
+ }
+ // Through a local, as the byte loader does: a pointer's
+ // null is its address.
+ if (comptime @typeInfo(C) == .pointer) {
+ var payload: C = undefined;
+ try l.value(C, c, &payload);
+ out.* = payload;
+ return;
+ }
+ out.* = @as(C, undefined);
+ try l.fresh(C, &out.*.?);
+ l.value(C, c, &out.*.?) catch |err| {
+ out.* = null;
+ return err;
+ };
+ },
+ .one => {
+ const C = @typeInfo(T).pointer.child;
+ if (try l.again(slot)) |bytes| {
+ out.* = @ptrCast(@alignCast(bytes.ptr));
+ return;
+ }
+ const one = try l.gpa.create(C);
+ const at = l.keep(std.mem.asBytes(one), .of(C), slot) catch |err| {
+ l.gpa.destroy(one);
+ return err;
+ };
+ try l.fresh(C, one);
+ l.value(C, c, one) catch |err| {
+ // Left half made: no later value may be it.
+ l.made.items[at].dead = true;
+ return err;
+ };
+ out.* = one;
+ },
+ .slice => {
+ if (comptime @typeInfo(T).pointer.sentinel_ptr != null) @compileError("walk: a sentinel slice: " ++ @typeName(T));
+ out.* = try l.slice(@typeInfo(T).pointer.child, c, slot);
+ },
+ .array => try l.elements(@typeInfo(T).array.child, c, slot, out),
+ .prefix => unreachable,
+ .@"struct" => try l.fields(T, c, out, false),
+ .@"union" => {
+ const u = @typeInfo(T).@"union";
+ const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible;
+ var it = map.iterator() catch return error.BadDump;
+ const kv_at = (it.next() catch return error.BadDump) orelse return error.Incompatible;
+ const kv = kv_at.readKeyValuePair() catch return error.BadDump;
+ var buf: [256]u8 = undefined;
+ const name = kv.key_cursor.readBytes(&buf) catch return error.Incompatible;
+ inline for (u.field_names, u.field_types) |field, F| if (std.mem.eql(u8, name, field)) {
+ out.* = @unionInit(T, field, undefined);
+ try l.fresh(F, &@field(out.*, field));
+ return l.value(F, kv.value_cursor, &@field(out.*, field));
+ };
+ l.note("{s}: no {s} here", .{ l.field, name });
+ return error.Incompatible;
+ },
+ .err, .error_union => unreachable,
+ .custom => {
+ const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible;
+ defer l.own.free(got.value);
+ T.dumpRead(out, l.gpa, got.value) catch |err| {
+ if (err == error.OutOfMemory) return error.OutOfMemory;
+ l.note("{s}: {t}", .{ l.field, err });
+ return error.Incompatible;
+ };
+ if (comptime @hasDecl(T, "dumpFree")) try l.customs.append(l.own, .{ .ptr = out, .gpa = l.gpa, .free = @ptrCast(&T.dumpFree) });
+ },
+ .bitset => {
+ const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible;
+ defer l.own.free(got.value);
+ if (got.value.len < 8) return error.BadDump;
+ const bits = countOf(std.mem.bytesToValue(u64, got.value[0..8])) catch return error.BadDump;
+ const words = (bits + @bitSizeOf(usize) - 1) / @bitSizeOf(usize);
+ if (got.value.len != 8 + words * @sizeOf(usize)) return error.BadDump;
+ out.* = try .initEmpty(l.gpa, bits);
+ if (words > 0) @memcpy(std.mem.sliceAsBytes(out.masks[0..words]), got.value[8..]);
+ },
+ }
+ }
+
+ fn int(l: *Get, comptime I: type, c: Cursor, slot: xitdb.Slot) Error!I {
+ _ = l;
+ return switch (slot.tag) {
+ .uint => std.math.cast(I, slot.value) orelse error.Incompatible,
+ .int => std.math.cast(I, @as(i64, @bitCast(slot.value))) orelse error.Incompatible,
+ .bytes, .short_bytes => wide: {
+ var buf: [16]u8 = undefined;
+ const got = c.readBytesObject(&buf) catch return error.Incompatible;
+ if (got.value.len != 16 or !std.mem.eql(u8, &(got.format_tag orelse return error.Incompatible), &wide_tag)) return error.Incompatible;
+ break :wide std.math.cast(I, std.mem.bytesToValue(i128, got.value[0..16])) orelse error.Incompatible;
+ },
+ else => error.Incompatible,
+ };
+ }
+
+ /// Numbers' `bytes` (format `tag`) as `E`s into `out`, as many as fit:
+ /// another width of the same kind (another build's) converted one by
+ /// one. How many the bytes hold.
+ fn convert(comptime E: type, bytes: []const u8, tag: [2]u8, out: []E) Error!usize {
+ const width: usize = if (tag[1] >= '1' and tag[1] <= '8') tag[1] - '0' else return error.BadDump;
+ if (bytes.len % width != 0) return error.BadDump;
+ const n = bytes.len / width;
+ const fits = @min(n, out.len);
+ if (std.mem.eql(u8, &tag, &numbersTag(E))) {
+ @memcpy(std.mem.sliceAsBytes(out[0..fits]), bytes[0 .. fits * width]);
+ return n;
+ }
+ if (tag[0] != numbersTag(E)[0]) return error.Incompatible;
+ for (out[0..fits], 0..) |*e, i| {
+ const raw = bytes[i * width ..][0..width];
+ e.* = switch (@typeInfo(E)) {
+ .int => |info| if (info.signedness == .signed) cast: {
+ var wide: i64 = 0;
+ @memcpy(std.mem.asBytes(&wide)[0..width], raw);
+ if (width < 8) {
+ const shift: u6 = @intCast(64 - width * 8);
+ wide = (wide << shift) >> shift;
+ }
+ break :cast std.math.cast(E, wide) orelse return error.Incompatible;
+ } else cast: {
+ var wide: u64 = 0;
+ @memcpy(std.mem.asBytes(&wide)[0..width], raw);
+ break :cast std.math.cast(E, wide) orelse return error.Incompatible;
+ },
+ .float => if (width == 8) @floatCast(std.mem.bytesToValue(f64, raw[0..8])) else if (width == 4) @floatCast(std.mem.bytesToValue(f32, raw[0..4])) else return error.Incompatible,
+ .bool => raw[0] != 0,
+ else => unreachable,
+ };
+ }
+ return n;
+ }
+
+ /// The numbers' bytes at `c` and their format tag, `alloc`'s.
+ fn numbersAt(c: Cursor, alloc: std.mem.Allocator) Error!struct { bytes: []u8, tag: [2]u8 } {
+ const got = c.readBytesObjectAlloc(alloc, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.Incompatible;
+ const bytes: []u8 = @constCast(got.value);
+ const tag = got.format_tag orelse {
+ alloc.free(bytes);
+ return error.Incompatible;
+ };
+ return .{ .bytes = bytes, .tag = tag };
+ }
+
+ /// A slice of `E` at `c`: numbers' bytes, a blob's text by its hash, or
+ /// a list; one met again is the same.
+ fn slice(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot) Error![]E {
+ if (try l.again(slot)) |bytes| return @as([*]E, @ptrCast(@alignCast(bytes.ptr)))[0 .. bytes.len / @max(1, @sizeOf(E))];
+ if (comptime numbers(E)) {
+ switch (slot.tag) {
+ .bytes, .short_bytes => {},
+ else => return error.Incompatible,
+ }
+ if (E == u8) if (try l.blob(c, slot)) |text| return text;
+ // A text, as it is written: its bytes are the slice.
+ if (E == u8) {
+ const got = try numbersAt(c, l.gpa);
+ if (!std.mem.eql(u8, &got.tag, &numbersTag(u8))) {
+ l.gpa.free(got.bytes);
+ return error.Incompatible;
+ }
+ if (got.bytes.len == 0) {
+ l.gpa.free(got.bytes);
+ return &.{};
+ }
+ _ = l.keep(got.bytes, .@"1", slot) catch |err| {
+ l.gpa.free(got.bytes);
+ return err;
+ };
+ return got.bytes;
+ }
+ const got = try numbersAt(c, l.own);
+ defer l.own.free(got.bytes);
+ const width: usize = if (got.tag[1] >= '1' and got.tag[1] <= '8') got.tag[1] - '0' else return error.BadDump;
+ const n = got.bytes.len / width;
+ if (n == 0) return &.{};
+ const many = try l.gpa.alloc(E, n);
+ const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| {
+ l.gpa.free(many);
+ return err;
+ };
+ _ = convert(E, got.bytes, got.tag, many) catch |err| {
+ l.made.items[at].dead = true;
+ return err;
+ };
+ return many;
+ }
+ const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible;
+ const n = countOf(list.count() catch return error.BadDump) catch return error.BadDump;
+ if (n == 0) return &.{};
+ const many = try l.gpa.alloc(E, n);
+ const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| {
+ l.gpa.free(many);
+ return err;
+ };
+ l.listInto(E, list, many) catch |err| {
+ l.made.items[at].dead = true;
+ return err;
+ };
+ return many;
+ }
+
+ /// A blob's text, when `c` holds a blob's hash: made in `gpa`'s.
+ fn blob(l: *Get, c: Cursor, slot: xitdb.Slot) Error!?[]u8 {
+ var buf: [32]u8 = undefined;
+ const got = c.readBytesObject(&buf) catch return null;
+ const tag = got.format_tag orelse return null;
+ if (!std.mem.eql(u8, &tag, &blob_tag) or got.value.len != 32) return null;
+ const texts = l.blobs orelse return error.BadDump;
+ const at = (texts.getCursor(std.mem.readInt(u256, got.value[0..32], .big)) catch return error.BadDump) orelse return error.BadDump;
+ const text = at.readBytesAlloc(l.gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.BadDump;
+ const bytes: []u8 = @constCast(text);
+ _ = l.keep(bytes, .@"1", slot) catch |err| {
+ l.gpa.free(bytes);
+ return err;
+ };
+ return bytes;
+ }
+
+ /// An array of `E` at `c`: as many as it holds, the rest as they are.
+ fn elements(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot, out: []E) Error!void {
+ if (comptime numbers(E)) {
+ switch (slot.tag) {
+ .bytes, .short_bytes => {},
+ else => return error.Incompatible,
+ }
+ const got = try numbersAt(c, l.own);
+ defer l.own.free(got.bytes);
+ _ = try convert(E, got.bytes, got.tag, out);
+ return;
+ }
+ const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible;
+ try l.listInto(E, list, out);
+ }
+
+ /// The list's elements into `out`, as many as fit; an element of an
+ /// array of optional pointers (a session's panes) that cannot be read is
+ /// left out, not the array.
+ fn listInto(l: *Get, comptime E: type, list: Mem.ArrayList(.read_only), out: []E) Error!void {
+ const droppable = comptime @typeInfo(E) == .optional and @typeInfo(@typeInfo(E).optional.child) == .pointer;
+ var failed = false;
+ var it = list.iterator() catch return error.BadDump;
+ var i: usize = 0;
+ while (it.next() catch return error.BadDump) |at| : (i += 1) {
+ if (i >= out.len) break;
+ try l.fresh(E, &out[i]);
+ l.value(E, at, &out[i]) catch |err| {
+ if (err != error.Incompatible) return err;
+ if (droppable) {
+ out[i] = null;
+ l.note("{s}[{d}] left out", .{ l.field, i });
+ } else failed = true;
+ };
+ }
+ if (failed) return error.Incompatible;
+ }
+
+ fn fields(l: *Get, comptime T: type, c: Cursor, out: *T, existing: bool) Error!void {
+ const names = comptime dumpFields(T);
+ const outer = l.field;
+ defer l.field = outer;
+ const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible;
+ var found: std.StaticBitSet(@max(1, names.len)) = .empty;
+ var failed = false;
+ var it = map.iterator() catch return error.BadDump;
+ while (it.next() catch return error.BadDump) |kv_at| {
+ const kv = kv_at.readKeyValuePair() catch return error.BadDump;
+ var buf: [256]u8 = undefined;
+ // A key that is no field's name of this build: passed over.
+ const name = kv.key_cursor.readBytes(&buf) catch continue;
+ const k: ?usize = inline for (names, 0..) |field, k| {
+ if (std.mem.eql(u8, name, field)) break k;
+ } else null;
+ const at = k orelse continue;
+ found.set(at);
+ inline for (names, 0..) |field, i| if (i == at) {
+ l.field = field;
+ l.oneField(T, field, kv.value_cursor, out, existing) catch |err| {
+ if (err != error.Incompatible) return err;
+ const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?];
+ if (comptime attr.default_value_ptr) |dp| {
+ @field(out, field) = @as(*const @FieldType(T, field), @ptrCast(@alignCast(dp))).*;
+ l.note("{s}.{s}: its kind changed; its default taken", .{ @typeName(T), field });
+ } else if (existing) {
+ l.note("{s}.{s}: its kind changed; left as it was", .{ @typeName(T), field });
+ } else failed = true;
+ };
+ };
+ }
+ // A field the writer did not have (or held null at a null default):
+ // its default (`fresh`), or the value cannot be made.
+ inline for (names, 0..) |field, i| if (!found.isSet(i)) {
+ const F = @FieldType(T, field);
+ const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?];
+ if (comptime attr.default_value_ptr == null and kindOf(Canon(F)) != .void) {
+ if (!existing) {
+ l.note("{s}.{s}: not in the dump", .{ @typeName(T), field });
+ failed = true;
+ }
+ } else if (comptime @typeInfo(F) == .optional and defaultIsNull(T, field)) if (existing) {
+ // A null left out: null here too.
+ @field(out, field) = null;
+ };
+ };
+ if (failed) return error.Incompatible;
+ }
+
+ fn oneField(l: *Get, comptime T: type, comptime field: []const u8, c: Cursor, out: *T, existing: bool) Error!void {
+ const F = @FieldType(T, field);
+ const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed";
+ const gpa = l.gpa;
+ defer l.gpa = gpa;
+ if (comptime allocOf(T, field)) |which| if (@field(l.allocs, which)) |a| {
+ l.gpa = a;
+ };
+ // A field this build compiles out (a PDF slot where there is no
+ // MuPDF): passed over, said when it held something.
+ if (comptime kindOf(Canon(F)) == .void) {
+ if ((try slotOf(c)).tag != .none) l.note("{s}: not in this build", .{l.field});
+ return;
+ }
+ if (comptime cutArray(T, field)) {
+ const slot = try slotOf(c);
+ return l.elements(@typeInfo(F).array.child, c, slot, &@field(out, field));
+ }
+ // A struct held in place in one read as it stands (the root's):
+ // what it does not hold stays as it is too.
+ if (comptime kindOf(Canon(F)) == .@"struct" and Canon(F) == F and @typeInfo(F).@"struct".layout != .@"packed") if (existing) {
+ return l.fields(F, c, &@field(out, field), true);
+ };
+ if (packed_struct) {
+ var copy: F = undefined;
+ try l.fresh(F, &copy);
+ try l.value(F, c, &copy);
+ @field(out, field) = copy;
+ return;
+ }
+ try l.fresh(F, &@field(out, field));
+ try l.value(F, c, &@field(out, field));
+ }
+};
+
test "an optional slice comes back, and the fields after it" {
const S = struct { before: u32 = 0, command: ?[]u8 = null, after: u32 = 0 };
var text = "sleep 30".*;