diff options
Diffstat (limited to 'src/walk.zig')
| -rw-r--r-- | src/walk.zig | 808 |
1 files changed, 785 insertions, 23 deletions
diff --git a/src/walk.zig b/src/walk.zig index 426408c8..8360a8b6 100644 --- a/src/walk.zig +++ b/src/walk.zig @@ -1,9 +1,11 @@ //! The reflective walk over the core's state: one comptime `@typeInfo` walk //! that hashes a session into a digest (the replay harness compares two), //! encodes values for the tape, and dumps a session for another build to -//! load by name (`Dump`, `Load`: Restore, -l, the store, a handover, and -//! the join of phase 5). The rollback copy is to walk the same way -//! (docs/next-daemon.md). +//! load by name: into a xitdb database's maps (`Put`, `Get`: a Dump, +//! Restore, -l, an Upgrade, a handover; Db.zig), or as bytes with a table +//! of its types (`Dump`, `Load`: a closed pane, the join of phase 5, and +//! for one release the `.dump` files before). The rollback copy is to walk +//! the same way (docs/next-daemon.md). //! //! Values, never addresses: a pointer is followed to what it points at, a //! slice is its elements, an ArrayList its items, a hash map its entries in @@ -33,6 +35,7 @@ //! digest(*const T, *Hasher) void` of its own hashes itself; one with //! `dumpWrite`, `dumpRead` (and `dumpFree`) dumps itself as bytes. const std = @import("std"); +const xitdb = @import("xitdb"); pub const Hasher = std.hash.Wyhash; const builtin = @import("builtin"); @@ -1331,27 +1334,8 @@ pub fn Load(comptime roots: []const type) type { try l.fields(T, s, out, true); } - /// `out` as a new `T` is before its fields are read: each field's - /// default, an allocator the dump's. fn fresh(l: *Self, comptime T: type, out: *T) DecodeError!void { - if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom) return; - if (comptime kindOf(T) == .skip) return; - const s = @typeInfo(T).@"struct"; - inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| { - if (attr.@"comptime") continue; - if (comptime attr.default_value_ptr) |dp| { - @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*; - } else if (comptime F == std.mem.Allocator) { - @field(out, field) = l.gpa; - } else if (comptime !walkedField(T, field)) { - @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to"); - } - } - } - - fn walkedField(comptime T: type, comptime field: []const u8) bool { - inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true; - return false; + freshOf(T, out, l.gpa); } /// Whether the source's `ty` and `T` are of one kind, else the @@ -1731,6 +1715,28 @@ pub fn Load(comptime roots: []const type) type { }; } +/// `out` as a new `T` is before a load reads its fields: each field's +/// default, an allocator `gpa`. +fn freshOf(comptime T: type, out: *T, gpa: std.mem.Allocator) void { + if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom or kindOf(T) == .skip) return; + const s = @typeInfo(T).@"struct"; + inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| { + if (attr.@"comptime") continue; + if (comptime attr.default_value_ptr) |dp| { + @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*; + } else if (comptime F == std.mem.Allocator) { + @field(out, field) = gpa; + } else if (comptime !walkedField(T, field)) { + @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to"); + } + } +} + +fn walkedField(comptime T: type, comptime field: []const u8) bool { + inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true; + return false; +} + /// A value `save` made, `gpa`'s; what was let go is said in `notes` when /// it is given. pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes: ?*std.ArrayList(u8)) DecodeError!T { @@ -1742,6 +1748,762 @@ pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes: return v; } +// ---- a dump as a database: the same walk, by name, into xitdb's maps ---- +// +// A dump (Db.zig) is a xitdb database in memory, its bytes one value of the +// state file, written by the walk by name: a struct is a map whose keys are +// its fields' names (`putKey`: a reader goes through them by name, never by +// a hash of its own), a union a map of its one field, a slice, an array or +// a list of records a list, of plain numbers one bytes value (its format +// tag the numbers' kind and width), an enum the bytes of its name, a number +// a uint or an int (a wider one its bytes), an optional an absent key (an +// explicit none where the field's default is not null), a type with +// `dumpWrite` its bytes, and a `.blob`'s text of `blob_min` bytes and up +// its BLAKE3 hash (tag "b3"): the text is kept once beside the dump +// (`Blob`). A pointer or a slice met again is the first one's value again: +// what two fields share is shared again. +// +// Another build reads it by name: a field the dump has not takes its +// default, a key this build has not is passed over, a value of another kind +// takes its default and is said (`notes`); with no default the value it is +// part of is refused, and an element of an array of optional pointers (a +// session's panes) is then left out, nothing above it. + +pub const Mem = xitdb.Database(.memory, u256); + +/// A text a dump keeps beside it, once: its BLAKE3 hash and its bytes (the +/// session's, while the dump is made). +pub const Blob = struct { hash: [32]u8, bytes: []const u8 }; + +/// A field's key: its name's word, unique within its struct (a reader +/// matches names, so another build's words need not be these). +fn nameKey(comptime name: []const u8) u256 { + return comptime std.hash.Wyhash.hash(0, name); +} + +/// Whether a run of `E` is one bytes value: plain numbers. +fn numbers(comptime E: type) bool { + return switch (@typeInfo(E)) { + .int => |i| @sizeOf(E) * 8 == i.bits and @sizeOf(E) <= 8, + .bool => true, + .float => |f| f.bits == 32 or f.bits == 64, + else => false, + }; +} + +/// A run of numbers' format tag: their kind and byte width ("u4", "i8", "b1"). +fn numbersTag(comptime E: type) [2]u8 { + const kind: u8 = switch (@typeInfo(E)) { + .int => |i| if (i.signedness == .signed) 'i' else 'u', + .bool => 'b', + .float => 'f', + else => unreachable, + }; + return .{ kind, '0' + @sizeOf(E) }; +} + +const blob_tag = "b3".*; +const wide_tag = "w8".*; +const custom_tag = "cu".*; +const bitset_tag = "bs".*; + +fn typeWord(comptime T: type) u64 { + return comptime std.hash.Wyhash.hash(0, @typeName(T)); +} + +/// Writes values into a database in memory, by name (see above). +pub const Put = struct { + gpa: std.mem.Allocator, + /// Each pointer and slice written, by address, length and type, and + /// where its value is: met again, that value is written again. + memo: std.AutoHashMapUnmanaged(Key, xitdb.Slot) = .empty, + /// The blobs met, each once. + blobs: std.ArrayListUnmanaged(Blob) = .empty, + in_blob: bool = false, + + const Key = struct { addr: usize, len: usize, ty: u64 }; + const Cursor = Mem.Cursor(.read_write); + + pub fn deinit(w: *Put) void { + w.memo.deinit(w.gpa); + w.blobs.deinit(w.gpa); + } + + /// `v` written as `name`'s value in `map` (a dump's roots). + pub fn member(w: *Put, map: Mem.HashMap(.read_write), comptime name: []const u8, comptime T: type, v: *const T) anyerror!void { + try map.putKey(comptime nameKey(name), .{ .bytes = name }); + var at = try map.putCursor(comptime nameKey(name)); + _ = try w.value(T, v, &at); + } + + /// `v` written at `c`; where its value is. + pub fn value(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot { + if (comptime isAtomic(T)) return w.value(@FieldType(T, "raw"), &v.raw, c); + if (comptime isList(T)) return w.many(@typeInfo(@FieldType(T, "items")).pointer.child, v.items, c, true); + switch (comptime kindOf(T)) { + .skip, .void => try c.write(.{ .slot = null }), + .bool => try c.write(.{ .uint = @intFromBool(v.*) }), + .int => if (@typeInfo(T).int.bits <= 64) { + try c.write(if (@typeInfo(T).int.signedness == .signed) .{ .int = v.* } else .{ .uint = v.* }); + } else { + const wide: i128 = @intCast(v.*); + try c.write(.{ .bytes_object = .{ .value = std.mem.asBytes(&wide), .format_tag = wide_tag } }); + }, + .float => try c.write(.{ .float = v.* }), + .@"enum" => if (std.enums.tagName(T, v.*)) |name| { + try c.write(.{ .bytes = name }); + } else try c.write(.{ .int = @intCast(@intFromEnum(v.*)) }), + .optional => { + const C = @typeInfo(T).optional.child; + if (v.*) |*payload| { + // An allocator, say, is no state: there or not, which a + // load answers with its own. + if (comptime kindOf(Canon(C)) == .skip) try c.write(.{ .uint = 1 }) else return w.value(C, payload, c); + } else try c.write(.{ .slot = null }); + }, + .one => { + const C = @typeInfo(T).pointer.child; + const key: Key = .{ .addr = @intFromPtr(v.*), .len = @max(1, @sizeOf(C)), .ty = typeWord(T) }; + if (w.memo.get(key)) |slot| { + try c.write(.{ .slot = slot }); + return slot; + } + const slot = try w.value(C, v.*, c); + try w.memo.put(w.gpa, key, slot); + return slot; + }, + .slice => return w.many(@typeInfo(T).pointer.child, v.*, c, true), + .array => return w.many(@typeInfo(T).array.child, v, c, false), + .prefix => unreachable, + .@"struct" => return w.fields(T, v, c), + .@"union" => { + const map = try Mem.HashMap(.read_write).init(c.*); + switch (v.*) { + inline else => |*payload, tag| { + const name = @tagName(tag); + try map.putKey(comptime nameKey(name), .{ .bytes = name }); + var at = try map.putCursor(comptime nameKey(name)); + _ = try w.value(@TypeOf(payload.*), payload, &at); + }, + } + return map.slot(); + }, + // No type a dump walks has them. + .err, .error_union => unreachable, + .custom => { + var buf: [4096]u8 = undefined; + var out = try c.writer(&buf); + try T.dumpWrite(v, w.gpa, &out.interface); + out.format_tag = custom_tag; + try out.finish(); + }, + .bitset => { + const words = (v.bit_length + @bitSizeOf(usize) - 1) / @bitSizeOf(usize); + var buf: [64]u8 = undefined; + var out = try c.writer(&buf); + try out.interface.writeAll(std.mem.asBytes(&@as(u64, v.bit_length))); + if (words > 0) try out.interface.writeAll(std.mem.sliceAsBytes(v.masks[0..words])); + out.format_tag = bitset_tag; + try out.finish(); + }, + } + return c.slot(); + } + + /// A run of `E`: numbers as one bytes value (a blob's text by its + /// hash), records as a list; a slice met again (`shared`) its first + /// value. + fn many(w: *Put, comptime E: type, items: []const E, c: *Cursor, comptime shared: bool) anyerror!xitdb.Slot { + const key: Key = .{ .addr = @intFromPtr(items.ptr), .len = items.len, .ty = typeWord([]const E) }; + if (shared and items.len > 0) if (w.memo.get(key)) |slot| { + try c.write(.{ .slot = slot }); + return slot; + }; + const slot = if (comptime numbers(E)) slot: { + const bytes = std.mem.sliceAsBytes(items); + if (E == u8 and w.in_blob and bytes.len >= blob_min) { + var sum: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(bytes, &sum, .{}); + const known = for (w.blobs.items) |b| { + if (std.mem.eql(u8, &b.hash, &sum)) break true; + } else false; + if (!known) try w.blobs.append(w.gpa, .{ .hash = sum, .bytes = bytes }); + try c.write(.{ .bytes_object = .{ .value = &sum, .format_tag = blob_tag } }); + } else try c.write(.{ .bytes_object = .{ .value = bytes, .format_tag = numbersTag(E) } }); + break :slot try c.slot(); + } else slot: { + const list = try Mem.ArrayList(.read_write).init(c.*); + for (items) |*e| { + var at = try list.appendCursor(); + _ = try w.value(E, e, &at); + } + break :slot try list.slot(); + }; + if (shared and items.len > 0) try w.memo.put(w.gpa, key, slot); + return slot; + } + + fn fields(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot { + const map = try Mem.HashMap(.read_write).init(c.*); + const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed"; + inline for (comptime dumpFields(T)) |name| { + const F = @FieldType(T, name); + if (comptime kindOf(Canon(F)) != .void) put: { + // Null where the field's default is: no key at all. + if (comptime @typeInfo(F) == .optional and defaultIsNull(T, name)) if (@field(v, name) == null) break :put; + const was = w.in_blob; + w.in_blob = was or comptime blob(T, name); + defer w.in_blob = was; + try map.putKey(comptime nameKey(name), .{ .bytes = name }); + var at = try map.putCursor(comptime nameKey(name)); + if (comptime cutArray(T, name)) { + const all = &@field(v, name); + _ = try w.many(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))], &at, false); + } else if (comptime packed_struct) { + const copy: F = @field(v, name); + _ = try w.value(F, ©, &at); + } else _ = try w.value(F, &@field(v, name), &at); + } + } + return map.slot(); + } +}; + +fn defaultIsNull(comptime T: type, comptime name: []const u8) bool { + const F = @FieldType(T, name); + const attr = @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, name).?]; + const dp = attr.default_value_ptr orelse return false; + return @as(*const F, @ptrCast(@alignCast(dp))).* == null; +} + +/// Reads values a `Put` wrote, by name, as another build may have written +/// them (see above). +pub const Get = struct { + /// Where what is read is made (a `replica.alloc` field's own allocator + /// while it is read). + gpa: std.mem.Allocator, + /// The reader's own lists'. + own: std.mem.Allocator, + /// The texts its blobs' hashes name (Db.zig's `blobs`), by hash. + blobs: ?Mem.HashMap(.read_only) = null, + /// Each pointer and slice made so far: what a value met again is, and + /// what `rollback` frees. + made: std.ArrayList(Made) = .empty, + /// Where each value made is: met again, the same one. + memo: std.AutoHashMapUnmanaged(u128, usize) = .empty, + /// What a type's `dumpRead` made, freed by its `dumpFree` too. + customs: std.ArrayList(struct { ptr: *anyopaque, gpa: std.mem.Allocator, free: *const fn (*anyopaque, std.mem.Allocator) void }) = .empty, + /// What was let go, for the person whose session it is. + notes: std.ArrayList(u8) = .empty, + /// The field being read, for a note. + field: []const u8 = "", + allocs: Allocators = .{}, + + const Cursor = Mem.Cursor(.read_only); + pub const Error = error{ BadDump, Incompatible, OutOfMemory }; + + pub fn init(gpa: std.mem.Allocator) Get { + return .{ .gpa = gpa, .own = gpa }; + } + + pub fn deinit(l: *Get) void { + l.made.deinit(l.own); + l.memo.deinit(l.own); + l.customs.deinit(l.own); + l.notes.deinit(l.own); + } + + /// Everything read so far freed: a load that fails leaves nothing. + /// What it was read into must not be used after (nor freed). + pub fn rollback(l: *Get) void { + for (l.customs.items) |c| c.free(c.ptr, c.gpa); + for (l.made.items) |m| if (m.bytes.len > 0) m.gpa.rawFree(m.bytes, m.alignment, @returnAddress()); + l.customs.clearRetainingCapacity(); + l.made.clearRetainingCapacity(); + l.memo.clearRetainingCapacity(); + } + + pub fn note(l: *Get, comptime fmt: []const u8, args: anytype) void { + if (l.notes.items.len > 4096) return; + l.notes.print(l.own, fmt ++ "\n", args) catch {}; + } + + /// The value at `c`, made: what it points at is `gpa`'s. + pub fn get(l: *Get, comptime T: type, c: Cursor) Error!T { + var out: T = undefined; + try l.fresh(T, &out); + try l.value(T, c, &out); + return out; + } + + /// The struct at `c` read into `out` as it stands: a field the dump + /// holds is replaced (what it held is not freed), every other one is + /// left as it is, and one another build cannot give is left too, and + /// noted. + pub fn into(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void { + try l.fields(T, c, out, true); + } + + /// Where the value of `name` is in the map at `c` (a dump's roots), by + /// its name; null when it is not there. + pub fn member(c: Cursor, name: []const u8) Error!?Cursor { + const map = Mem.HashMap(.read_only).init(c) catch return error.BadDump; + var it = map.iterator() catch return error.BadDump; + while (it.next() catch return error.BadDump) |kv_at| { + const kv = kv_at.readKeyValuePair() catch return error.BadDump; + var buf: [64]u8 = undefined; + const got = kv.key_cursor.readBytes(&buf) catch continue; + if (std.mem.eql(u8, got, name)) return kv.value_cursor; + } + return null; + } + + fn fresh(l: *Get, comptime T: type, out: *T) Error!void { + freshOf(T, out, l.gpa); + } + + fn slotWord(slot: xitdb.Slot) u128 { + return @as(u72, @bitCast(slot)); + } + + fn slotOf(c: Cursor) Error!xitdb.Slot { + return c.slot() catch error.BadDump; + } + + /// Whether the value at `slot` has a place of its own (an inline number + /// or a short text is its slot alone: no identity to share). + fn placed(slot: xitdb.Slot) bool { + return switch (slot.tag) { + .none, .uint, .int, .float, .short_bytes => false, + else => true, + }; + } + + /// An allocation made for the value at `slot`, kept: what a value met + /// again is, and what `rollback` frees. + fn keep(l: *Get, bytes: []u8, alignment: std.mem.Alignment, slot: xitdb.Slot) Error!usize { + try l.made.append(l.own, .{ .bytes = bytes, .alignment = alignment, .gpa = l.gpa }); + const at = l.made.items.len - 1; + // Not kept after all: the caller frees it. + if (placed(slot)) l.memo.put(l.own, slotWord(slot), at) catch |err| { + _ = l.made.pop(); + return err; + }; + return at; + } + + /// The value made before for the same place, when there was one. + fn again(l: *Get, slot: xitdb.Slot) Error!?[]u8 { + if (!placed(slot)) return null; + const at = l.memo.get(slotWord(slot)) orelse return null; + // Made, then let go of by the reader's caller (a dump's mounts). + if (at >= l.made.items.len) return null; + const m = l.made.items[at]; + if (m.dead) return error.Incompatible; + return m.bytes; + } + + /// `T` read from `c` into `out`, which `fresh` has made ready. + fn value(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void { + if (comptime isAtomic(T)) { + const R = @FieldType(T, "raw"); + var raw: R = undefined; + try l.fresh(R, &raw); + try l.value(R, c, &raw); + out.* = .{ .raw = raw }; + return; + } + const slot = try slotOf(c); + if (comptime isList(T)) { + const got = try l.slice(@typeInfo(@FieldType(T, "items")).pointer.child, c, slot); + out.* = .empty; + out.items = got; + out.capacity = got.len; + return; + } + switch (comptime kindOf(T)) { + .skip, .void => {}, + .bool => out.* = switch (slot.tag) { + .uint => slot.value != 0, + else => return error.Incompatible, + }, + .int => out.* = try l.int(T, c, slot), + .float => out.* = switch (slot.tag) { + .float => @floatCast(@as(f64, @bitCast(slot.value))), + else => return error.Incompatible, + }, + .@"enum" => { + const e = @typeInfo(T).@"enum"; + switch (slot.tag) { + .bytes, .short_bytes => { + var buf: [256]u8 = undefined; + const name = c.readBytes(&buf) catch return error.Incompatible; + inline for (e.field_names) |field| if (std.mem.eql(u8, name, field)) { + out.* = @field(T, field); + return; + }; + l.note("{s}: no {s} here", .{ l.field, name }); + return error.Incompatible; + }, + .int, .uint => if (e.mode == .nonexhaustive) { + const n: i64 = @bitCast(slot.value); + out.* = @enumFromInt(std.math.cast(e.tag_type, n) orelse return error.Incompatible); + } else return error.Incompatible, + else => return error.Incompatible, + } + }, + .optional => { + const C = @typeInfo(T).optional.child; + if (slot.tag == .none) { + out.* = null; + return; + } + if (comptime kindOf(Canon(C)) == .skip) { + out.* = if (comptime C == std.mem.Allocator) l.gpa else null; + return; + } + // Through a local, as the byte loader does: a pointer's + // null is its address. + if (comptime @typeInfo(C) == .pointer) { + var payload: C = undefined; + try l.value(C, c, &payload); + out.* = payload; + return; + } + out.* = @as(C, undefined); + try l.fresh(C, &out.*.?); + l.value(C, c, &out.*.?) catch |err| { + out.* = null; + return err; + }; + }, + .one => { + const C = @typeInfo(T).pointer.child; + if (try l.again(slot)) |bytes| { + out.* = @ptrCast(@alignCast(bytes.ptr)); + return; + } + const one = try l.gpa.create(C); + const at = l.keep(std.mem.asBytes(one), .of(C), slot) catch |err| { + l.gpa.destroy(one); + return err; + }; + try l.fresh(C, one); + l.value(C, c, one) catch |err| { + // Left half made: no later value may be it. + l.made.items[at].dead = true; + return err; + }; + out.* = one; + }, + .slice => { + if (comptime @typeInfo(T).pointer.sentinel_ptr != null) @compileError("walk: a sentinel slice: " ++ @typeName(T)); + out.* = try l.slice(@typeInfo(T).pointer.child, c, slot); + }, + .array => try l.elements(@typeInfo(T).array.child, c, slot, out), + .prefix => unreachable, + .@"struct" => try l.fields(T, c, out, false), + .@"union" => { + const u = @typeInfo(T).@"union"; + const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible; + var it = map.iterator() catch return error.BadDump; + const kv_at = (it.next() catch return error.BadDump) orelse return error.Incompatible; + const kv = kv_at.readKeyValuePair() catch return error.BadDump; + var buf: [256]u8 = undefined; + const name = kv.key_cursor.readBytes(&buf) catch return error.Incompatible; + inline for (u.field_names, u.field_types) |field, F| if (std.mem.eql(u8, name, field)) { + out.* = @unionInit(T, field, undefined); + try l.fresh(F, &@field(out.*, field)); + return l.value(F, kv.value_cursor, &@field(out.*, field)); + }; + l.note("{s}: no {s} here", .{ l.field, name }); + return error.Incompatible; + }, + .err, .error_union => unreachable, + .custom => { + const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible; + defer l.own.free(got.value); + T.dumpRead(out, l.gpa, got.value) catch |err| { + if (err == error.OutOfMemory) return error.OutOfMemory; + l.note("{s}: {t}", .{ l.field, err }); + return error.Incompatible; + }; + if (comptime @hasDecl(T, "dumpFree")) try l.customs.append(l.own, .{ .ptr = out, .gpa = l.gpa, .free = @ptrCast(&T.dumpFree) }); + }, + .bitset => { + const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible; + defer l.own.free(got.value); + if (got.value.len < 8) return error.BadDump; + const bits = countOf(std.mem.bytesToValue(u64, got.value[0..8])) catch return error.BadDump; + const words = (bits + @bitSizeOf(usize) - 1) / @bitSizeOf(usize); + if (got.value.len != 8 + words * @sizeOf(usize)) return error.BadDump; + out.* = try .initEmpty(l.gpa, bits); + if (words > 0) @memcpy(std.mem.sliceAsBytes(out.masks[0..words]), got.value[8..]); + }, + } + } + + fn int(l: *Get, comptime I: type, c: Cursor, slot: xitdb.Slot) Error!I { + _ = l; + return switch (slot.tag) { + .uint => std.math.cast(I, slot.value) orelse error.Incompatible, + .int => std.math.cast(I, @as(i64, @bitCast(slot.value))) orelse error.Incompatible, + .bytes, .short_bytes => wide: { + var buf: [16]u8 = undefined; + const got = c.readBytesObject(&buf) catch return error.Incompatible; + if (got.value.len != 16 or !std.mem.eql(u8, &(got.format_tag orelse return error.Incompatible), &wide_tag)) return error.Incompatible; + break :wide std.math.cast(I, std.mem.bytesToValue(i128, got.value[0..16])) orelse error.Incompatible; + }, + else => error.Incompatible, + }; + } + + /// Numbers' `bytes` (format `tag`) as `E`s into `out`, as many as fit: + /// another width of the same kind (another build's) converted one by + /// one. How many the bytes hold. + fn convert(comptime E: type, bytes: []const u8, tag: [2]u8, out: []E) Error!usize { + const width: usize = if (tag[1] >= '1' and tag[1] <= '8') tag[1] - '0' else return error.BadDump; + if (bytes.len % width != 0) return error.BadDump; + const n = bytes.len / width; + const fits = @min(n, out.len); + if (std.mem.eql(u8, &tag, &numbersTag(E))) { + @memcpy(std.mem.sliceAsBytes(out[0..fits]), bytes[0 .. fits * width]); + return n; + } + if (tag[0] != numbersTag(E)[0]) return error.Incompatible; + for (out[0..fits], 0..) |*e, i| { + const raw = bytes[i * width ..][0..width]; + e.* = switch (@typeInfo(E)) { + .int => |info| if (info.signedness == .signed) cast: { + var wide: i64 = 0; + @memcpy(std.mem.asBytes(&wide)[0..width], raw); + if (width < 8) { + const shift: u6 = @intCast(64 - width * 8); + wide = (wide << shift) >> shift; + } + break :cast std.math.cast(E, wide) orelse return error.Incompatible; + } else cast: { + var wide: u64 = 0; + @memcpy(std.mem.asBytes(&wide)[0..width], raw); + break :cast std.math.cast(E, wide) orelse return error.Incompatible; + }, + .float => if (width == 8) @floatCast(std.mem.bytesToValue(f64, raw[0..8])) else if (width == 4) @floatCast(std.mem.bytesToValue(f32, raw[0..4])) else return error.Incompatible, + .bool => raw[0] != 0, + else => unreachable, + }; + } + return n; + } + + /// The numbers' bytes at `c` and their format tag, `alloc`'s. + fn numbersAt(c: Cursor, alloc: std.mem.Allocator) Error!struct { bytes: []u8, tag: [2]u8 } { + const got = c.readBytesObjectAlloc(alloc, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.Incompatible; + const bytes: []u8 = @constCast(got.value); + const tag = got.format_tag orelse { + alloc.free(bytes); + return error.Incompatible; + }; + return .{ .bytes = bytes, .tag = tag }; + } + + /// A slice of `E` at `c`: numbers' bytes, a blob's text by its hash, or + /// a list; one met again is the same. + fn slice(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot) Error![]E { + if (try l.again(slot)) |bytes| return @as([*]E, @ptrCast(@alignCast(bytes.ptr)))[0 .. bytes.len / @max(1, @sizeOf(E))]; + if (comptime numbers(E)) { + switch (slot.tag) { + .bytes, .short_bytes => {}, + else => return error.Incompatible, + } + if (E == u8) if (try l.blob(c, slot)) |text| return text; + // A text, as it is written: its bytes are the slice. + if (E == u8) { + const got = try numbersAt(c, l.gpa); + if (!std.mem.eql(u8, &got.tag, &numbersTag(u8))) { + l.gpa.free(got.bytes); + return error.Incompatible; + } + if (got.bytes.len == 0) { + l.gpa.free(got.bytes); + return &.{}; + } + _ = l.keep(got.bytes, .@"1", slot) catch |err| { + l.gpa.free(got.bytes); + return err; + }; + return got.bytes; + } + const got = try numbersAt(c, l.own); + defer l.own.free(got.bytes); + const width: usize = if (got.tag[1] >= '1' and got.tag[1] <= '8') got.tag[1] - '0' else return error.BadDump; + const n = got.bytes.len / width; + if (n == 0) return &.{}; + const many = try l.gpa.alloc(E, n); + const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| { + l.gpa.free(many); + return err; + }; + _ = convert(E, got.bytes, got.tag, many) catch |err| { + l.made.items[at].dead = true; + return err; + }; + return many; + } + const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible; + const n = countOf(list.count() catch return error.BadDump) catch return error.BadDump; + if (n == 0) return &.{}; + const many = try l.gpa.alloc(E, n); + const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| { + l.gpa.free(many); + return err; + }; + l.listInto(E, list, many) catch |err| { + l.made.items[at].dead = true; + return err; + }; + return many; + } + + /// A blob's text, when `c` holds a blob's hash: made in `gpa`'s. + fn blob(l: *Get, c: Cursor, slot: xitdb.Slot) Error!?[]u8 { + var buf: [32]u8 = undefined; + const got = c.readBytesObject(&buf) catch return null; + const tag = got.format_tag orelse return null; + if (!std.mem.eql(u8, &tag, &blob_tag) or got.value.len != 32) return null; + const texts = l.blobs orelse return error.BadDump; + const at = (texts.getCursor(std.mem.readInt(u256, got.value[0..32], .big)) catch return error.BadDump) orelse return error.BadDump; + const text = at.readBytesAlloc(l.gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.BadDump; + const bytes: []u8 = @constCast(text); + _ = l.keep(bytes, .@"1", slot) catch |err| { + l.gpa.free(bytes); + return err; + }; + return bytes; + } + + /// An array of `E` at `c`: as many as it holds, the rest as they are. + fn elements(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot, out: []E) Error!void { + if (comptime numbers(E)) { + switch (slot.tag) { + .bytes, .short_bytes => {}, + else => return error.Incompatible, + } + const got = try numbersAt(c, l.own); + defer l.own.free(got.bytes); + _ = try convert(E, got.bytes, got.tag, out); + return; + } + const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible; + try l.listInto(E, list, out); + } + + /// The list's elements into `out`, as many as fit; an element of an + /// array of optional pointers (a session's panes) that cannot be read is + /// left out, not the array. + fn listInto(l: *Get, comptime E: type, list: Mem.ArrayList(.read_only), out: []E) Error!void { + const droppable = comptime @typeInfo(E) == .optional and @typeInfo(@typeInfo(E).optional.child) == .pointer; + var failed = false; + var it = list.iterator() catch return error.BadDump; + var i: usize = 0; + while (it.next() catch return error.BadDump) |at| : (i += 1) { + if (i >= out.len) break; + try l.fresh(E, &out[i]); + l.value(E, at, &out[i]) catch |err| { + if (err != error.Incompatible) return err; + if (droppable) { + out[i] = null; + l.note("{s}[{d}] left out", .{ l.field, i }); + } else failed = true; + }; + } + if (failed) return error.Incompatible; + } + + fn fields(l: *Get, comptime T: type, c: Cursor, out: *T, existing: bool) Error!void { + const names = comptime dumpFields(T); + const outer = l.field; + defer l.field = outer; + const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible; + var found: std.StaticBitSet(@max(1, names.len)) = .empty; + var failed = false; + var it = map.iterator() catch return error.BadDump; + while (it.next() catch return error.BadDump) |kv_at| { + const kv = kv_at.readKeyValuePair() catch return error.BadDump; + var buf: [256]u8 = undefined; + // A key that is no field's name of this build: passed over. + const name = kv.key_cursor.readBytes(&buf) catch continue; + const k: ?usize = inline for (names, 0..) |field, k| { + if (std.mem.eql(u8, name, field)) break k; + } else null; + const at = k orelse continue; + found.set(at); + inline for (names, 0..) |field, i| if (i == at) { + l.field = field; + l.oneField(T, field, kv.value_cursor, out, existing) catch |err| { + if (err != error.Incompatible) return err; + const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?]; + if (comptime attr.default_value_ptr) |dp| { + @field(out, field) = @as(*const @FieldType(T, field), @ptrCast(@alignCast(dp))).*; + l.note("{s}.{s}: its kind changed; its default taken", .{ @typeName(T), field }); + } else if (existing) { + l.note("{s}.{s}: its kind changed; left as it was", .{ @typeName(T), field }); + } else failed = true; + }; + }; + } + // A field the writer did not have (or held null at a null default): + // its default (`fresh`), or the value cannot be made. + inline for (names, 0..) |field, i| if (!found.isSet(i)) { + const F = @FieldType(T, field); + const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?]; + if (comptime attr.default_value_ptr == null and kindOf(Canon(F)) != .void) { + if (!existing) { + l.note("{s}.{s}: not in the dump", .{ @typeName(T), field }); + failed = true; + } + } else if (comptime @typeInfo(F) == .optional and defaultIsNull(T, field)) if (existing) { + // A null left out: null here too. + @field(out, field) = null; + }; + }; + if (failed) return error.Incompatible; + } + + fn oneField(l: *Get, comptime T: type, comptime field: []const u8, c: Cursor, out: *T, existing: bool) Error!void { + const F = @FieldType(T, field); + const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed"; + const gpa = l.gpa; + defer l.gpa = gpa; + if (comptime allocOf(T, field)) |which| if (@field(l.allocs, which)) |a| { + l.gpa = a; + }; + // A field this build compiles out (a PDF slot where there is no + // MuPDF): passed over, said when it held something. + if (comptime kindOf(Canon(F)) == .void) { + if ((try slotOf(c)).tag != .none) l.note("{s}: not in this build", .{l.field}); + return; + } + if (comptime cutArray(T, field)) { + const slot = try slotOf(c); + return l.elements(@typeInfo(F).array.child, c, slot, &@field(out, field)); + } + // A struct held in place in one read as it stands (the root's): + // what it does not hold stays as it is too. + if (comptime kindOf(Canon(F)) == .@"struct" and Canon(F) == F and @typeInfo(F).@"struct".layout != .@"packed") if (existing) { + return l.fields(F, c, &@field(out, field), true); + }; + if (packed_struct) { + var copy: F = undefined; + try l.fresh(F, ©); + try l.value(F, c, ©); + @field(out, field) = copy; + return; + } + try l.fresh(F, &@field(out, field)); + try l.value(F, c, &@field(out, field)); + } +}; + test "an optional slice comes back, and the fields after it" { const S = struct { before: u32 = 0, command: ?[]u8 = null, after: u32 = 0 }; var text = "sleep 30".*; |
