summaryrefslogtreecommitdiff
path: root/src/host/memory.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/host/memory.zig')
-rw-r--r--src/host/memory.zig482
1 files changed, 442 insertions, 40 deletions
diff --git a/src/host/memory.zig b/src/host/memory.zig
index fc44e50..5fbb88f 100644
--- a/src/host/memory.zig
+++ b/src/host/memory.zig
@@ -20,7 +20,11 @@ pub const page = std.heap.page_size_min;
/// Slot 0 is live; 1..7 are snapshots.
pub const slot_count = 8;
pub const header_len = 64 * 1024;
-pub const file_magic = "SKETCH01".*;
+/// Layouts with element leaves (`list[].x`, `StoredField` with `dims`).
+pub const file_magic = "SKETCH02".*;
+/// What hosts before element leaves wrote: arrays are whole leaves
+/// (`StoredFieldV1`). Read, and rewritten as `file_magic` on adopt.
+pub const file_magic_v1 = "SKETCH01".*;
pub const Header = extern struct {
magic: [8]u8 = file_magic,
@@ -36,7 +40,8 @@ pub const Header = extern struct {
reserved: u32 = 0,
};
-/// A leaf as stored in the file: strings are offsets into the string table.
+/// A leaf as stored in the file: strings are offsets into the string table,
+/// and so is `dims`, `rank` little-endian (count, stride) u32 pairs.
pub const StoredField = extern struct {
offset: u32,
size: u32,
@@ -46,9 +51,27 @@ pub const StoredField = extern struct {
tags_len: u16,
tags: u32,
kind: abi.Field.Kind,
+ rank: u8 = 0,
+ reserved: [2]u8 = @splat(0),
+ dims: u32 = 0,
+ reserved2: u32 = 0,
+};
+
+/// A leaf as `file_magic_v1` files store it: no element leaves.
+pub const StoredFieldV1 = extern struct {
+ offset: u32,
+ size: u32,
+ hash: u64,
+ path: u32,
+ path_len: u16,
+ tags_len: u16,
+ tags: u32,
+ kind: abi.Field.Kind,
reserved: [3]u8 = @splat(0),
};
+const Dim = abi.Field.Dim;
+
/// A layout the host owns: copied out of a sketch, or read from a file.
pub const Layout = struct {
size: usize = 0,
@@ -56,6 +79,8 @@ pub const Layout = struct {
hash: u64 = 0,
fields: []const Leaf = &.{},
+ /// A leaf with `dims` stands for all its elements (`list[].x`), at
+ /// `offset` plus index times stride per dimension. `find` resolves one.
pub const Leaf = struct {
path: []const u8,
offset: u32,
@@ -63,10 +88,38 @@ pub const Layout = struct {
kind: abi.Field.Kind,
hash: u64,
tags: []const u8,
+ dims: []const Dim = &.{},
pub fn bytes(leaf: Leaf, memory: []const u8) []const u8 {
return memory[leaf.offset..][0..leaf.size];
}
+ /// The leaf at `offset` (from `elements`), as a leaf of its own.
+ pub fn at(leaf: Leaf, offset: usize) Leaf {
+ var one = leaf;
+ one.offset = @intCast(offset);
+ one.dims = &.{};
+ return one;
+ }
+ /// A whole array, kept for copying in one piece; its elements are
+ /// leaves of their own.
+ pub fn whole(leaf: Leaf) bool {
+ return leaf.kind == .array;
+ }
+ /// Offsets of every element, first index slowest.
+ pub fn elements(leaf: Leaf) Elements {
+ return .init(leaf.offset, leaf.dims, leaf.dims, null);
+ }
+ /// Writes the path naming the element at `offset`, indices filled in.
+ pub fn name(leaf: Leaf, offset: usize, w: *std.Io.Writer) std.Io.Writer.Error!void {
+ var rest = offset - leaf.offset;
+ var parts = std.mem.splitSequence(u8, leaf.path, "[]");
+ try w.writeAll(parts.first());
+ for (leaf.dims) |d| {
+ try w.print("[{d}]", .{rest / d.stride});
+ rest %= d.stride;
+ try w.writeAll(parts.next() orelse "");
+ }
+ }
};
pub fn copy(arena: std.mem.Allocator, l: abi.Layout) !Layout {
@@ -78,37 +131,149 @@ pub const Layout = struct {
.kind = from.kind,
.hash = from.hash,
.tags = try arena.dupe(u8, std.mem.span(from.tags)),
+ .dims = try arena.dupe(Dim, from.dims[0..from.rank]),
};
return .{ .size = l.size, .alignment = l.alignment, .hash = l.hash, .fields = fields };
}
+ /// The leaf `path` names, with indices for every array it is in
+ /// (`list[3].pos[0]`), as a leaf of its own; null when there is none
+ /// or an index is out of range.
pub fn find(self: Layout, path: []const u8) ?Leaf {
+ for (self.fields) |leaf| {
+ const delta = abi.layout.match(leaf.path, leaf.dims, path) orelse continue;
+ return leaf.at(leaf.offset + delta);
+ }
+ return null;
+ }
+
+ /// The leaf whose pattern is exactly `pattern` (`list[].x`).
+ pub fn pattern(self: Layout, path: []const u8) ?Leaf {
for (self.fields) |leaf| if (std.mem.eql(u8, leaf.path, path)) return leaf;
return null;
}
};
+/// Walks element offsets of one leaf, or of the elements two layouts of a
+/// leaf share (`limit`, the smaller count per dimension), with offsets in
+/// both.
+pub const Elements = struct {
+ index: [abi.layout.max_rank]u32 = @splat(0),
+ limit: [abi.layout.max_rank]u32 = @splat(0),
+ rank: usize,
+ a: Side,
+ b: Side,
+ done: bool,
+
+ const Side = struct { base: usize, dims: []const Dim };
+
+ pub fn init(base: usize, dims: []const Dim, other_dims: []const Dim, other_base: ?usize) Elements {
+ var e: Elements = .{ .rank = dims.len, .a = .{ .base = base, .dims = dims }, .b = .{ .base = other_base orelse base, .dims = other_dims }, .done = false };
+ for (dims, other_dims, 0..) |x, y, i| {
+ e.limit[i] = @min(x.count, y.count);
+ if (e.limit[i] == 0) e.done = true;
+ }
+ return e;
+ }
+
+ fn offset(e: *const Elements, side: Side) usize {
+ var at = side.base;
+ for (side.dims, 0..) |d, i| at += @as(usize, e.index[i]) * d.stride;
+ return at;
+ }
+
+ pub fn next(e: *Elements) ?usize {
+ const pair = e.nextPair() orelse return null;
+ return pair[0];
+ }
+
+ pub fn nextPair(e: *Elements) ?[2]usize {
+ if (e.done) return null;
+ const pair: [2]usize = .{ e.offset(e.a), e.offset(e.b) };
+ var i = e.rank;
+ while (true) {
+ if (i == 0) {
+ e.done = true;
+ break;
+ }
+ i -= 1;
+ e.index[i] += 1;
+ if (e.index[i] < e.limit[i]) break;
+ e.index[i] = 0;
+ }
+ return pair;
+ }
+};
+
pub const Migration = struct { kept: usize = 0, reset: usize = 0 };
/// Copies every leaf of `from` whose path and type are unchanged into `to`,
-/// which already holds the new defaults.
+/// which already holds the new defaults. Arrays copy element by element:
+/// the elements both layouts have (the common prefix of each dimension; a
+/// shrink drops the rest, a growth keeps the new defaults), field by field
+/// inside each, enums by tag name. An array whose type is unchanged copies
+/// in one piece. Counts are of leaves, not elements.
pub fn migrate(from: Layout, old: []const u8, to: Layout, new: []u8) Migration {
var result: Migration = .{};
+ // The array copied whole most recently: its elements' leaves follow it.
+ var copied: ?[]const u8 = null;
for (to.fields) |leaf| {
- const before = from.find(leaf.path) orelse {
- result.reset += 1;
+ if (copied) |prefix| {
+ if (leaf.path.len > prefix.len and std.mem.startsWith(u8, leaf.path, prefix) and leaf.path[prefix.len] == '[') continue;
+ copied = null;
+ }
+ const before = from.pattern(leaf.path) orelse {
+ if (!leaf.whole()) result.reset += 1;
continue;
};
+ if (before.dims.len != leaf.dims.len) {
+ if (!leaf.whole()) result.reset += 1;
+ continue;
+ }
if (before.hash == leaf.hash and before.size == leaf.size) {
- @memcpy(new[leaf.offset..][0..leaf.size], before.bytes(old));
+ var it: Elements = .init(leaf.offset, leaf.dims, before.dims, before.offset);
+ while (it.nextPair()) |pair| @memcpy(new[pair[0]..][0..leaf.size], old[pair[1]..][0..leaf.size]);
+ if (leaf.whole()) copied = leaf.path;
result.kept += 1;
- } else if (before.kind == .@"enum" and leaf.kind == .@"enum" and remapEnum(before, old, leaf, new)) {
- result.kept += 1;
- } else result.reset += 1;
+ } else if (before.kind == .@"enum" and leaf.kind == .@"enum") {
+ var it: Elements = .init(leaf.offset, leaf.dims, before.dims, before.offset);
+ var any = false;
+ var seen: usize = 0;
+ while (it.nextPair()) |pair| : (seen += 1) {
+ if (remapEnum(before.at(pair[1]), old, leaf.at(pair[0]), new)) any = true;
+ }
+ if (any or seen == 0) result.kept += 1 else result.reset += 1;
+ } else if (!leaf.whole()) result.reset += 1;
}
return result;
}
+pub const Diff = struct {
+ /// Leaf values (elements counted one by one) that differ.
+ count: usize = 0,
+ first: ?Place = null,
+ last: ?Place = null,
+
+ pub const Place = struct { leaf: Layout.Leaf, offset: usize };
+};
+
+/// Compares two copies of state leaf by leaf, so padding (a Debug
+/// `@Vector`'s spare lane, the gaps between fields) never differs.
+pub fn compare(layout: Layout, a: []const u8, b: []const u8) Diff {
+ var diff: Diff = .{};
+ for (layout.fields) |leaf| {
+ if (leaf.whole()) continue;
+ var it = leaf.elements();
+ while (it.next()) |offset| {
+ if (std.mem.eql(u8, a[offset..][0..leaf.size], b[offset..][0..leaf.size])) continue;
+ diff.count += 1;
+ if (diff.first == null) diff.first = .{ .leaf = leaf, .offset = offset };
+ diff.last = .{ .leaf = leaf, .offset = offset };
+ }
+ }
+ return diff;
+}
+
/// Carries an enum over by tag name; false when its name no longer exists.
fn remapEnum(from: Layout.Leaf, old: []const u8, to: Layout.Leaf, new: []u8) bool {
const name = field_value.tagName(from.tags, field_value.readInt(i64, from.bytes(old))) orelse return false;
@@ -124,22 +289,23 @@ pub const StateFile = struct {
layout: Layout,
arena: std.heap.ArenaAllocator,
- /// Opens or creates `path`. A missing or foreign file has no layout and no
- /// used slots: the first `adopt` fills it.
+ /// Opens or creates `path`. A new (empty) file has no layout and no used
+ /// slots: the first `adopt` fills it. A file that is not a state file
+ /// this host can read is refused, never truncated: it may be someone's
+ /// work, written by another build.
pub fn open(gpa: std.mem.Allocator, path: []const u8) !StateFile {
const fd = try posix.openat(posix.AT.FDCWD, path, .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600);
errdefer _ = posix.system.close(fd);
var self: StateFile = .{ .fd = fd, .map = &.{}, .layout = .{}, .arena = .init(gpa) };
errdefer self.arena.deinit();
const len = try fileSize(fd);
- if (len >= header_len) {
+ if (len > 0) {
+ if (len < header_len) return error.NotAStateFile;
self.map = try mapFile(fd, @intCast(len), .{ .READ = true, .WRITE = true });
- if (self.valid()) {
- self.layout = try readLayout(self.arena.allocator(), self.map) orelse .{};
- return self;
- }
- posix.munmap(self.map);
- self.map = &.{};
+ errdefer posix.munmap(self.map);
+ try self.check();
+ self.layout = try readLayout(self.arena.allocator(), self.map) orelse .{};
+ return self;
}
try self.resize(0);
self.header().* = .{ .size = 0, .alignment = 1, .hash = 0, .stride = 0, .used = 0, .field_count = 0, .strings_len = 0 };
@@ -181,7 +347,18 @@ pub const StateFile = struct {
/// Makes the file hold `new`'s layout. Used slots are migrated: `defaults`
/// writes the new default values, then unchanged fields are copied over.
pub fn adopt(self: *StateFile, gpa: std.mem.Allocator, new: abi.Layout, defaults: *const fn (abi.Memory, [*]u8) callconv(.c) void) !Adoption {
- if (self.layout.hash == new.hash and self.layout.size == new.size and self.header().used & 1 != 0) return .unchanged;
+ if (self.layout.hash == new.hash and self.layout.size == new.size and self.header().used & 1 != 0) {
+ // Same type, older header: describe it again, bytes untouched.
+ if (std.mem.eql(u8, &self.header().magic, &file_magic_v1)) {
+ var kept = self.arena;
+ defer kept.deinit();
+ self.arena = .init(gpa);
+ self.layout = try Layout.copy(self.arena.allocator(), new);
+ try writeLayout(self.map[0..header_len], self.layout);
+ self.header().magic = file_magic;
+ }
+ return .unchanged;
+ }
const had_state = self.header().used & 1 != 0;
// Keep the old bytes and layout while the file is rebuilt.
@@ -202,7 +379,7 @@ pub const StateFile = struct {
const stride = std.mem.alignForward(usize, @max(new.size, 1), page);
try self.resize(stride);
const h = self.header();
- h.* = .{ .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = stride, .used = old_used, .field_count = 0, .strings_len = 0 };
+ h.* = .{ .magic = file_magic, .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = stride, .used = old_used, .field_count = 0, .strings_len = 0 };
try writeLayout(self.map[0..header_len], self.layout);
var total: Migration = .{};
@@ -222,10 +399,10 @@ pub const StateFile = struct {
return .{ .migrated = total };
}
- fn valid(self: *StateFile) bool {
+ fn check(self: *StateFile) !void {
const h = self.header();
- if (!std.mem.eql(u8, &h.magic, &file_magic)) return false;
- return self.map.len >= header_len + slot_count * h.stride and h.size <= h.stride;
+ if (!std.mem.eql(u8, &h.magic, &file_magic) and !std.mem.eql(u8, &h.magic, &file_magic_v1)) return error.NotAStateFile;
+ if (self.map.len < header_len + slot_count * h.stride or h.size > h.stride) return error.CorruptStateFile;
}
fn resize(self: *StateFile, stride: usize) !void {
@@ -240,17 +417,24 @@ pub const StateFile = struct {
}
};
-/// Writes `layout`'s leaves after the `Header` at the start of `region`.
+/// Writes `layout`'s leaves after the `Header` at the start of `region`,
+/// in the current format (`StoredField`); the caller sets the magic.
fn writeLayout(region: []u8, layout: Layout) !void {
const h: *Header = @ptrCast(@alignCast(region.ptr));
const fields: [*]StoredField = @ptrCast(@alignCast(region.ptr + @sizeOf(Header)));
const strings_start = @sizeOf(Header) + layout.fields.len * @sizeOf(StoredField);
var strings: usize = 0;
for (layout.fields, 0..) |leaf, i| {
- if (strings_start + strings + leaf.path.len + leaf.tags.len > region.len) return error.LayoutTooLarge;
+ const dims_len = leaf.dims.len * @sizeOf(Dim);
+ if (strings_start + strings + leaf.path.len + leaf.tags.len + dims_len > region.len) return error.LayoutTooLarge;
const base = strings_start + strings;
@memcpy(region[base..][0..leaf.path.len], leaf.path);
@memcpy(region[base + leaf.path.len ..][0..leaf.tags.len], leaf.tags);
+ for (leaf.dims, 0..) |d, j| {
+ const at = base + leaf.path.len + leaf.tags.len + j * @sizeOf(Dim);
+ std.mem.writeInt(u32, region[at..][0..4], d.count, .little);
+ std.mem.writeInt(u32, region[at + 4 ..][0..4], d.stride, .little);
+ }
fields[i] = .{
.offset = leaf.offset,
.size = leaf.size,
@@ -260,24 +444,42 @@ fn writeLayout(region: []u8, layout: Layout) !void {
.path_len = @intCast(leaf.path.len),
.tags = @intCast(strings + leaf.path.len),
.tags_len = @intCast(leaf.tags.len),
+ .rank = @intCast(leaf.dims.len),
+ .dims = @intCast(strings + leaf.path.len + leaf.tags.len),
};
- strings += leaf.path.len + leaf.tags.len;
+ strings += leaf.path.len + leaf.tags.len + dims_len;
}
h.field_count = @intCast(layout.fields.len);
h.strings_len = @intCast(strings);
}
-/// The layout stored in a state file's header, or null if it has none.
+/// The layout stored in a state or store file's header, or null if it has
+/// none. Reads both formats, by magic: a v1 header (`file_magic_v1`,
+/// `store_layout_magic_v1`) has no element leaves, so its arrays are whole
+/// leaves only, which migrate while their type is unchanged.
pub fn readLayout(arena: std.mem.Allocator, map: []const u8) !?Layout {
const h: *const Header = @ptrCast(@alignCast(map.ptr));
if (h.hash == 0 and h.field_count == 0) return null;
- const stored: [*]const StoredField = @ptrCast(@alignCast(map.ptr + @sizeOf(Header)));
- const strings_start = @sizeOf(Header) + h.field_count * @sizeOf(StoredField);
- if (strings_start + h.strings_len > header_len) return error.CorruptHeader;
+ const v1 = std.mem.eql(u8, &h.magic, &file_magic_v1) or std.mem.eql(u8, &h.magic, &store_layout_magic_v1);
+ const stride: usize = if (v1) @sizeOf(StoredFieldV1) else @sizeOf(StoredField);
+ const strings_start = @sizeOf(Header) + @as(usize, h.field_count) * stride;
+ if (strings_start + h.strings_len > header_len or map.len < header_len) return error.CorruptHeader;
const strings = map[strings_start..][0..h.strings_len];
const leaves = try arena.alloc(Layout.Leaf, h.field_count);
- for (leaves, stored[0..h.field_count]) |*leaf, f| {
- if (f.path + f.path_len > strings.len or f.tags + f.tags_len > strings.len) return error.CorruptHeader;
+ for (leaves, 0..) |*leaf, i| {
+ const at = @sizeOf(Header) + i * stride;
+ const f: StoredField = if (v1) blk: {
+ const old: *const StoredFieldV1 = @ptrCast(@alignCast(map[at..].ptr));
+ break :blk .{ .offset = old.offset, .size = old.size, .hash = old.hash, .path = old.path, .path_len = old.path_len, .tags = old.tags, .tags_len = old.tags_len, .kind = old.kind };
+ } else @as(*const StoredField, @ptrCast(@alignCast(map[at..].ptr))).*;
+ const dims_len = @as(usize, f.rank) * @sizeOf(Dim);
+ if (f.path + f.path_len > strings.len or f.tags + f.tags_len > strings.len or f.dims + dims_len > strings.len) return error.CorruptHeader;
+ if (f.rank > abi.layout.max_rank) return error.CorruptHeader;
+ const dims = try arena.alloc(Dim, f.rank);
+ for (dims, 0..) |*d, j| {
+ const raw = strings[f.dims + j * @sizeOf(Dim) ..];
+ d.* = .{ .count = std.mem.readInt(u32, raw[0..4], .little), .stride = std.mem.readInt(u32, raw[4..8], .little) };
+ }
// Copied: the mapping moves when the file is resized.
leaf.* = .{
.path = try arena.dupe(u8, strings[f.path..][0..f.path_len]),
@@ -286,7 +488,12 @@ pub fn readLayout(arena: std.mem.Allocator, map: []const u8) !?Layout {
.kind = f.kind,
.hash = f.hash,
.tags = try arena.dupe(u8, strings[f.tags..][0..f.tags_len]),
+ .dims = dims,
};
+ // Every element must lie inside the state.
+ var end: usize = @as(usize, leaf.offset) + leaf.size;
+ for (dims) |d| end += @as(usize, d.stride) * (d.count -| 1);
+ if (end > h.size) return error.CorruptHeader;
}
return .{ .size = h.size, .alignment = h.alignment, .hash = h.hash, .fields = leaves };
}
@@ -318,7 +525,11 @@ pub fn mapReadOnly(path: []const u8) []align(page) const u8 {
pub const store_base: usize = 0x2000_0000_0000;
/// The most a store can grow to. Only reserved: pages exist as they are used.
pub const store_reserve: usize = 64 << 30;
+/// The allocator's record (`Meta`); unchanged by the layout format.
pub const store_magic = "SKSTORE1".*;
+/// The store's layout header, like `file_magic` and `file_magic_v1`.
+pub const store_layout_magic = "SKSTORE2".*;
+pub const store_layout_magic_v1 = "SKSTORE1".*;
/// The layout region is a state file header; the allocator's own record
/// follows, then the heap.
const meta_offset = header_len;
@@ -368,13 +579,15 @@ pub const StoreFile = struct {
var self: StoreFile = .{ .fd = fd, .len = 0, .layout = .{}, .arena = .init(gpa) };
errdefer self.arena.deinit();
const size: usize = @intCast(try fileSize(fd));
- if (!fresh and size >= heap_offset) {
+ if (!fresh and size > 0) {
+ // Checked before mapping, which may lengthen the file: a file
+ // that is not a store is refused untouched, never wiped.
+ var m: Meta = undefined;
+ const read = linux.pread(fd, std.mem.asBytes(&m), @sizeOf(Meta), meta_offset);
+ if (size < heap_offset or read != @sizeOf(Meta) or !std.mem.eql(u8, &m.magic, &store_magic) or m.base != store_base or m.top > size) return error.NotAStoreFile;
try self.grow(size);
- const m = self.meta();
- if (std.mem.eql(u8, &m.magic, &store_magic) and m.base == store_base and m.top <= size) {
- self.layout = try readLayout(self.arena.allocator(), self.region()) orelse .{};
- return self;
- }
+ self.layout = try readLayout(self.arena.allocator(), self.region()) orelse .{};
+ return self;
}
try self.wipe();
return self;
@@ -396,7 +609,7 @@ pub const StoreFile = struct {
try truncate(self.fd, heap_offset);
try self.grow(heap_offset);
@memset(self.region(), 0);
- self.header().* = .{ .magic = store_magic, .size = 0, .alignment = 1, .hash = 0, .stride = 0, .used = 0, .field_count = 0, .strings_len = 0 };
+ self.header().* = .{ .magic = store_layout_magic, .size = 0, .alignment = 1, .hash = 0, .stride = 0, .used = 0, .field_count = 0, .strings_len = 0 };
self.meta().* = .{};
self.layout = .{};
}
@@ -452,7 +665,7 @@ pub const StoreFile = struct {
result = .{ .migrated = migrate(old_layout, old[0..old_layout.size], self.layout, fresh[0..new.size]) };
a.rawFree(old[0..@max(old_layout.size, 1)], .fromByteUnits(@max(old_layout.alignment, 1)), @returnAddress());
}
- self.header().* = .{ .magic = store_magic, .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = 0, .used = 1, .field_count = 0, .strings_len = 0 };
+ self.header().* = .{ .magic = store_layout_magic, .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = 0, .used = 1, .field_count = 0, .strings_len = 0 };
try writeLayout(self.region(), self.layout);
m.root = @intFromPtr(fresh) - store_base;
return result;
@@ -723,3 +936,192 @@ test "a store keeps its root and what it points to across a reopen, and migrates
defer store.close();
try std.testing.expectEqual(StoreFile.Adoption.created, try store.adopt(gpa, abi.layout.of(V2, .store), D.v2));
}
+
+test "arrays migrate element by element: fields, enums by name, shrink and growth" {
+ const E1 = struct { x: f32 = 0, mood: enum(u8) { calm = 1, wild = 2 } = .calm };
+ const E2 = struct { added: u8 = 9, mood: enum(u8) { wild = 5, calm = 6, sleepy = 7 } = .sleepy, x: f32 = 0 };
+ const V1 = struct { list: struct { buffer: [4]E1 = @splat(.{}), len: u32 = 0 } = .{}, samples: [4]u16 = @splat(0), pos: [2]f32 = @splat(0), grid: [2][3]u8 = @splat(@splat(0)) };
+ const V2 = struct { list: struct { buffer: [2]E2 = @splat(.{}), len: u32 = 0 } = .{}, samples: [6]u16 = @splat(5), pos: @Vector(2, f32) = @splat(0), grid: [3][2]u8 = @splat(@splat(0)) };
+ var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer arena.deinit();
+ const l1 = try Layout.copy(arena.allocator(), abi.layout.of(V1, .state));
+ const l2 = try Layout.copy(arena.allocator(), abi.layout.of(V2, .state));
+ var old: V1 = .{ .samples = .{ 1, 2, 3, 4 }, .pos = .{ 3, 4 }, .grid = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } } };
+ for (&old.list.buffer, 0..) |*e, i| e.* = .{ .x = @floatFromInt(i + 1), .mood = if (i % 2 == 0) .calm else .wild };
+ old.list.len = 4;
+ var new: V2 = .{};
+ _ = migrate(l1, std.mem.asBytes(&old), l2, std.mem.asBytes(&new));
+ try std.testing.expectEqual(E2{ .added = 9, .mood = .calm, .x = 1 }, new.list.buffer[0]);
+ try std.testing.expectEqual(E2{ .added = 9, .mood = .wild, .x = 2 }, new.list.buffer[1]);
+ // `len` is kept as it was: a List clamps it where it is read.
+ try std.testing.expectEqual(@as(u32, 4), new.list.len);
+ try std.testing.expectEqual([6]u16{ 1, 2, 3, 4, 5, 5 }, new.samples);
+ try std.testing.expectEqual(@as(f32, 4), new.pos[1]);
+ try std.testing.expectEqual([3][2]u8{ .{ 1, 2 }, .{ 4, 5 }, .{ 0, 0 } }, new.grid);
+
+ // Growing back: the kept elements return, new ones take defaults.
+ var back: V1 = .{};
+ _ = migrate(l2, std.mem.asBytes(&new), l1, std.mem.asBytes(&back));
+ try std.testing.expectEqual(@as(f32, 2), back.list.buffer[1].x);
+ try std.testing.expectEqual(@as(f32, 0), back.list.buffer[2].x);
+ try std.testing.expectEqual(.wild, back.list.buffer[1].mood);
+}
+
+test "an array of songs keeps its songs when a song gains a field" {
+ const Note = struct { pitch: u8 = 0, length: u8 = 1 };
+ const Song1 = struct { name: [8]u8 = @splat(0), notes: [16]Note = @splat(.{}), tempo: u16 = 120 };
+ const Song2 = struct { tempo: u16 = 120, swing: f32 = 0.5, notes: [16]Note = @splat(.{}), name: [8]u8 = @splat(0) };
+ const V1 = struct { songs: [8]Song1 = @splat(.{}), other: u32 = 0 };
+ const V2 = struct { added: u8 = 1, songs: [8]Song2 = @splat(.{}) };
+ var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer arena.deinit();
+ const l1 = try Layout.copy(arena.allocator(), abi.layout.of(V1, .state));
+ const l2 = try Layout.copy(arena.allocator(), abi.layout.of(V2, .state));
+ var old: V1 = .{};
+ for (&old.songs, 0..) |*song, i| {
+ song.name = "song 0".*;
+ song.name[7] = '0' + @as(u8, @intCast(i));
+ song.tempo = @intCast(100 + i);
+ for (&song.notes, 0..) |*n, j| n.* = .{ .pitch = @intCast(i * 16 + j), .length = 2 };
+ }
+ var new: V2 = .{};
+ const m = migrate(l1, std.mem.asBytes(&old), l2, std.mem.asBytes(&new));
+ for (old.songs, new.songs) |a, b| {
+ try std.testing.expectEqualSlices(u8, &a.name, &b.name);
+ try std.testing.expectEqual(a.tempo, b.tempo);
+ try std.testing.expectEqual(@as(f32, 0.5), b.swing);
+ for (a.notes, b.notes) |x, y| try std.testing.expectEqual(x, y);
+ }
+ // songs[].notes is unchanged, so it copies whole; swing and added are new.
+ try std.testing.expectEqual(@as(usize, 2), m.reset);
+ // The same type on both sides copies in one piece.
+ var same: V1 = .{};
+ try std.testing.expectEqual(@as(usize, 2), migrate(l1, std.mem.asBytes(&old), l1, std.mem.asBytes(&same)).kept);
+ try std.testing.expectEqualDeep(old, same);
+}
+
+test "compare goes leaf by leaf: padding never differs" {
+ const S = struct { a: u8 = 1, b: u32 = 2, v: @Vector(3, f32) = .{ 1, 2, 3 }, list: [2]struct { c: u8 = 0, d: u16 = 0 } = @splat(.{}) };
+ var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer arena.deinit();
+ const layout = try Layout.copy(arena.allocator(), abi.layout.of(S, .state));
+ var x: S = .{};
+ var y: S = .{};
+ const a = std.mem.asBytes(&x);
+ const b = std.mem.asBytes(&y);
+ @memcpy(b, a);
+ // Every byte no leaf covers.
+ var covered: [@sizeOf(S)]bool = @splat(false);
+ for (layout.fields) |leaf| {
+ if (leaf.whole()) continue;
+ var it = leaf.elements();
+ while (it.next()) |offset| @memset(covered[offset..][0..leaf.size], true);
+ }
+ var padding: usize = 0;
+ for (covered, 0..) |c, i| if (!c) {
+ b[i] ^= 0xff;
+ padding += 1;
+ };
+ try std.testing.expect(padding >= 4);
+ try std.testing.expectEqual(@as(usize, 0), compare(layout, a, b).count);
+ y.list[1].d = 7;
+ y.v[2] = 0;
+ const diff = compare(layout, a, b);
+ try std.testing.expectEqual(@as(usize, 2), diff.count);
+ var buf: [64]u8 = undefined;
+ var w: std.Io.Writer = .fixed(&buf);
+ try diff.last.?.leaf.name(diff.last.?.offset, &w);
+ try std.testing.expectEqualStrings("list[1].d", w.buffered());
+}
+
+fn testPath(buf: []u8, tmp: anytype, name: []const u8) ![]const u8 {
+ return std.mem.print(buf, ".zig-cache/tmp/{s}/{s}", .{ &tmp.sub_path, name });
+}
+
+test "a file that is not a state file is refused, untouched" {
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ const io = std.testing.io;
+ var buf: [std.fs.max_path_bytes]u8 = undefined;
+ for ([_]usize{ 100, 3 * header_len }) |len| {
+ const path = try testPath(&buf, tmp, "notes.state");
+ const text = try std.testing.allocator.alloc(u8, len);
+ defer std.testing.allocator.free(text);
+ @memset(text, 'x');
+ try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = text });
+ try std.testing.expectError(error.NotAStateFile, StateFile.open(std.testing.allocator, path));
+ const after = try std.Io.Dir.cwd().readFileAlloc(io, path, std.testing.allocator, .unlimited);
+ defer std.testing.allocator.free(after);
+ try std.testing.expectEqualSlices(u8, text, after);
+ }
+}
+
+test "a state file from before element leaves (drat-meout's) keeps every value" {
+ const gpa = std.testing.allocator;
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var buf: [std.fs.max_path_bytes]u8 = undefined;
+ const path = try testPath(&buf, tmp, "game.state");
+ const original = try std.Io.Dir.cwd().readFileAlloc(io, "test/fixtures/drat-meout-v1.state-fixture", gpa, .unlimited);
+ defer gpa.free(original);
+ try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = original });
+ try std.testing.expectEqualStrings(&file_magic_v1, original[0..8]);
+
+ var arena: std.heap.ArenaAllocator = .init(gpa);
+ defer arena.deinit();
+ const a = arena.allocator();
+ const old = (try readLayout(a, original)).?;
+ try std.testing.expect(old.find("tracker.songs") != null);
+ const h: *const Header = @ptrCast(@alignCast(original.ptr));
+ const live = original[header_len..][0..h.size];
+
+ // The layout the game had when it wrote the file, as a sketch would
+ // describe it: unchanged, so nothing migrates and the header is
+ // rewritten in the new format.
+ const same = try asSketchLayout(a, old, &.{});
+ {
+ var file = try StateFile.open(gpa, path);
+ defer file.close();
+ try std.testing.expectEqual(StateFile.Adoption.unchanged, try file.adopt(gpa, same, NoDefaults.f));
+ try std.testing.expectEqualStrings(&file_magic, &file.header().magic);
+ }
+ var file = try StateFile.open(gpa, path);
+ defer file.close();
+ try std.testing.expectEqual(old.fields.len, file.layout.fields.len);
+ try std.testing.expectEqualSlices(u8, live, file.slot(0));
+
+ // A changed layout (one field gone, one new): every other leaf keeps
+ // its value, the songs included (an unchanged array type, so whole).
+ const changed = try asSketchLayout(a, old, "scene");
+ _ = try file.adopt(gpa, changed, NoDefaults.f);
+ for (old.fields) |leaf| {
+ if (std.mem.eql(u8, leaf.path, "scene")) continue;
+ try std.testing.expectEqualSlices(u8, leaf.bytes(live), leaf.bytes(file.slot(0)));
+ }
+ try std.testing.expectEqual(@as(u8, 0), file.slot(0)[h.size]);
+}
+
+const NoDefaults = struct {
+ fn f(_: abi.Memory, _: [*]u8) callconv(.c) void {}
+};
+
+/// `l` as `abi.Layout`, without the leaf at `drop` and with a new byte
+/// leaf `added` past the end, the type hash changed accordingly.
+fn asSketchLayout(a: std.mem.Allocator, l: Layout, drop: []const u8) !abi.Layout {
+ var fields: std.ArrayList(abi.Field) = .empty;
+ for (l.fields) |leaf| {
+ if (std.mem.eql(u8, leaf.path, drop)) continue;
+ try fields.append(a, .{
+ .path = try a.dupeSentinel(u8, leaf.path, 0),
+ .offset = leaf.offset,
+ .size = leaf.size,
+ .kind = leaf.kind,
+ .hash = leaf.hash,
+ .tags = try a.dupeSentinel(u8, leaf.tags, 0),
+ });
+ }
+ if (drop.len == 0) return .{ .size = l.size, .alignment = l.alignment, .hash = l.hash, .fields = fields.items };
+ try fields.append(a, .{ .path = "added", .offset = @intCast(l.size), .size = 1, .kind = .uint, .hash = abi.layout.hash(u8) });
+ return .{ .size = l.size + 1, .alignment = l.alignment, .hash = l.hash ^ 1, .fields = fields.items };
+}