diff options
Diffstat (limited to 'src/host/memory.zig')
| -rw-r--r-- | src/host/memory.zig | 303 |
1 files changed, 112 insertions, 191 deletions
diff --git a/src/host/memory.zig b/src/host/memory.zig index 796fb39..2749ea1 100644 --- a/src/host/memory.zig +++ b/src/host/memory.zig @@ -1,9 +1,12 @@ //! Sketch memory by lifetime. //! -//! `StateFile`: one `MAP_SHARED` file holding the live state, plus a header -//! describing the layout, so the state outlives restarts, -//! a probe can read it from another process, and a reload with a changed -//! layout can carry the old values over field by field. +//! `State`: process memory in the layout the sketch declares; a reload +//! with a changed layout carries the old values over field by field. It +//! outlives the process in the sessions database (Sessions.zig), whose +//! stretches keep its layout in the header format below. +//! +//! `OldState`: a state file from before the database held the present, +//! read once to import it. //! //! `Session`: this process's memory, kept across reloads. const std = @import("std"); @@ -14,7 +17,7 @@ const field_value = @import("fields.zig"); pub const page = std.heap.page_size_min; pub const header_len = 64 * 1024; -/// One slot, the live state: snapshots are marks in the sessions database. +/// A layout header, and an old file's one slot, the live state. pub const file_magic = "SKETCH03".*; /// Before sessions: slot 0 live and 1..7 snapshots (`old_slot_count`), which /// the host imports as marks and then drops (`dropSnapshots`). @@ -287,167 +290,93 @@ fn remapEnum(from: Layout.Leaf, old: []const u8, to: Layout.Leaf, new: []u8) boo return true; } -pub const StateFile = struct { - fd: posix.fd_t, - map: []align(page) u8, - /// 1, or `old_slot_count` in a file from before sessions. - slots: usize = 1, - /// Layout of the bytes in the file, owned by `arena`. - layout: Layout, +/// Live State: page aligned, in the running sketch's layout; `header` is +/// that layout as `layoutOf` reads it back, for the sessions' stretches. +pub const State = struct { + bytes: []align(page) u8 = &.{}, + layout: Layout = .{}, + header: []const u8 = "", + /// Owns `layout` and `header`. arena: std.heap.ArenaAllocator, - /// Opens or creates `path`. A new (empty) file has no layout and no used - /// slots: the first `adopt` fills it. A file that is not a state file - /// this host can read is refused, never truncated: it may be someone's - /// work, written by another build. - pub fn open(gpa: std.mem.Allocator, path: []const u8) !StateFile { - const fd = try posix.openat(posix.AT.FDCWD, path, .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600); - errdefer _ = posix.system.close(fd); - var self: StateFile = .{ .fd = fd, .map = &.{}, .layout = .{}, .arena = .init(gpa) }; - errdefer self.arena.deinit(); - const len = try fileSize(fd); - if (len > 0) { - if (len < header_len) return error.NotAStateFile; - self.map = try mapFile(fd, @intCast(len), .{ .READ = true, .WRITE = true }); - errdefer posix.munmap(self.map); - try self.check(); - self.layout = try readLayout(self.arena.allocator(), self.map) orelse .{}; - return self; - } - try self.resize(0); - self.header().* = .{ .size = 0, .alignment = 1, .hash = 0, .stride = 0, .used = 0, .field_count = 0, .strings_len = 0 }; - return self; + pub fn init(gpa: std.mem.Allocator) State { + return .{ .arena = .init(gpa) }; } - pub fn close(self: *StateFile) void { - if (self.map.len != 0) posix.munmap(self.map); - _ = posix.system.close(self.fd); + pub fn deinit(self: *State, gpa: std.mem.Allocator) void { + gpa.free(self.bytes); self.arena.deinit(); } - pub fn header(self: *StateFile) *Header { - return @ptrCast(self.map.ptr); - } - - /// The header and the layout after it, as `readLayout` reads them back. - pub fn layoutBytes(self: *StateFile) []const u8 { - const h = self.header(); - return self.map[0 .. @sizeOf(Header) + @as(usize, h.field_count) * @sizeOf(StoredField) + h.strings_len]; + pub fn live(self: *State) []align(page) u8 { + return self.bytes[0..self.layout.size]; } - pub fn slot(self: *StateFile, index: usize) []align(page) u8 { - std.debug.assert(index < self.slots); - const h = self.header(); - const start = header_len + index * h.stride; - return @alignCast(self.map[start..][0..h.size]); + /// Makes State hold `new`'s layout: a new allocation of its defaults + /// (`defaults`), then every leaf `migrate` keeps of the old State. Null + /// when the layout is unchanged, or there was none. + pub fn adopt(self: *State, gpa: std.mem.Allocator, new: abi.Layout, defaults: *const fn (abi.Memory, [*]u8) callconv(.c) void) !?Migration { + const first = self.bytes.len == 0; + if (!first and self.layout.hash == new.hash and self.layout.size == new.size) return null; + var arena: std.heap.ArenaAllocator = .init(gpa); + errdefer arena.deinit(); + const layout = try Layout.copy(arena.allocator(), new); + const region = try arena.allocator().alignedAlloc(u8, .of(Header), header_len); + @memset(region, 0); + const h: *Header = @ptrCast(region.ptr); + h.* = .{ .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = std.mem.alignForward(usize, @max(new.size, 1), page), .used = 1, .field_count = 0, .strings_len = 0 }; + try writeLayout(region, layout); + const bytes = try gpa.alignedAlloc(u8, .fromByteUnits(page), @max(new.size, 1)); + @memset(bytes, 0); + defaults(.state, bytes.ptr); + const m = if (first) null else migrate(self.layout, self.live(), layout, bytes); + self.deinit(gpa); + self.* = .{ .bytes = bytes, .layout = layout, .header = region[0 .. @sizeOf(Header) + @as(usize, h.field_count) * @sizeOf(StoredField) + h.strings_len], .arena = arena }; + return m; } +}; - pub fn used(self: *StateFile, index: usize) bool { - return self.header().used & (@as(u32, 1) << @intCast(index)) != 0; - } +/// A state file from before the sessions database held the present: a +/// layout header, then one slot (`file_magic`), or eight (`file_magic_v2`, +/// `file_magic_v1`), the first live State and the others snapshots. Mapped +/// read-only: imported once, never changed. +pub const OldState = struct { + map: []align(page) const u8, + slots: usize, - pub fn markUsed(self: *StateFile, index: usize, value: bool) void { - const bit = @as(u32, 1) << @intCast(index); - if (value) self.header().used |= bit else self.header().used &= ~bit; + /// error.FileNotFound when `path` is missing or empty; a file that is + /// no state file this host reads is refused. + pub fn open(path: []const u8) !OldState { + const map = mapReadOnly(path); + if (map.len == 0) return error.FileNotFound; + errdefer posix.munmap(map); + if (map.len < header_len) return error.NotAStateFile; + const h: *const Header = @ptrCast(map.ptr); + const slots: usize = if (std.mem.eql(u8, &h.magic, &file_magic)) 1 else if (std.mem.eql(u8, &h.magic, &file_magic_v2) or std.mem.eql(u8, &h.magic, &file_magic_v1)) old_slot_count else return error.NotAStateFile; + if (map.len < header_len + slots * h.stride or h.size > h.stride) return error.CorruptStateFile; + return .{ .map = map, .slots = slots }; } - pub const Adoption = union(enum) { - unchanged, - created, - migrated: Migration, - }; - - /// Makes the file hold `new`'s layout. Used slots are migrated: `defaults` - /// writes the new default values, then unchanged fields are copied over. - pub fn adopt(self: *StateFile, gpa: std.mem.Allocator, new: abi.Layout, defaults: *const fn (abi.Memory, [*]u8) callconv(.c) void) !Adoption { - if (self.layout.hash == new.hash and self.layout.size == new.size and self.header().used & 1 != 0) { - // Same type, older header: describe it again, bytes untouched. - if (std.mem.eql(u8, &self.header().magic, &file_magic_v1)) { - var kept = self.arena; - defer kept.deinit(); - self.arena = .init(gpa); - self.layout = try Layout.copy(self.arena.allocator(), new); - try writeLayout(self.map[0..header_len], self.layout); - self.header().magic = file_magic_v2; - } - return .unchanged; - } - const had_state = self.header().used & 1 != 0; - - // Keep the old bytes and layout while the file is rebuilt. - var old_arena: std.heap.ArenaAllocator = .init(gpa); - defer old_arena.deinit(); - const old_layout = self.layout; - var old_slots: [old_slot_count]?[]u8 = @splat(null); - for (old_slots[0..self.slots], 0..) |*copy, i| { - if (self.used(i)) copy.* = try old_arena.allocator().dupe(u8, self.slot(i)); - } - const old_used = self.header().used; - // `old_layout` points into `self.arena`: move it aside before resetting. - var kept = self.arena; - defer kept.deinit(); - self.arena = .init(gpa); - - self.layout = try Layout.copy(self.arena.allocator(), new); - const stride = std.mem.alignForward(usize, @max(new.size, 1), page); - try self.resize(stride); - const h = self.header(); - h.* = .{ .magic = if (self.slots == 1) file_magic else file_magic_v2, .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = stride, .used = old_used, .field_count = 0, .strings_len = 0 }; - try writeLayout(self.map[0..header_len], self.layout); - - var total: Migration = .{}; - for (old_slots, 0..) |maybe_old, i| { - const old = maybe_old orelse continue; - const bytes = self.slot(i); - @memset(bytes, 0); - defaults(.state, bytes.ptr); - const m = migrate(old_layout, old, self.layout, bytes); - total.kept += m.kept; - if (i == 0) total.reset = m.reset; - } - if (!had_state) { - self.markUsed(0, false); - return .created; - } - return .{ .migrated = total }; + pub fn close(self: OldState) void { + posix.munmap(self.map); } - /// Makes a file from before sessions one of a single slot, its - /// snapshots (imported as marks by now) dropped. - pub fn dropSnapshots(self: *StateFile) !void { - if (self.slots == 1) return; - const stride = self.header().stride; - self.slots = 1; - try self.resize(stride); - self.header().magic = file_magic; - self.header().used &= 1; + /// The header and layout, as `restore` and `layoutOf` take them. + pub fn layoutBytes(self: OldState) []const u8 { + return self.map[0..header_len]; } - fn check(self: *StateFile) !void { - const h = self.header(); - if (std.mem.eql(u8, &h.magic, &file_magic)) { - self.slots = 1; - } else if (std.mem.eql(u8, &h.magic, &file_magic_v2) or std.mem.eql(u8, &h.magic, &file_magic_v1)) { - self.slots = old_slot_count; - } else return error.NotAStateFile; - if (self.map.len < header_len + self.slots * h.stride or h.size > h.stride) return error.CorruptStateFile; - } - - fn resize(self: *StateFile, stride: usize) !void { - const len = header_len + self.slots * stride; - if (self.map.len != 0) posix.munmap(self.map); - self.map = &.{}; - switch (posix.errno(posix.system.ftruncate(self.fd, @intCast(len)))) { - .SUCCESS => {}, - else => |err| return posix.unexpectedErrno(err), - } - self.map = try mapFile(self.fd, len, .{ .READ = true, .WRITE = true }); + /// Slot `i`'s State, null when the slot is unused. + pub fn slot(self: OldState, i: usize) ?[]const u8 { + const h: *const Header = @ptrCast(self.map.ptr); + if (h.used & (@as(u32, 1) << @intCast(i)) == 0) return null; + return self.map[header_len + i * h.stride ..][0..h.size]; } }; /// Fills `live` (laid out as `to`) from `image`, state kept in another -/// layout, described by `layout_bytes` (a header as `StateFile.layoutBytes` -/// gives it): `defaults`, then every leaf `migrate` keeps. Returns the +/// layout, described by `layout_bytes` (a header as `State.header` holds +/// it): `defaults`, then every leaf `migrate` keeps. Returns the /// image's layout, `arena`'s. pub fn restore(arena: std.mem.Allocator, layout_bytes: []const u8, image: []const u8, to: Layout, live: []u8, defaults: *const fn (abi.Memory, [*]u8) callconv(.c) void) !Layout { const from = try layoutOf(arena, layout_bytes); @@ -458,7 +387,7 @@ pub fn restore(arena: std.mem.Allocator, layout_bytes: []const u8, image: []cons return from; } -/// The layout in a header as `StateFile.layoutBytes` gives it, `arena`'s. +/// The layout in a header as `State.header` holds it, `arena`'s. pub fn layoutOf(arena: std.mem.Allocator, layout_bytes: []const u8) !Layout { if (layout_bytes.len < @sizeOf(Header) or layout_bytes.len > header_len) return error.CorruptKeyframe; const region = try arena.alignedAlloc(u8, .of(Header), header_len); @@ -556,10 +485,6 @@ pub fn readLayout(arena: std.mem.Allocator, map: []const u8) !?Layout { return .{ .size = h.size, .alignment = h.alignment, .hash = h.hash, .fields = leaves }; } -pub fn mapFile(fd: posix.fd_t, len: usize, prot: posix.PROT) ![]align(page) u8 { - return posix.mmap(null, len, prot, .{ .TYPE = .SHARED }, fd, 0); -} - pub fn fileSize(fd: posix.fd_t) !u64 { var stat: linux.Statx = undefined; switch (linux.errno(linux.statx(fd, "", linux.AT.EMPTY_PATH, .{ .SIZE = true }, &stat))) { @@ -715,7 +640,7 @@ test "migration keeps unchanged fields and defaults the rest" { try std.testing.expectEqual(@as(usize, 1), m.reset); } -test "state file migrates live state, and a reopen reads the layout back" { +test "State migrates to a changed layout, and keeps itself when it is unchanged" { const V1 = extern struct { score: u32 = 0, name: [4]u8 = "none".* }; const V2 = extern struct { level: u16 = 1, score: u32 = 0 }; const D = struct { @@ -726,24 +651,20 @@ test "state file migrates live state, and a reopen reads the layout back" { @as(*V2, @ptrCast(@alignCast(p))).* = .{}; } }; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var buf: [std.fs.max_path_bytes]u8 = undefined; - const path = try std.mem.print(&buf, ".zig-cache/tmp/{s}/test.state", .{&tmp.sub_path}); const gpa = std.testing.allocator; - { - var file = try StateFile.open(gpa, path); - defer file.close(); - try std.testing.expectEqual(StateFile.Adoption.created, try file.adopt(gpa, abi.layout.of(V1, .state), D.v1)); - file.markUsed(0, true); - @as(*V1, @ptrCast(file.slot(0).ptr)).score = 99; - } - var file = try StateFile.open(gpa, path); - defer file.close(); - try std.testing.expectEqual(abi.layout.hash(V1), file.layout.hash); - const adoption = try file.adopt(gpa, abi.layout.of(V2, .state), D.v2); - try std.testing.expectEqual(@as(usize, 1), adoption.migrated.reset); - try std.testing.expectEqual(V2{ .level = 1, .score = 99 }, @as(*V2, @ptrCast(file.slot(0).ptr)).*); + var state: State = .init(gpa); + defer state.deinit(gpa); + try std.testing.expectEqual(@as(?Migration, null), try state.adopt(gpa, abi.layout.of(V1, .state), D.v1)); + @as(*V1, @ptrCast(state.live().ptr)).score = 99; + try std.testing.expectEqual(@as(?Migration, null), try state.adopt(gpa, abi.layout.of(V1, .state), D.v1)); + try std.testing.expectEqual(@as(u32, 99), @as(*V1, @ptrCast(state.live().ptr)).score); + const m = (try state.adopt(gpa, abi.layout.of(V2, .state), D.v2)).?; + try std.testing.expectEqual(@as(usize, 1), m.reset); + try std.testing.expectEqual(V2{ .level = 1, .score = 99 }, @as(*V2, @ptrCast(@alignCast(state.live().ptr))).*); + // The header reads back as the layout. + var arena: std.heap.ArenaAllocator = .init(gpa); + defer arena.deinit(); + try std.testing.expectEqual(abi.layout.hash(V2), (try layoutOf(arena.allocator(), state.header)).hash); } test "arrays migrate element by element: fields, enums by name, shrink and growth" { @@ -858,14 +779,15 @@ test "a file that is not a state file is refused, untouched" { defer std.testing.allocator.free(text); @memset(text, 'x'); try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = text }); - try std.testing.expectError(error.NotAStateFile, StateFile.open(std.testing.allocator, path)); + try std.testing.expectError(error.NotAStateFile, OldState.open(path)); const after = try std.Io.Dir.cwd().readFileAlloc(io, path, std.testing.allocator, .unlimited); defer std.testing.allocator.free(after); try std.testing.expectEqualSlices(u8, text, after); } + try std.testing.expectError(error.FileNotFound, OldState.open(try testPath(&buf, tmp, "none.state"))); } -test "a state file from before element leaves (drat-meout's) keeps every value" { +test "a state file from before element leaves (drat-meout's) imports with every value" { const gpa = std.testing.allocator; const io = std.testing.io; var tmp = std.testing.tmpDir(.{}); @@ -885,30 +807,29 @@ test "a state file from before element leaves (drat-meout's) keeps every value" const h: *const Header = @ptrCast(@alignCast(original.ptr)); const live = original[header_len..][0..h.size]; - // The layout the game had when it wrote the file, as a sketch would - // describe it: unchanged, so nothing migrates and the header is - // rewritten in the new format. - const same = try asSketchLayout(a, old, &.{}); - { - var file = try StateFile.open(gpa, path); - defer file.close(); - try std.testing.expectEqual(StateFile.Adoption.unchanged, try file.adopt(gpa, same, NoDefaults.f)); - try std.testing.expectEqualStrings(&file_magic_v2, &file.header().magic); - } - var file = try StateFile.open(gpa, path); + const file = try OldState.open(path); defer file.close(); - try std.testing.expectEqual(old.fields.len, file.layout.fields.len); - try std.testing.expectEqualSlices(u8, live, file.slot(0)); - - // A changed layout (one field gone, one new): every other leaf keeps - // its value, the songs included (an unchanged array type, so whole). - const changed = try asSketchLayout(a, old, "scene"); - _ = try file.adopt(gpa, changed, NoDefaults.f); - for (old.fields) |leaf| { - if (std.mem.eql(u8, leaf.path, "scene")) continue; - try std.testing.expectEqualSlices(u8, leaf.bytes(live), leaf.bytes(file.slot(0))); + try std.testing.expectEqual(@as(usize, old_slot_count), file.slots); + // Into the layout the game had when it wrote the file, as a sketch would + // describe it: every byte; then into a changed one (one field gone, one + // new): every other leaf keeps its value, the songs included (an + // unchanged array type, so whole). + for ([_][]const u8{ "", "scene" }) |drop| { + var state: State = .init(gpa); + defer state.deinit(gpa); + _ = try state.adopt(gpa, try asSketchLayout(a, old, drop), NoDefaults.f); + @memset(state.live(), 0); + _ = try restore(a, file.layoutBytes(), file.slot(0).?, state.layout, state.live(), NoDefaults.f); + if (drop.len == 0) try std.testing.expectEqualSlices(u8, live, state.live()); + for (old.fields) |leaf| { + if (drop.len > 0 and std.mem.eql(u8, leaf.path, drop)) continue; + try std.testing.expectEqualSlices(u8, leaf.bytes(live), leaf.bytes(state.live())); + } } - try std.testing.expectEqual(@as(u8, 0), file.slot(0)[h.size]); + // The file is as it was. + const after = try tmp.dir.readFileAlloc(io, "game.state", gpa, .unlimited); + defer gpa.free(after); + try std.testing.expectEqualSlices(u8, original, after); } const NoDefaults = struct { |
