diff options
| author | Gabriel Schneider <[email protected]> | 2026-11-17 00:45:31 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-11-17 12:15:01 -0300 |
| commit | e699ffe4182f66e28180ea2d91ab1fb8c408d49e (patch) | |
| tree | 0577374fb6164ef214cd727b662a23498ad547eb /src/host/memory.zig | |
| parent | a1ae80096bd69ee7d92ea9e4564e960d8553395e (diff) | |
| download | sketchbook-e699ffe4182f66e28180ea2d91ab1fb8c408d49e.tar.gz sketchbook-e699ffe4182f66e28180ea2d91ab1fb8c408d49e.zip | |
Delete the Store; State is checked against the 64 MiB a save holds
A sketch's data is State: flat bytes, what grows has its capacity in its
type. The Store, a heap in a file mapped at 0x2000_0000_0000 with 64 GiB
reserved, its power-of-two allocator, its root adopt and migration, and
its test, are gone from memory.zig; so are Context.store and store_root,
Info.store, Memory.store, the Store declaration's glue, Init.store,
Frame.store and Console.store, the web host's store in wasm memory, the
host's StoreFile wiring and `<state>.store` file, the `.store` suffix in
moving an old state into .sketchbook/, the build's tests' `rm` of it, and
layout.check's `pointers` flag (State is the one checked memory now).
Its one real user, micscope, moved off it in the change before. Old
`.state.store` files are not read, converted or deleted, and stay
ignored by .gitignore. A sketch that still declares a Store gets no
error for the declaration: nothing reads it, and a `*Store` parameter is
a compile error (no argument of that type).
layout.of(T, .state) refuses a State bigger than layout.max_state, 64 MiB,
one constant: "State is N bytes, and a save holds at most 64 MiB: lower
a capacity (an array, an sk.List), or keep what need not be saved in
Session." The pointer-hash gap the spec found (a Store pointer's
element type changing kept the old bytes) is gone with the only
persisted memory that held pointers.
README: no Store row, allocator row, migration paragraph, file bullet or
known limits; draw and command lose their store parameter; State's row
says 64 MiB at most; data that grows and is the user's work goes in a
List in State or, bigger than a save, in files the sketch appends to.
Tests: zig build test, 166/166 steps (the store's unit test gone; new: a
65-MiB State fails to compile with the message above, by expect_errors).
micscope, spheres, noc's e2e and drat-meout's test unchanged and green.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_016Y9MNBECMWcJD5pdS4SR6T
Diffstat (limited to 'src/host/memory.zig')
| -rw-r--r-- | src/host/memory.zig | 257 |
1 files changed, 5 insertions, 252 deletions
diff --git a/src/host/memory.zig b/src/host/memory.zig index 044fc3b..796fb39 100644 --- a/src/host/memory.zig +++ b/src/host/memory.zig @@ -5,10 +5,6 @@ //! a probe can read it from another process, and a reload with a changed //! layout can carry the old values over field by field. //! -//! `StoreFile`: a heap in a file, mapped at the same address in every run, -//! so what it holds may point into it: the sketch's `Store` and everything -//! allocated for it outlive restarts. -//! //! `Session`: this process's memory, kept across reloads. const std = @import("std"); const posix = std.posix; @@ -515,14 +511,14 @@ fn writeLayout(region: []u8, layout: Layout) !void { h.strings_len = @intCast(strings); } -/// The layout stored in a state or store file's header, or null if it has -/// none. Reads both formats, by magic: a v1 header (`file_magic_v1`, -/// `store_layout_magic_v1`) has no element leaves, so its arrays are whole -/// leaves only, which migrate while their type is unchanged. +/// The layout stored in a state file's header, or null if it has none. +/// Reads both formats, by magic: a v1 header (`file_magic_v1`) has no +/// element leaves, so its arrays are whole leaves only, which migrate while +/// their type is unchanged. pub fn readLayout(arena: std.mem.Allocator, map: []const u8) !?Layout { const h: *const Header = @ptrCast(@alignCast(map.ptr)); if (h.hash == 0 and h.field_count == 0) return null; - const v1 = std.mem.eql(u8, &h.magic, &file_magic_v1) or std.mem.eql(u8, &h.magic, &store_layout_magic_v1); + const v1 = std.mem.eql(u8, &h.magic, &file_magic_v1); const stride: usize = if (v1) @sizeOf(StoredFieldV1) else @sizeOf(StoredField); const strings_start = @sizeOf(Header) + @as(usize, h.field_count) * stride; if (strings_start + h.strings_len > header_len or map.len < header_len) return error.CorruptHeader; @@ -582,208 +578,6 @@ pub fn mapReadOnly(path: []const u8) []align(page) const u8 { return posix.mmap(null, @intCast(len), .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 0) catch &.{}; } -/// Where every store is mapped, in every process: far from where Linux puts -/// executables (0x55...) and mappings (0x7f...). -pub const store_base: usize = 0x2000_0000_0000; -/// The most a store can grow to. Only reserved: pages exist as they are used. -pub const store_reserve: usize = 64 << 30; -/// The allocator's record (`Meta`); unchanged by the layout format. -pub const store_magic = "SKSTORE1".*; -/// The store's layout header, like `file_magic` and `file_magic_v1`. -pub const store_layout_magic = "SKSTORE2".*; -pub const store_layout_magic_v1 = "SKSTORE1".*; -/// The layout region is a state file header; the allocator's own record -/// follows, then the heap. -const meta_offset = header_len; -const heap_offset = 2 * header_len; - -/// A heap in a file, with the sketch's `Store` at its root. Mapped at -/// `store_base` in every run, so pointers it holds into itself stay valid -/// across restarts. Not crash-safe: a crash in the middle of an allocation -/// can leave it inconsistent (`--fresh` starts over). -/// -/// Blocks come in power-of-two sizes, each class with a free list threaded -/// through the free blocks, so growing a list by doubling reuses what it -/// frees. The file grows by mapping more of it in place: nothing moves. -pub const StoreFile = struct { - fd: posix.fd_t, - /// Bytes of the file mapped at `store_base`. - len: usize, - /// The root's layout, owned by `arena`. - layout: Layout, - arena: std.heap.ArenaAllocator, - - const Meta = extern struct { - magic: [8]u8 = store_magic, - base: u64 = store_base, - /// Offset of the first byte never handed out. - top: u64 = heap_offset, - /// Offset of the sketch's Store, or 0 before the first adopt. - root: u64 = 0, - /// Offsets of the first free block of each size class (2^i bytes), 0 - /// when none; a free block's first 8 bytes hold the next one's. - free: [64]u64 = @splat(0), - }; - - pub fn open(gpa: std.mem.Allocator, path: []const u8, fresh: bool) !StoreFile { - const fd = try posix.openat(posix.AT.FDCWD, path, .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600); - errdefer _ = posix.system.close(fd); - // The same file mapped by two processes would be two heaps writing - // over each other. - if (linux.errno(linux.flock(fd, posix.LOCK.EX | posix.LOCK.NB)) != .SUCCESS) return error.StoreInUse; - const reserved = posix.mmap(@ptrFromInt(store_base), store_reserve, .{}, .{ .TYPE = .PRIVATE, .ANONYMOUS = true, .NORESERVE = true, .FIXED_NOREPLACE = true }, -1, 0) catch return error.StoreAddressTaken; - // Old kernels treat FIXED_NOREPLACE as a hint. - if (@intFromPtr(reserved.ptr) != store_base) { - posix.munmap(reserved); - return error.StoreAddressTaken; - } - errdefer posix.munmap(reserved); - var self: StoreFile = .{ .fd = fd, .len = 0, .layout = .{}, .arena = .init(gpa) }; - errdefer self.arena.deinit(); - const size: usize = @intCast(try fileSize(fd)); - if (!fresh and size > 0) { - // Checked before mapping, which may lengthen the file: a file - // that is not a store is refused untouched, never wiped. - var m: Meta = undefined; - const read = linux.pread(fd, std.mem.asBytes(&m), @sizeOf(Meta), meta_offset); - if (size < heap_offset or read != @sizeOf(Meta) or !std.mem.eql(u8, &m.magic, &store_magic) or m.base != store_base or m.top > size) return error.NotAStoreFile; - try self.grow(size); - self.layout = try readLayout(self.arena.allocator(), self.region()) orelse .{}; - return self; - } - try self.wipe(); - return self; - } - - pub fn close(self: *StoreFile) void { - posix.munmap(@as([*]align(page) u8, @ptrFromInt(store_base))[0..store_reserve]); - _ = posix.system.close(self.fd); - self.arena.deinit(); - } - - /// Empties the store: no root, nothing allocated. - fn wipe(self: *StoreFile) !void { - // Mapped pages past the new end would fault: map the reserve back. - if (self.len > heap_offset) { - _ = try posix.mmap(@ptrFromInt(store_base + heap_offset), self.len - heap_offset, .{}, .{ .TYPE = .PRIVATE, .ANONYMOUS = true, .NORESERVE = true, .FIXED = true }, -1, 0); - self.len = heap_offset; - } - try truncate(self.fd, heap_offset); - try self.grow(heap_offset); - @memset(self.region(), 0); - self.header().* = .{ .magic = store_layout_magic, .size = 0, .alignment = 1, .hash = 0, .stride = 0, .used = 0, .field_count = 0, .strings_len = 0 }; - self.meta().* = .{}; - self.layout = .{}; - } - - /// Maps the file up to `need` bytes, growing it when it is shorter. - fn grow(self: *StoreFile, need: usize) !void { - if (need <= self.len) return; - const len = std.mem.alignForward(usize, @max(need, self.len * 2, 1 << 20), page); - if (len > store_reserve) return error.StoreFull; - if (try fileSize(self.fd) < len) try truncate(self.fd, len); - _ = try posix.mmap(@ptrFromInt(store_base + self.len), len - self.len, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED, .FIXED = true }, self.fd, self.len); - self.len = len; - } - - fn region(_: *StoreFile) []u8 { - return @as([*]u8, @ptrFromInt(store_base))[0..header_len]; - } - fn header(_: *StoreFile) *Header { - return @ptrFromInt(store_base); - } - fn meta(_: *StoreFile) *Meta { - return @ptrFromInt(store_base + meta_offset); - } - - /// The sketch's Store. - pub fn root(self: *StoreFile) [*]u8 { - const offset = self.meta().root; - return @ptrFromInt(store_base + if (offset == 0) heap_offset else offset); - } - - pub const Adoption = StateFile.Adoption; - - /// Makes the root hold `new`'s layout: the new defaults, then every field - /// whose path and type are unchanged, copied over. Pointer fields keep - /// pointing where they did, since the store never moves. - pub fn adopt(self: *StoreFile, gpa: std.mem.Allocator, new: abi.Layout, defaults: *const fn (abi.Memory, [*]u8) callconv(.c) void) !Adoption { - const m = self.meta(); - if (m.root != 0 and self.layout.hash == new.hash and self.layout.size == new.size) return .unchanged; - const a = self.allocator(); - const alignment: std.mem.Alignment = .fromByteUnits(@max(new.alignment, 1)); - const fresh = a.rawAlloc(@max(new.size, 1), alignment, @returnAddress()) orelse return error.StoreFull; - @memset(fresh[0..@max(new.size, 1)], 0); - defaults(.store, fresh); - - var kept = self.arena; - defer kept.deinit(); - self.arena = .init(gpa); - const old_layout = self.layout; - self.layout = try Layout.copy(self.arena.allocator(), new); - var result: Adoption = .created; - if (m.root != 0) { - const old: [*]u8 = @ptrFromInt(store_base + m.root); - result = .{ .migrated = migrate(old_layout, old[0..old_layout.size], self.layout, fresh[0..new.size]) }; - a.rawFree(old[0..@max(old_layout.size, 1)], .fromByteUnits(@max(old_layout.alignment, 1)), @returnAddress()); - } - self.header().* = .{ .magic = store_layout_magic, .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = 0, .used = 1, .field_count = 0, .strings_len = 0 }; - try writeLayout(self.region(), self.layout); - m.root = @intFromPtr(fresh) - store_base; - return result; - } - - pub fn allocator(self: *StoreFile) std.mem.Allocator { - return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; - } - - fn class(len: usize, alignment: std.mem.Alignment) u6 { - return @intCast(std.math.log2_int_ceil(usize, @max(len, alignment.toByteUnits(), 16))); - } - - fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { - const self: *StoreFile = @ptrCast(@alignCast(ctx)); - const m = self.meta(); - const c = class(len, alignment); - const size = @as(usize, 1) << c; - if (m.free[c] != 0) { - const block: [*]u8 = @ptrFromInt(store_base + m.free[c]); - if (alignment.check(@intFromPtr(block))) { - m.free[c] = @as(*const u64, @ptrCast(@alignCast(block))).*; - return block; - } - } - const offset = std.mem.alignForward(usize, m.top, @max(@min(size, page), alignment.toByteUnits())); - self.grow(offset + size) catch return null; - m.top = offset + size; - return @ptrFromInt(store_base + offset); - } - - /// In place within the block's size class. - fn resize(_: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, _: usize) bool { - return class(memory.len, alignment) == class(new_len, alignment); - } - - fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret: usize) ?[*]u8 { - return if (resize(ctx, memory, alignment, new_len, ret)) memory.ptr else null; - } - - fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, _: usize) void { - const self: *StoreFile = @ptrCast(@alignCast(ctx)); - const m = self.meta(); - const c = class(memory.len, alignment); - @as(*u64, @ptrCast(@alignCast(memory.ptr))).* = m.free[c]; - m.free[c] = @intFromPtr(memory.ptr) - store_base; - } -}; - -fn truncate(fd: posix.fd_t, len: usize) !void { - switch (posix.errno(posix.system.ftruncate(fd, @intCast(len)))) { - .SUCCESS => {}, - else => |err| return posix.unexpectedErrno(err), - } -} - /// Session memory: survives reloads while its layout hash is unchanged. /// Everything allocated from `heap` is released with it. pub const Session = struct { @@ -952,47 +746,6 @@ test "state file migrates live state, and a reopen reads the layout back" { try std.testing.expectEqual(V2{ .level = 1, .score = 99 }, @as(*V2, @ptrCast(file.slot(0).ptr)).*); } -test "a store keeps its root and what it points to across a reopen, and migrates by field" { - const V1 = struct { list: std.ArrayList(u32) = .empty, count: u32 = 0 }; - const V2 = struct { added: u8 = 7, list: std.ArrayList(u32) = .empty }; - const D = struct { - fn v1(_: abi.Memory, p: [*]u8) callconv(.c) void { - @as(*V1, @ptrCast(@alignCast(p))).* = .{}; - } - fn v2(_: abi.Memory, p: [*]u8) callconv(.c) void { - @as(*V2, @ptrCast(@alignCast(p))).* = .{}; - } - }; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var buf: [std.fs.max_path_bytes]u8 = undefined; - const path = try std.mem.print(&buf, ".zig-cache/tmp/{s}/test.store", .{&tmp.sub_path}); - const gpa = std.testing.allocator; - { - var store = try StoreFile.open(gpa, path, false); - defer store.close(); - try std.testing.expectEqual(StoreFile.Adoption.created, try store.adopt(gpa, abi.layout.of(V1, .store), D.v1)); - const v: *V1 = @ptrCast(@alignCast(store.root())); - for (0..10_000) |i| try v.list.append(store.allocator(), @intCast(i)); - v.count = 3; - try std.testing.expectError(error.StoreInUse, StoreFile.open(gpa, path, false)); - } - { - var store = try StoreFile.open(gpa, path, false); - defer store.close(); - try std.testing.expectEqual(StoreFile.Adoption.unchanged, try store.adopt(gpa, abi.layout.of(V1, .store), D.v1)); - const adoption = try store.adopt(gpa, abi.layout.of(V2, .store), D.v2); - try std.testing.expectEqual(@as(usize, 1), adoption.migrated.reset); - const v: *V2 = @ptrCast(@alignCast(store.root())); - try std.testing.expectEqual(@as(u8, 7), v.added); - try std.testing.expectEqual(@as(u32, 9999), v.list.items[9999]); - v.list.deinit(store.allocator()); - } - var store = try StoreFile.open(gpa, path, true); - defer store.close(); - try std.testing.expectEqual(StoreFile.Adoption.created, try store.adopt(gpa, abi.layout.of(V2, .store), D.v2)); -} - test "arrays migrate element by element: fields, enums by name, shrink and growth" { const E1 = struct { x: f32 = 0, mood: enum(u8) { calm = 1, wild = 2 } = .calm }; const E2 = struct { added: u8 = 9, mood: enum(u8) { wild = 5, calm = 6, sleepy = 7 } = .sleepy, x: f32 = 0 }; |
