summaryrefslogtreecommitdiff
path: root/src/host/memory.zig
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-11-17 00:45:31 -0300
committerGabriel Schneider <[email protected]>2026-11-17 12:15:01 -0300
commite699ffe4182f66e28180ea2d91ab1fb8c408d49e (patch)
tree0577374fb6164ef214cd727b662a23498ad547eb /src/host/memory.zig
parenta1ae80096bd69ee7d92ea9e4564e960d8553395e (diff)
downloadsketchbook-e699ffe4182f66e28180ea2d91ab1fb8c408d49e.tar.gz
sketchbook-e699ffe4182f66e28180ea2d91ab1fb8c408d49e.zip
Delete the Store; State is checked against the 64 MiB a save holds
A sketch's data is State: flat bytes, what grows has its capacity in its type. The Store, a heap in a file mapped at 0x2000_0000_0000 with 64 GiB reserved, its power-of-two allocator, its root adopt and migration, and its test, are gone from memory.zig; so are Context.store and store_root, Info.store, Memory.store, the Store declaration's glue, Init.store, Frame.store and Console.store, the web host's store in wasm memory, the host's StoreFile wiring and `<state>.store` file, the `.store` suffix in moving an old state into .sketchbook/, the build's tests' `rm` of it, and layout.check's `pointers` flag (State is the one checked memory now). Its one real user, micscope, moved off it in the change before. Old `.state.store` files are not read, converted or deleted, and stay ignored by .gitignore. A sketch that still declares a Store gets no error for the declaration: nothing reads it, and a `*Store` parameter is a compile error (no argument of that type). layout.of(T, .state) refuses a State bigger than layout.max_state, 64 MiB, one constant: "State is N bytes, and a save holds at most 64 MiB: lower a capacity (an array, an sk.List), or keep what need not be saved in Session." The pointer-hash gap the spec found (a Store pointer's element type changing kept the old bytes) is gone with the only persisted memory that held pointers. README: no Store row, allocator row, migration paragraph, file bullet or known limits; draw and command lose their store parameter; State's row says 64 MiB at most; data that grows and is the user's work goes in a List in State or, bigger than a save, in files the sketch appends to. Tests: zig build test, 166/166 steps (the store's unit test gone; new: a 65-MiB State fails to compile with the message above, by expect_errors). micscope, spheres, noc's e2e and drat-meout's test unchanged and green. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y9MNBECMWcJD5pdS4SR6T
Diffstat (limited to 'src/host/memory.zig')
-rw-r--r--src/host/memory.zig257
1 files changed, 5 insertions, 252 deletions
diff --git a/src/host/memory.zig b/src/host/memory.zig
index 044fc3b..796fb39 100644
--- a/src/host/memory.zig
+++ b/src/host/memory.zig
@@ -5,10 +5,6 @@
//! a probe can read it from another process, and a reload with a changed
//! layout can carry the old values over field by field.
//!
-//! `StoreFile`: a heap in a file, mapped at the same address in every run,
-//! so what it holds may point into it: the sketch's `Store` and everything
-//! allocated for it outlive restarts.
-//!
//! `Session`: this process's memory, kept across reloads.
const std = @import("std");
const posix = std.posix;
@@ -515,14 +511,14 @@ fn writeLayout(region: []u8, layout: Layout) !void {
h.strings_len = @intCast(strings);
}
-/// The layout stored in a state or store file's header, or null if it has
-/// none. Reads both formats, by magic: a v1 header (`file_magic_v1`,
-/// `store_layout_magic_v1`) has no element leaves, so its arrays are whole
-/// leaves only, which migrate while their type is unchanged.
+/// The layout stored in a state file's header, or null if it has none.
+/// Reads both formats, by magic: a v1 header (`file_magic_v1`) has no
+/// element leaves, so its arrays are whole leaves only, which migrate while
+/// their type is unchanged.
pub fn readLayout(arena: std.mem.Allocator, map: []const u8) !?Layout {
const h: *const Header = @ptrCast(@alignCast(map.ptr));
if (h.hash == 0 and h.field_count == 0) return null;
- const v1 = std.mem.eql(u8, &h.magic, &file_magic_v1) or std.mem.eql(u8, &h.magic, &store_layout_magic_v1);
+ const v1 = std.mem.eql(u8, &h.magic, &file_magic_v1);
const stride: usize = if (v1) @sizeOf(StoredFieldV1) else @sizeOf(StoredField);
const strings_start = @sizeOf(Header) + @as(usize, h.field_count) * stride;
if (strings_start + h.strings_len > header_len or map.len < header_len) return error.CorruptHeader;
@@ -582,208 +578,6 @@ pub fn mapReadOnly(path: []const u8) []align(page) const u8 {
return posix.mmap(null, @intCast(len), .{ .READ = true }, .{ .TYPE = .SHARED }, fd, 0) catch &.{};
}
-/// Where every store is mapped, in every process: far from where Linux puts
-/// executables (0x55...) and mappings (0x7f...).
-pub const store_base: usize = 0x2000_0000_0000;
-/// The most a store can grow to. Only reserved: pages exist as they are used.
-pub const store_reserve: usize = 64 << 30;
-/// The allocator's record (`Meta`); unchanged by the layout format.
-pub const store_magic = "SKSTORE1".*;
-/// The store's layout header, like `file_magic` and `file_magic_v1`.
-pub const store_layout_magic = "SKSTORE2".*;
-pub const store_layout_magic_v1 = "SKSTORE1".*;
-/// The layout region is a state file header; the allocator's own record
-/// follows, then the heap.
-const meta_offset = header_len;
-const heap_offset = 2 * header_len;
-
-/// A heap in a file, with the sketch's `Store` at its root. Mapped at
-/// `store_base` in every run, so pointers it holds into itself stay valid
-/// across restarts. Not crash-safe: a crash in the middle of an allocation
-/// can leave it inconsistent (`--fresh` starts over).
-///
-/// Blocks come in power-of-two sizes, each class with a free list threaded
-/// through the free blocks, so growing a list by doubling reuses what it
-/// frees. The file grows by mapping more of it in place: nothing moves.
-pub const StoreFile = struct {
- fd: posix.fd_t,
- /// Bytes of the file mapped at `store_base`.
- len: usize,
- /// The root's layout, owned by `arena`.
- layout: Layout,
- arena: std.heap.ArenaAllocator,
-
- const Meta = extern struct {
- magic: [8]u8 = store_magic,
- base: u64 = store_base,
- /// Offset of the first byte never handed out.
- top: u64 = heap_offset,
- /// Offset of the sketch's Store, or 0 before the first adopt.
- root: u64 = 0,
- /// Offsets of the first free block of each size class (2^i bytes), 0
- /// when none; a free block's first 8 bytes hold the next one's.
- free: [64]u64 = @splat(0),
- };
-
- pub fn open(gpa: std.mem.Allocator, path: []const u8, fresh: bool) !StoreFile {
- const fd = try posix.openat(posix.AT.FDCWD, path, .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600);
- errdefer _ = posix.system.close(fd);
- // The same file mapped by two processes would be two heaps writing
- // over each other.
- if (linux.errno(linux.flock(fd, posix.LOCK.EX | posix.LOCK.NB)) != .SUCCESS) return error.StoreInUse;
- const reserved = posix.mmap(@ptrFromInt(store_base), store_reserve, .{}, .{ .TYPE = .PRIVATE, .ANONYMOUS = true, .NORESERVE = true, .FIXED_NOREPLACE = true }, -1, 0) catch return error.StoreAddressTaken;
- // Old kernels treat FIXED_NOREPLACE as a hint.
- if (@intFromPtr(reserved.ptr) != store_base) {
- posix.munmap(reserved);
- return error.StoreAddressTaken;
- }
- errdefer posix.munmap(reserved);
- var self: StoreFile = .{ .fd = fd, .len = 0, .layout = .{}, .arena = .init(gpa) };
- errdefer self.arena.deinit();
- const size: usize = @intCast(try fileSize(fd));
- if (!fresh and size > 0) {
- // Checked before mapping, which may lengthen the file: a file
- // that is not a store is refused untouched, never wiped.
- var m: Meta = undefined;
- const read = linux.pread(fd, std.mem.asBytes(&m), @sizeOf(Meta), meta_offset);
- if (size < heap_offset or read != @sizeOf(Meta) or !std.mem.eql(u8, &m.magic, &store_magic) or m.base != store_base or m.top > size) return error.NotAStoreFile;
- try self.grow(size);
- self.layout = try readLayout(self.arena.allocator(), self.region()) orelse .{};
- return self;
- }
- try self.wipe();
- return self;
- }
-
- pub fn close(self: *StoreFile) void {
- posix.munmap(@as([*]align(page) u8, @ptrFromInt(store_base))[0..store_reserve]);
- _ = posix.system.close(self.fd);
- self.arena.deinit();
- }
-
- /// Empties the store: no root, nothing allocated.
- fn wipe(self: *StoreFile) !void {
- // Mapped pages past the new end would fault: map the reserve back.
- if (self.len > heap_offset) {
- _ = try posix.mmap(@ptrFromInt(store_base + heap_offset), self.len - heap_offset, .{}, .{ .TYPE = .PRIVATE, .ANONYMOUS = true, .NORESERVE = true, .FIXED = true }, -1, 0);
- self.len = heap_offset;
- }
- try truncate(self.fd, heap_offset);
- try self.grow(heap_offset);
- @memset(self.region(), 0);
- self.header().* = .{ .magic = store_layout_magic, .size = 0, .alignment = 1, .hash = 0, .stride = 0, .used = 0, .field_count = 0, .strings_len = 0 };
- self.meta().* = .{};
- self.layout = .{};
- }
-
- /// Maps the file up to `need` bytes, growing it when it is shorter.
- fn grow(self: *StoreFile, need: usize) !void {
- if (need <= self.len) return;
- const len = std.mem.alignForward(usize, @max(need, self.len * 2, 1 << 20), page);
- if (len > store_reserve) return error.StoreFull;
- if (try fileSize(self.fd) < len) try truncate(self.fd, len);
- _ = try posix.mmap(@ptrFromInt(store_base + self.len), len - self.len, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED, .FIXED = true }, self.fd, self.len);
- self.len = len;
- }
-
- fn region(_: *StoreFile) []u8 {
- return @as([*]u8, @ptrFromInt(store_base))[0..header_len];
- }
- fn header(_: *StoreFile) *Header {
- return @ptrFromInt(store_base);
- }
- fn meta(_: *StoreFile) *Meta {
- return @ptrFromInt(store_base + meta_offset);
- }
-
- /// The sketch's Store.
- pub fn root(self: *StoreFile) [*]u8 {
- const offset = self.meta().root;
- return @ptrFromInt(store_base + if (offset == 0) heap_offset else offset);
- }
-
- pub const Adoption = StateFile.Adoption;
-
- /// Makes the root hold `new`'s layout: the new defaults, then every field
- /// whose path and type are unchanged, copied over. Pointer fields keep
- /// pointing where they did, since the store never moves.
- pub fn adopt(self: *StoreFile, gpa: std.mem.Allocator, new: abi.Layout, defaults: *const fn (abi.Memory, [*]u8) callconv(.c) void) !Adoption {
- const m = self.meta();
- if (m.root != 0 and self.layout.hash == new.hash and self.layout.size == new.size) return .unchanged;
- const a = self.allocator();
- const alignment: std.mem.Alignment = .fromByteUnits(@max(new.alignment, 1));
- const fresh = a.rawAlloc(@max(new.size, 1), alignment, @returnAddress()) orelse return error.StoreFull;
- @memset(fresh[0..@max(new.size, 1)], 0);
- defaults(.store, fresh);
-
- var kept = self.arena;
- defer kept.deinit();
- self.arena = .init(gpa);
- const old_layout = self.layout;
- self.layout = try Layout.copy(self.arena.allocator(), new);
- var result: Adoption = .created;
- if (m.root != 0) {
- const old: [*]u8 = @ptrFromInt(store_base + m.root);
- result = .{ .migrated = migrate(old_layout, old[0..old_layout.size], self.layout, fresh[0..new.size]) };
- a.rawFree(old[0..@max(old_layout.size, 1)], .fromByteUnits(@max(old_layout.alignment, 1)), @returnAddress());
- }
- self.header().* = .{ .magic = store_layout_magic, .size = new.size, .alignment = new.alignment, .hash = new.hash, .stride = 0, .used = 1, .field_count = 0, .strings_len = 0 };
- try writeLayout(self.region(), self.layout);
- m.root = @intFromPtr(fresh) - store_base;
- return result;
- }
-
- pub fn allocator(self: *StoreFile) std.mem.Allocator {
- return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } };
- }
-
- fn class(len: usize, alignment: std.mem.Alignment) u6 {
- return @intCast(std.math.log2_int_ceil(usize, @max(len, alignment.toByteUnits(), 16)));
- }
-
- fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 {
- const self: *StoreFile = @ptrCast(@alignCast(ctx));
- const m = self.meta();
- const c = class(len, alignment);
- const size = @as(usize, 1) << c;
- if (m.free[c] != 0) {
- const block: [*]u8 = @ptrFromInt(store_base + m.free[c]);
- if (alignment.check(@intFromPtr(block))) {
- m.free[c] = @as(*const u64, @ptrCast(@alignCast(block))).*;
- return block;
- }
- }
- const offset = std.mem.alignForward(usize, m.top, @max(@min(size, page), alignment.toByteUnits()));
- self.grow(offset + size) catch return null;
- m.top = offset + size;
- return @ptrFromInt(store_base + offset);
- }
-
- /// In place within the block's size class.
- fn resize(_: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, _: usize) bool {
- return class(memory.len, alignment) == class(new_len, alignment);
- }
-
- fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ret: usize) ?[*]u8 {
- return if (resize(ctx, memory, alignment, new_len, ret)) memory.ptr else null;
- }
-
- fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, _: usize) void {
- const self: *StoreFile = @ptrCast(@alignCast(ctx));
- const m = self.meta();
- const c = class(memory.len, alignment);
- @as(*u64, @ptrCast(@alignCast(memory.ptr))).* = m.free[c];
- m.free[c] = @intFromPtr(memory.ptr) - store_base;
- }
-};
-
-fn truncate(fd: posix.fd_t, len: usize) !void {
- switch (posix.errno(posix.system.ftruncate(fd, @intCast(len)))) {
- .SUCCESS => {},
- else => |err| return posix.unexpectedErrno(err),
- }
-}
-
/// Session memory: survives reloads while its layout hash is unchanged.
/// Everything allocated from `heap` is released with it.
pub const Session = struct {
@@ -952,47 +746,6 @@ test "state file migrates live state, and a reopen reads the layout back" {
try std.testing.expectEqual(V2{ .level = 1, .score = 99 }, @as(*V2, @ptrCast(file.slot(0).ptr)).*);
}
-test "a store keeps its root and what it points to across a reopen, and migrates by field" {
- const V1 = struct { list: std.ArrayList(u32) = .empty, count: u32 = 0 };
- const V2 = struct { added: u8 = 7, list: std.ArrayList(u32) = .empty };
- const D = struct {
- fn v1(_: abi.Memory, p: [*]u8) callconv(.c) void {
- @as(*V1, @ptrCast(@alignCast(p))).* = .{};
- }
- fn v2(_: abi.Memory, p: [*]u8) callconv(.c) void {
- @as(*V2, @ptrCast(@alignCast(p))).* = .{};
- }
- };
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
- var buf: [std.fs.max_path_bytes]u8 = undefined;
- const path = try std.mem.print(&buf, ".zig-cache/tmp/{s}/test.store", .{&tmp.sub_path});
- const gpa = std.testing.allocator;
- {
- var store = try StoreFile.open(gpa, path, false);
- defer store.close();
- try std.testing.expectEqual(StoreFile.Adoption.created, try store.adopt(gpa, abi.layout.of(V1, .store), D.v1));
- const v: *V1 = @ptrCast(@alignCast(store.root()));
- for (0..10_000) |i| try v.list.append(store.allocator(), @intCast(i));
- v.count = 3;
- try std.testing.expectError(error.StoreInUse, StoreFile.open(gpa, path, false));
- }
- {
- var store = try StoreFile.open(gpa, path, false);
- defer store.close();
- try std.testing.expectEqual(StoreFile.Adoption.unchanged, try store.adopt(gpa, abi.layout.of(V1, .store), D.v1));
- const adoption = try store.adopt(gpa, abi.layout.of(V2, .store), D.v2);
- try std.testing.expectEqual(@as(usize, 1), adoption.migrated.reset);
- const v: *V2 = @ptrCast(@alignCast(store.root()));
- try std.testing.expectEqual(@as(u8, 7), v.added);
- try std.testing.expectEqual(@as(u32, 9999), v.list.items[9999]);
- v.list.deinit(store.allocator());
- }
- var store = try StoreFile.open(gpa, path, true);
- defer store.close();
- try std.testing.expectEqual(StoreFile.Adoption.created, try store.adopt(gpa, abi.layout.of(V2, .store), D.v2));
-}
-
test "arrays migrate element by element: fields, enums by name, shrink and growth" {
const E1 = struct { x: f32 = 0, mood: enum(u8) { calm = 1, wild = 2 } = .calm };
const E2 = struct { added: u8 = 9, mood: enum(u8) { wild = 5, calm = 6, sleepy = 7 } = .sleepy, x: f32 = 0 };