diff options
| author | Gabriel Schneider <[email protected]> | 2026-11-16 23:49:21 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-11-17 12:15:01 -0300 |
| commit | 3cf0b76e12f06a68cc0d84b2877f7f0261fe8d19 (patch) | |
| tree | c0e5685890395e3108d2fd761975f73d018fb43d /src/host/Sessions.zig | |
| parent | 5dc2c61f1df30d2dd84fbba6d0e8895a62c3b2b5 (diff) | |
| download | sketchbook-3cf0b76e12f06a68cc0d84b2877f7f0261fe8d19.tar.gz sketchbook-3cf0b76e12f06a68cc0d84b2877f7f0261fe8d19.zip | |
Mend a sessions database a crash tore, compact one at exit
A database whose last commit a power loss tore (its root header written,
its bytes not all) was refused and renamed aside, a new one begun. Now it
is opened at the last commit before the tear whose trailer verifies and
whose sessions read whole, scanned back from the end, as pardes's state
file is; renaming it to <path>.broken.<seconds> stays the last resort,
when no commit is whole, and nothing is deleted.
Every commit leaves the nodes it copied behind, about a kilobyte a second
of play: a clean exit compacts the file (into <path>.new, synced, renamed
over it) once it is 64 MiB and past twice what its batches and keyframes
hold plus 64 MiB.
Tests: a database of two sessions, cut at six places inside its last
commit, opens at the commit before with every frame of both; one with no
whole commit is kept aside byte for byte; a compacted one is smaller and
holds every frame and keyframe.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_016Y9MNBECMWcJD5pdS4SR6T
Diffstat (limited to 'src/host/Sessions.zig')
| -rw-r--r-- | src/host/Sessions.zig | 230 |
1 files changed, 225 insertions, 5 deletions
diff --git a/src/host/Sessions.zig b/src/host/Sessions.zig index c5fc185..32b660b 100644 --- a/src/host/Sessions.zig +++ b/src/host/Sessions.zig @@ -25,7 +25,9 @@ //! //! One transaction (`commit`) a second of session time, and at once for a //! reload, a mark, a clip's start or end, and exit: a crash loses less than -//! a second of play; fsync only at exit and for a mark. +//! a second of play; fsync only at exit and for a mark. A file a crash or +//! power loss tore opens at its last whole commit (`mend`); a clean exit +//! compacts one that commits left past twice its contents (`close`). const Sessions = @This(); const std = @import("std"); const xitdb = @import("xitdb"); @@ -99,8 +101,9 @@ line: [256]u8 = undefined, line_len: usize = 0, /// Opens the database at `path` (file mode: created with its directory when -/// missing; a file xitdb refuses is renamed aside, never deleted) and starts -/// this run's session in it. +/// missing; one a crash or power loss tore is opened at its last whole +/// commit (`mend`); one still refused is renamed aside, never deleted) and +/// starts this run's session in it. pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, first: Stretch, started: u64) !Sessions { var s: Sessions = .{ .gpa = gpa, .io = io }; if (want == .off) return s; @@ -123,11 +126,20 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, fi s.file = file; break; } else |err| { - file.close(io); switch (err) { error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, - else => return err, + else => { + file.close(io); + return err; + }, + } + if (mend(io, file, buffer)) |db| { + s.db = .{ .file = db }; + s.file = file; + std.log.warn("{s} was torn after its last whole commit: opened at it", .{path}); + break; } + file.close(io); if (aside) return err; aside = true; var buf: [std.fs.max_path_bytes + 32]u8 = undefined; @@ -165,6 +177,106 @@ pub fn close(s: *Sessions, state: []const u8, ended: u64) void { s.stopClip() catch {}; _ = s.keyframe(.exit, state) catch {}; s.commitSynced(.{ .end = ended }) catch |err| std.log.warn("sessions: last commit failed: {t}", .{err}); + if (s.db != .file) return; + const size = s.file.?.length(s.io) catch return; + if (size < compact_floor) return; + const live = liveSize(&s.db.file) catch return; + if (size > 2 * live + compact_floor) s.compact() catch |err| std.log.warn("sessions: not compacted: {t}", .{err}); +} + +/// The database's file compacted at a clean exit, once it is past twice what +/// its batches and keyframes hold plus this: every commit leaves the +/// nodes it copied behind. +const compact_floor: u64 = 64 << 20; + +/// Writes what the database holds now, without the nodes commits left +/// behind, to `<path>.new`, synced, and renames it over the file. +fn compact(s: *Sessions) !void { + const cwd = std.Io.Dir.cwd(); + var buf: [std.fs.max_path_bytes + 8]u8 = undefined; + const new = try std.fmt.bufPrint(&buf, "{s}.new", .{s.path}); + const file = try cwd.createFile(s.io, new, .{ .read = true, .truncate = true }); + errdefer cwd.deleteFile(s.io, new) catch {}; + defer file.close(s.io); + var buffer: std.Io.Writer.Allocating = .init(s.gpa); + defer buffer.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(s.gpa); + defer offsets.deinit(); + _ = try s.db.file.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = true }, &offsets); + try cwd.rename(new, cwd, s.path, s.io); +} + +/// The bytes of every session's batches and keyframes: what compaction keeps. +/// Reading them all is also what proves a mended file whole. +fn liveSize(db: *Disk) !u64 { + const R = Read(Disk); + const sessions = try R.list(try R.root(db), .sessions) orelse return 0; + var live: u64 = 0; + var it = try sessions.iterator(); + while (try it.next()) |c| { + const session = try R.Map.init(c); + if (try R.list(session, .batches)) |batches| { + var b = try batches.iterator(); + while (try b.next()) |blob| live += try blob.count(); + } + if (try R.list(session, .keys)) |keys| { + var k = try keys.iterator(); + while (try k.next()) |key_cursor| live += try (try R.field(try R.Map.init(key_cursor), .state)).count(); + } + } + return live; +} + +// xitdb's layout, as `mend` reads it (zig-pkg's xitdb src/lib.zig): the +// database header (its file id the last 8 of its 20 bytes, big endian), the +// root header (the root slot's 9 bytes, then the committed size's 8), and +// the trailer ending every commit (the root slot's 9 bytes, the commit id's +// 8, and their CRC-64/XZ's 8, with the file id and its own position). +const db_header_len = 20; +const root_header_end = db_header_len + 17; +const trailer_len = 25; +const hash_map_tag = 4; + +/// A file xitdb refused because a crash or power loss tore its last commit +/// (its root header written, the commit's bytes not all): opened at the last +/// commit before the tear whose trailer verifies and whose sessions read +/// whole, scanned back from the file's end, its root header put back at it. +/// Null, the file as it was, when none does. +fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk { + var head: [root_header_end]u8 = undefined; + if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null; + const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big); + var chunk: [64 << 10]u8 = undefined; + var end = file.length(io) catch return null; + while (end >= root_header_end + trailer_len) { + const start = @max(root_header_end, end - @min(end, chunk.len)); + const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null; + if (got != end - start) return null; + var i = got; + while (i >= trailer_len) : (i -= 1) { + const t = chunk[i - trailer_len ..][0..trailer_len]; + const at = start + i - trailer_len; + // The root is a map, at a slot written before its trailer. + if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue; + var sum: [33]u8 = undefined; + std.mem.writeInt(u64, sum[0..8], file_id, .big); + std.mem.writeInt(u64, sum[8..16], at, .big); + @memcpy(sum[16..33], t[0..17]); + if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue; + var root: [17]u8 = undefined; + @memcpy(root[0..9], t[0..9]); + std.mem.writeInt(u64, root[9..17], at + trailer_len, .big); + file.writePositionalAll(io, &root, db_header_len) catch return null; + var db = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false }) catch continue; + _ = liveSize(&db) catch continue; + file.sync(io) catch {}; + return db; + } + if (start == root_header_end) break; + end = start + trailer_len - 1; + } + file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {}; + return null; } pub fn mode(s: *const Sessions) Mode { @@ -804,3 +916,111 @@ test "frames written to a session read back byte for byte, in memory and through try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items)); } } + +test "a session database torn inside its last commit opens at the one before, every earlier session and frame in it; one that cannot be mended is kept aside" { + const gpa = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path}); + const first: Stretch = .{ .build = 7, .source = "", .layout = "a layout" }; + var live: std.ArrayList(abi.Event) = .empty; + defer live.deinit(gpa); + var state: u64 = 0; + // Session 0: 120 frames, closed. Session 1: 180 frames in three commits + // of a second, then the process gone (no exit commit). + var before: u64 = 0; + var whole: u64 = 0; + for ([_]u64{ 120, 180 }, 0..) |frames, session| { + var s = try open(gpa, io, .file, path, first, 1000 + session); + for (0..frames) |f| { + const t = f * std.time.ns_per_s / 60; + var one = [_]abi.Event{abi.Event.tick(t, std.time.ns_per_s / 60)}; + try s.frame(&one, std.mem.asBytes(&state), if (f == 0) .start else null); + try live.appendSlice(gpa, &one); + state += 1; + } + if (session == 0) { + s.close(std.mem.asBytes(&state), 2000); + continue; + } + before = try s.file.?.length(io); + try s.commit(.none); + whole = try s.file.?.length(io); + s.deinit(); + } + const bytes = try tmp.dir.readFileAlloc(io, "t.xitdb", gpa, .unlimited); + defer gpa.free(bytes); + try std.testing.expectEqual(whole, bytes.len); + for ([_]u64{ before, before + 1, (before + whole) / 2, whole - trailer_len, whole - 10, whole - 1 }) |cut| { + errdefer std.debug.print("cut at {d} of {d}..{d}\n", .{ cut, before, whole }); + try tmp.dir.writeFile(io, .{ .sub_path = "t.xitdb", .data = bytes[0..cut] }); + var s = try open(gpa, io, .file, path, first, 3000); + defer s.deinit(); + try std.testing.expectEqual(@as(u64, 2), s.session); + const s0 = try s.summary(0); + try std.testing.expectEqual(@as(u64, 120), s0.frames); + try std.testing.expectEqual(@as(u64, 2000), s0.ended); + const s1 = try s.summary(1); + try std.testing.expectEqual(@as(u64, 120), s1.frames); + try std.testing.expectEqual(@as(u64, 0), s1.ended); + var back: std.ArrayList(abi.Event) = .empty; + defer back.deinit(gpa); + try s.events(0, 0, 0, 120, &back); + try s.events(1, 0, 0, 120, &back); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items[0..240]), std.mem.sliceAsBytes(back.items)); + var dir = try tmp.dir.openDir(io, ".", .{ .iterate = true }); + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |e| try std.testing.expectEqualStrings("t.xitdb", e.name); + } + // Nothing whole to open at: kept aside as it was, a new database begun. + var torn = try gpa.dupe(u8, bytes); + defer gpa.free(torn); + @memset(torn[root_header_end..], 0); + try tmp.dir.writeFile(io, .{ .sub_path = "t.xitdb", .data = torn }); + var s = try open(gpa, io, .file, path, first, 4 * std.time.ns_per_s); + defer s.deinit(); + try std.testing.expectEqual(@as(u64, 0), s.session); + const aside = try tmp.dir.readFileAlloc(io, "t.xitdb.broken.4", gpa, .unlimited); + defer gpa.free(aside); + try std.testing.expectEqualSlices(u8, torn, aside); +} + +test "a compacted session database is smaller and holds every session and frame" { + const gpa = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path}); + const first: Stretch = .{ .build = 7, .source = "", .layout = "a layout" }; + var live: std.ArrayList(abi.Event) = .empty; + defer live.deinit(gpa); + var s = try open(gpa, io, .file, path, first, 1000); + var state: u64 = 0; + for (0..600) |f| { + const t = f * std.time.ns_per_s / 60; + var one = [_]abi.Event{abi.Event.tick(t, std.time.ns_per_s / 60)}; + try s.frame(&one, std.mem.asBytes(&state), if (f == 0) .start else if (f % 100 == 0) .jump else null); + try live.appendSlice(gpa, &one); + state += 1; + } + try s.commit(.none); + const size = try s.file.?.length(io); + const live_size = try liveSize(&s.db.file); + try std.testing.expectEqual(@as(u64, 600 * @sizeOf(abi.Event) + 10 * 8 + 6 * 8), live_size); + try s.compact(); + s.deinit(); + const compacted = (try tmp.dir.statFile(io, "t.xitdb", .{})).size; + try std.testing.expect(compacted < size); + var again = try open(gpa, io, .file, path, first, 2000); + defer again.deinit(); + try std.testing.expectEqual(@as(u64, 600), (try again.summary(0)).frames); + try std.testing.expectEqual(@as(u64, 6), (try again.summary(0)).keys); + var back: std.ArrayList(abi.Event) = .empty; + defer back.deinit(gpa); + try again.events(0, 0, 0, 600, &back); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items)); +} |
