diff options
| -rw-r--r-- | src/host/Sessions.zig | 230 |
1 files changed, 225 insertions, 5 deletions
diff --git a/src/host/Sessions.zig b/src/host/Sessions.zig index c5fc185..32b660b 100644 --- a/src/host/Sessions.zig +++ b/src/host/Sessions.zig @@ -25,7 +25,9 @@ //! //! One transaction (`commit`) a second of session time, and at once for a //! reload, a mark, a clip's start or end, and exit: a crash loses less than -//! a second of play; fsync only at exit and for a mark. +//! a second of play; fsync only at exit and for a mark. A file a crash or +//! power loss tore opens at its last whole commit (`mend`); a clean exit +//! compacts one that commits left past twice its contents (`close`). const Sessions = @This(); const std = @import("std"); const xitdb = @import("xitdb"); @@ -99,8 +101,9 @@ line: [256]u8 = undefined, line_len: usize = 0, /// Opens the database at `path` (file mode: created with its directory when -/// missing; a file xitdb refuses is renamed aside, never deleted) and starts -/// this run's session in it. +/// missing; one a crash or power loss tore is opened at its last whole +/// commit (`mend`); one still refused is renamed aside, never deleted) and +/// starts this run's session in it. pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, first: Stretch, started: u64) !Sessions { var s: Sessions = .{ .gpa = gpa, .io = io }; if (want == .off) return s; @@ -123,11 +126,20 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, fi s.file = file; break; } else |err| { - file.close(io); switch (err) { error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, - else => return err, + else => { + file.close(io); + return err; + }, + } + if (mend(io, file, buffer)) |db| { + s.db = .{ .file = db }; + s.file = file; + std.log.warn("{s} was torn after its last whole commit: opened at it", .{path}); + break; } + file.close(io); if (aside) return err; aside = true; var buf: [std.fs.max_path_bytes + 32]u8 = undefined; @@ -165,6 +177,106 @@ pub fn close(s: *Sessions, state: []const u8, ended: u64) void { s.stopClip() catch {}; _ = s.keyframe(.exit, state) catch {}; s.commitSynced(.{ .end = ended }) catch |err| std.log.warn("sessions: last commit failed: {t}", .{err}); + if (s.db != .file) return; + const size = s.file.?.length(s.io) catch return; + if (size < compact_floor) return; + const live = liveSize(&s.db.file) catch return; + if (size > 2 * live + compact_floor) s.compact() catch |err| std.log.warn("sessions: not compacted: {t}", .{err}); +} + +/// The database's file compacted at a clean exit, once it is past twice what +/// its batches and keyframes hold plus this: every commit leaves the +/// nodes it copied behind. +const compact_floor: u64 = 64 << 20; + +/// Writes what the database holds now, without the nodes commits left +/// behind, to `<path>.new`, synced, and renames it over the file. +fn compact(s: *Sessions) !void { + const cwd = std.Io.Dir.cwd(); + var buf: [std.fs.max_path_bytes + 8]u8 = undefined; + const new = try std.fmt.bufPrint(&buf, "{s}.new", .{s.path}); + const file = try cwd.createFile(s.io, new, .{ .read = true, .truncate = true }); + errdefer cwd.deleteFile(s.io, new) catch {}; + defer file.close(s.io); + var buffer: std.Io.Writer.Allocating = .init(s.gpa); + defer buffer.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(s.gpa); + defer offsets.deinit(); + _ = try s.db.file.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = true }, &offsets); + try cwd.rename(new, cwd, s.path, s.io); +} + +/// The bytes of every session's batches and keyframes: what compaction keeps. +/// Reading them all is also what proves a mended file whole. +fn liveSize(db: *Disk) !u64 { + const R = Read(Disk); + const sessions = try R.list(try R.root(db), .sessions) orelse return 0; + var live: u64 = 0; + var it = try sessions.iterator(); + while (try it.next()) |c| { + const session = try R.Map.init(c); + if (try R.list(session, .batches)) |batches| { + var b = try batches.iterator(); + while (try b.next()) |blob| live += try blob.count(); + } + if (try R.list(session, .keys)) |keys| { + var k = try keys.iterator(); + while (try k.next()) |key_cursor| live += try (try R.field(try R.Map.init(key_cursor), .state)).count(); + } + } + return live; +} + +// xitdb's layout, as `mend` reads it (zig-pkg's xitdb src/lib.zig): the +// database header (its file id the last 8 of its 20 bytes, big endian), the +// root header (the root slot's 9 bytes, then the committed size's 8), and +// the trailer ending every commit (the root slot's 9 bytes, the commit id's +// 8, and their CRC-64/XZ's 8, with the file id and its own position). +const db_header_len = 20; +const root_header_end = db_header_len + 17; +const trailer_len = 25; +const hash_map_tag = 4; + +/// A file xitdb refused because a crash or power loss tore its last commit +/// (its root header written, the commit's bytes not all): opened at the last +/// commit before the tear whose trailer verifies and whose sessions read +/// whole, scanned back from the file's end, its root header put back at it. +/// Null, the file as it was, when none does. +fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk { + var head: [root_header_end]u8 = undefined; + if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null; + const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big); + var chunk: [64 << 10]u8 = undefined; + var end = file.length(io) catch return null; + while (end >= root_header_end + trailer_len) { + const start = @max(root_header_end, end - @min(end, chunk.len)); + const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null; + if (got != end - start) return null; + var i = got; + while (i >= trailer_len) : (i -= 1) { + const t = chunk[i - trailer_len ..][0..trailer_len]; + const at = start + i - trailer_len; + // The root is a map, at a slot written before its trailer. + if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue; + var sum: [33]u8 = undefined; + std.mem.writeInt(u64, sum[0..8], file_id, .big); + std.mem.writeInt(u64, sum[8..16], at, .big); + @memcpy(sum[16..33], t[0..17]); + if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue; + var root: [17]u8 = undefined; + @memcpy(root[0..9], t[0..9]); + std.mem.writeInt(u64, root[9..17], at + trailer_len, .big); + file.writePositionalAll(io, &root, db_header_len) catch return null; + var db = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false }) catch continue; + _ = liveSize(&db) catch continue; + file.sync(io) catch {}; + return db; + } + if (start == root_header_end) break; + end = start + trailer_len - 1; + } + file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {}; + return null; } pub fn mode(s: *const Sessions) Mode { @@ -804,3 +916,111 @@ test "frames written to a session read back byte for byte, in memory and through try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items)); } } + +test "a session database torn inside its last commit opens at the one before, every earlier session and frame in it; one that cannot be mended is kept aside" { + const gpa = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path}); + const first: Stretch = .{ .build = 7, .source = "", .layout = "a layout" }; + var live: std.ArrayList(abi.Event) = .empty; + defer live.deinit(gpa); + var state: u64 = 0; + // Session 0: 120 frames, closed. Session 1: 180 frames in three commits + // of a second, then the process gone (no exit commit). + var before: u64 = 0; + var whole: u64 = 0; + for ([_]u64{ 120, 180 }, 0..) |frames, session| { + var s = try open(gpa, io, .file, path, first, 1000 + session); + for (0..frames) |f| { + const t = f * std.time.ns_per_s / 60; + var one = [_]abi.Event{abi.Event.tick(t, std.time.ns_per_s / 60)}; + try s.frame(&one, std.mem.asBytes(&state), if (f == 0) .start else null); + try live.appendSlice(gpa, &one); + state += 1; + } + if (session == 0) { + s.close(std.mem.asBytes(&state), 2000); + continue; + } + before = try s.file.?.length(io); + try s.commit(.none); + whole = try s.file.?.length(io); + s.deinit(); + } + const bytes = try tmp.dir.readFileAlloc(io, "t.xitdb", gpa, .unlimited); + defer gpa.free(bytes); + try std.testing.expectEqual(whole, bytes.len); + for ([_]u64{ before, before + 1, (before + whole) / 2, whole - trailer_len, whole - 10, whole - 1 }) |cut| { + errdefer std.debug.print("cut at {d} of {d}..{d}\n", .{ cut, before, whole }); + try tmp.dir.writeFile(io, .{ .sub_path = "t.xitdb", .data = bytes[0..cut] }); + var s = try open(gpa, io, .file, path, first, 3000); + defer s.deinit(); + try std.testing.expectEqual(@as(u64, 2), s.session); + const s0 = try s.summary(0); + try std.testing.expectEqual(@as(u64, 120), s0.frames); + try std.testing.expectEqual(@as(u64, 2000), s0.ended); + const s1 = try s.summary(1); + try std.testing.expectEqual(@as(u64, 120), s1.frames); + try std.testing.expectEqual(@as(u64, 0), s1.ended); + var back: std.ArrayList(abi.Event) = .empty; + defer back.deinit(gpa); + try s.events(0, 0, 0, 120, &back); + try s.events(1, 0, 0, 120, &back); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items[0..240]), std.mem.sliceAsBytes(back.items)); + var dir = try tmp.dir.openDir(io, ".", .{ .iterate = true }); + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |e| try std.testing.expectEqualStrings("t.xitdb", e.name); + } + // Nothing whole to open at: kept aside as it was, a new database begun. + var torn = try gpa.dupe(u8, bytes); + defer gpa.free(torn); + @memset(torn[root_header_end..], 0); + try tmp.dir.writeFile(io, .{ .sub_path = "t.xitdb", .data = torn }); + var s = try open(gpa, io, .file, path, first, 4 * std.time.ns_per_s); + defer s.deinit(); + try std.testing.expectEqual(@as(u64, 0), s.session); + const aside = try tmp.dir.readFileAlloc(io, "t.xitdb.broken.4", gpa, .unlimited); + defer gpa.free(aside); + try std.testing.expectEqualSlices(u8, torn, aside); +} + +test "a compacted session database is smaller and holds every session and frame" { + const gpa = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path}); + const first: Stretch = .{ .build = 7, .source = "", .layout = "a layout" }; + var live: std.ArrayList(abi.Event) = .empty; + defer live.deinit(gpa); + var s = try open(gpa, io, .file, path, first, 1000); + var state: u64 = 0; + for (0..600) |f| { + const t = f * std.time.ns_per_s / 60; + var one = [_]abi.Event{abi.Event.tick(t, std.time.ns_per_s / 60)}; + try s.frame(&one, std.mem.asBytes(&state), if (f == 0) .start else if (f % 100 == 0) .jump else null); + try live.appendSlice(gpa, &one); + state += 1; + } + try s.commit(.none); + const size = try s.file.?.length(io); + const live_size = try liveSize(&s.db.file); + try std.testing.expectEqual(@as(u64, 600 * @sizeOf(abi.Event) + 10 * 8 + 6 * 8), live_size); + try s.compact(); + s.deinit(); + const compacted = (try tmp.dir.statFile(io, "t.xitdb", .{})).size; + try std.testing.expect(compacted < size); + var again = try open(gpa, io, .file, path, first, 2000); + defer again.deinit(); + try std.testing.expectEqual(@as(u64, 600), (try again.summary(0)).frames); + try std.testing.expectEqual(@as(u64, 6), (try again.summary(0)).keys); + var back: std.ArrayList(abi.Event) = .empty; + defer back.deinit(gpa); + try again.events(0, 0, 0, 600, &back); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items)); +} |
