summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/host/Sessions.zig230
1 files changed, 225 insertions, 5 deletions
diff --git a/src/host/Sessions.zig b/src/host/Sessions.zig
index c5fc185..32b660b 100644
--- a/src/host/Sessions.zig
+++ b/src/host/Sessions.zig
@@ -25,7 +25,9 @@
//!
//! One transaction (`commit`) a second of session time, and at once for a
//! reload, a mark, a clip's start or end, and exit: a crash loses less than
-//! a second of play; fsync only at exit and for a mark.
+//! a second of play; fsync only at exit and for a mark. A file a crash or
+//! power loss tore opens at its last whole commit (`mend`); a clean exit
+//! compacts one that commits left past twice its contents (`close`).
const Sessions = @This();
const std = @import("std");
const xitdb = @import("xitdb");
@@ -99,8 +101,9 @@ line: [256]u8 = undefined,
line_len: usize = 0,
/// Opens the database at `path` (file mode: created with its directory when
-/// missing; a file xitdb refuses is renamed aside, never deleted) and starts
-/// this run's session in it.
+/// missing; one a crash or power loss tore is opened at its last whole
+/// commit (`mend`); one still refused is renamed aside, never deleted) and
+/// starts this run's session in it.
pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, first: Stretch, started: u64) !Sessions {
var s: Sessions = .{ .gpa = gpa, .io = io };
if (want == .off) return s;
@@ -123,11 +126,20 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, want: Mode, path: []const u8, fi
s.file = file;
break;
} else |err| {
- file.close(io);
switch (err) {
error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {},
- else => return err,
+ else => {
+ file.close(io);
+ return err;
+ },
+ }
+ if (mend(io, file, buffer)) |db| {
+ s.db = .{ .file = db };
+ s.file = file;
+ std.log.warn("{s} was torn after its last whole commit: opened at it", .{path});
+ break;
}
+ file.close(io);
if (aside) return err;
aside = true;
var buf: [std.fs.max_path_bytes + 32]u8 = undefined;
@@ -165,6 +177,106 @@ pub fn close(s: *Sessions, state: []const u8, ended: u64) void {
s.stopClip() catch {};
_ = s.keyframe(.exit, state) catch {};
s.commitSynced(.{ .end = ended }) catch |err| std.log.warn("sessions: last commit failed: {t}", .{err});
+ if (s.db != .file) return;
+ const size = s.file.?.length(s.io) catch return;
+ if (size < compact_floor) return;
+ const live = liveSize(&s.db.file) catch return;
+ if (size > 2 * live + compact_floor) s.compact() catch |err| std.log.warn("sessions: not compacted: {t}", .{err});
+}
+
+/// The database's file compacted at a clean exit, once it is past twice what
+/// its batches and keyframes hold plus this: every commit leaves the
+/// nodes it copied behind.
+const compact_floor: u64 = 64 << 20;
+
+/// Writes what the database holds now, without the nodes commits left
+/// behind, to `<path>.new`, synced, and renames it over the file.
+fn compact(s: *Sessions) !void {
+ const cwd = std.Io.Dir.cwd();
+ var buf: [std.fs.max_path_bytes + 8]u8 = undefined;
+ const new = try std.fmt.bufPrint(&buf, "{s}.new", .{s.path});
+ const file = try cwd.createFile(s.io, new, .{ .read = true, .truncate = true });
+ errdefer cwd.deleteFile(s.io, new) catch {};
+ defer file.close(s.io);
+ var buffer: std.Io.Writer.Allocating = .init(s.gpa);
+ defer buffer.deinit();
+ var offsets: std.AutoHashMap(u64, u64) = .init(s.gpa);
+ defer offsets.deinit();
+ _ = try s.db.file.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = true }, &offsets);
+ try cwd.rename(new, cwd, s.path, s.io);
+}
+
+/// The bytes of every session's batches and keyframes: what compaction keeps.
+/// Reading them all is also what proves a mended file whole.
+fn liveSize(db: *Disk) !u64 {
+ const R = Read(Disk);
+ const sessions = try R.list(try R.root(db), .sessions) orelse return 0;
+ var live: u64 = 0;
+ var it = try sessions.iterator();
+ while (try it.next()) |c| {
+ const session = try R.Map.init(c);
+ if (try R.list(session, .batches)) |batches| {
+ var b = try batches.iterator();
+ while (try b.next()) |blob| live += try blob.count();
+ }
+ if (try R.list(session, .keys)) |keys| {
+ var k = try keys.iterator();
+ while (try k.next()) |key_cursor| live += try (try R.field(try R.Map.init(key_cursor), .state)).count();
+ }
+ }
+ return live;
+}
+
+// xitdb's layout, as `mend` reads it (zig-pkg's xitdb src/lib.zig): the
+// database header (its file id the last 8 of its 20 bytes, big endian), the
+// root header (the root slot's 9 bytes, then the committed size's 8), and
+// the trailer ending every commit (the root slot's 9 bytes, the commit id's
+// 8, and their CRC-64/XZ's 8, with the file id and its own position).
+const db_header_len = 20;
+const root_header_end = db_header_len + 17;
+const trailer_len = 25;
+const hash_map_tag = 4;
+
+/// A file xitdb refused because a crash or power loss tore its last commit
+/// (its root header written, the commit's bytes not all): opened at the last
+/// commit before the tear whose trailer verifies and whose sessions read
+/// whole, scanned back from the file's end, its root header put back at it.
+/// Null, the file as it was, when none does.
+fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk {
+ var head: [root_header_end]u8 = undefined;
+ if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null;
+ const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big);
+ var chunk: [64 << 10]u8 = undefined;
+ var end = file.length(io) catch return null;
+ while (end >= root_header_end + trailer_len) {
+ const start = @max(root_header_end, end - @min(end, chunk.len));
+ const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null;
+ if (got != end - start) return null;
+ var i = got;
+ while (i >= trailer_len) : (i -= 1) {
+ const t = chunk[i - trailer_len ..][0..trailer_len];
+ const at = start + i - trailer_len;
+ // The root is a map, at a slot written before its trailer.
+ if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue;
+ var sum: [33]u8 = undefined;
+ std.mem.writeInt(u64, sum[0..8], file_id, .big);
+ std.mem.writeInt(u64, sum[8..16], at, .big);
+ @memcpy(sum[16..33], t[0..17]);
+ if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue;
+ var root: [17]u8 = undefined;
+ @memcpy(root[0..9], t[0..9]);
+ std.mem.writeInt(u64, root[9..17], at + trailer_len, .big);
+ file.writePositionalAll(io, &root, db_header_len) catch return null;
+ var db = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false }) catch continue;
+ _ = liveSize(&db) catch continue;
+ file.sync(io) catch {};
+ return db;
+ }
+ if (start == root_header_end) break;
+ end = start + trailer_len - 1;
+ }
+ file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {};
+ return null;
}
pub fn mode(s: *const Sessions) Mode {
@@ -804,3 +916,111 @@ test "frames written to a session read back byte for byte, in memory and through
try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items));
}
}
+
+test "a session database torn inside its last commit opens at the one before, every earlier session and frame in it; one that cannot be mended is kept aside" {
+ const gpa = std.testing.allocator;
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var buf: [std.fs.max_path_bytes]u8 = undefined;
+ const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path});
+ const first: Stretch = .{ .build = 7, .source = "", .layout = "a layout" };
+ var live: std.ArrayList(abi.Event) = .empty;
+ defer live.deinit(gpa);
+ var state: u64 = 0;
+ // Session 0: 120 frames, closed. Session 1: 180 frames in three commits
+ // of a second, then the process gone (no exit commit).
+ var before: u64 = 0;
+ var whole: u64 = 0;
+ for ([_]u64{ 120, 180 }, 0..) |frames, session| {
+ var s = try open(gpa, io, .file, path, first, 1000 + session);
+ for (0..frames) |f| {
+ const t = f * std.time.ns_per_s / 60;
+ var one = [_]abi.Event{abi.Event.tick(t, std.time.ns_per_s / 60)};
+ try s.frame(&one, std.mem.asBytes(&state), if (f == 0) .start else null);
+ try live.appendSlice(gpa, &one);
+ state += 1;
+ }
+ if (session == 0) {
+ s.close(std.mem.asBytes(&state), 2000);
+ continue;
+ }
+ before = try s.file.?.length(io);
+ try s.commit(.none);
+ whole = try s.file.?.length(io);
+ s.deinit();
+ }
+ const bytes = try tmp.dir.readFileAlloc(io, "t.xitdb", gpa, .unlimited);
+ defer gpa.free(bytes);
+ try std.testing.expectEqual(whole, bytes.len);
+ for ([_]u64{ before, before + 1, (before + whole) / 2, whole - trailer_len, whole - 10, whole - 1 }) |cut| {
+ errdefer std.debug.print("cut at {d} of {d}..{d}\n", .{ cut, before, whole });
+ try tmp.dir.writeFile(io, .{ .sub_path = "t.xitdb", .data = bytes[0..cut] });
+ var s = try open(gpa, io, .file, path, first, 3000);
+ defer s.deinit();
+ try std.testing.expectEqual(@as(u64, 2), s.session);
+ const s0 = try s.summary(0);
+ try std.testing.expectEqual(@as(u64, 120), s0.frames);
+ try std.testing.expectEqual(@as(u64, 2000), s0.ended);
+ const s1 = try s.summary(1);
+ try std.testing.expectEqual(@as(u64, 120), s1.frames);
+ try std.testing.expectEqual(@as(u64, 0), s1.ended);
+ var back: std.ArrayList(abi.Event) = .empty;
+ defer back.deinit(gpa);
+ try s.events(0, 0, 0, 120, &back);
+ try s.events(1, 0, 0, 120, &back);
+ try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items[0..240]), std.mem.sliceAsBytes(back.items));
+ var dir = try tmp.dir.openDir(io, ".", .{ .iterate = true });
+ defer dir.close(io);
+ var it = dir.iterate();
+ while (try it.next(io)) |e| try std.testing.expectEqualStrings("t.xitdb", e.name);
+ }
+ // Nothing whole to open at: kept aside as it was, a new database begun.
+ var torn = try gpa.dupe(u8, bytes);
+ defer gpa.free(torn);
+ @memset(torn[root_header_end..], 0);
+ try tmp.dir.writeFile(io, .{ .sub_path = "t.xitdb", .data = torn });
+ var s = try open(gpa, io, .file, path, first, 4 * std.time.ns_per_s);
+ defer s.deinit();
+ try std.testing.expectEqual(@as(u64, 0), s.session);
+ const aside = try tmp.dir.readFileAlloc(io, "t.xitdb.broken.4", gpa, .unlimited);
+ defer gpa.free(aside);
+ try std.testing.expectEqualSlices(u8, torn, aside);
+}
+
+test "a compacted session database is smaller and holds every session and frame" {
+ const gpa = std.testing.allocator;
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var buf: [std.fs.max_path_bytes]u8 = undefined;
+ const path = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/t.xitdb", .{&tmp.sub_path});
+ const first: Stretch = .{ .build = 7, .source = "", .layout = "a layout" };
+ var live: std.ArrayList(abi.Event) = .empty;
+ defer live.deinit(gpa);
+ var s = try open(gpa, io, .file, path, first, 1000);
+ var state: u64 = 0;
+ for (0..600) |f| {
+ const t = f * std.time.ns_per_s / 60;
+ var one = [_]abi.Event{abi.Event.tick(t, std.time.ns_per_s / 60)};
+ try s.frame(&one, std.mem.asBytes(&state), if (f == 0) .start else if (f % 100 == 0) .jump else null);
+ try live.appendSlice(gpa, &one);
+ state += 1;
+ }
+ try s.commit(.none);
+ const size = try s.file.?.length(io);
+ const live_size = try liveSize(&s.db.file);
+ try std.testing.expectEqual(@as(u64, 600 * @sizeOf(abi.Event) + 10 * 8 + 6 * 8), live_size);
+ try s.compact();
+ s.deinit();
+ const compacted = (try tmp.dir.statFile(io, "t.xitdb", .{})).size;
+ try std.testing.expect(compacted < size);
+ var again = try open(gpa, io, .file, path, first, 2000);
+ defer again.deinit();
+ try std.testing.expectEqual(@as(u64, 600), (try again.summary(0)).frames);
+ try std.testing.expectEqual(@as(u64, 6), (try again.summary(0)).keys);
+ var back: std.ArrayList(abi.Event) = .empty;
+ defer back.deinit(gpa);
+ try again.events(0, 0, 0, 600, &back);
+ try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(live.items), std.mem.sliceAsBytes(back.items));
+}