summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-11-05 19:36:39 -0300
committerGabriel Schneider <[email protected]>2026-11-05 19:36:48 -0300
commit992d5b1aa2813fcba05d8564872f8d426eade9f6 (patch)
tree11991c07584b361b2958cb7a1c7c19d76811c4b6
parentb3cf70c686d1505bc687cb1f0935959ffa5d119d (diff)
downloadpardes-992d5b1aa2813fcba05d8564872f8d426eade9f6.tar.gz
pardes-992d5b1aa2813fcba05d8564872f8d426eade9f6.zip
state file, a torn one mended: a power loss inside the writeback window after Recent's commit (not fsynced) can leave its root header on disk and its bytes not, and xitdb then refuses the file, which pardes set aside as `pardes.xitdb.broken` with every dump in it, whole but unreached; now the open scans back from the file's end for the last commit whose trailer verifies (its CRC-64 over the file id and its own place, its root a map written before it) and whose maps read whole, puts the root header back at it under the file's lock and opens the file there (src/Db.zig `mend`, xitdb's layout read by hand, xitdb itself checking the result), so the Recent's change or two after it are lost and the dumps before it are not; the file is set aside only when no commit is whole, its bytes as they were; on a 105 MB state file (Debug) a whole one opens in 78-93 us, one torn in Recent's last commit after a Dump in 5.0-5.5 ms, and one with no commit whole, scanned to its start, in 429-438 ms before it is set aside; tests: Db.zig's 'one a power loss tore is opened at its last whole commit, the dumps before it kept; it is set aside only when none is whole' is new (a file cut short in its last commit, one with garbage after its last commit and its root header pointing into it, one torn in Recent's commit after a Dump: the Dump read back and loaded each time, its text with it, Recent's change kept or gone as the tear says; one with no commit whole set aside byte for byte), and fails with the mend switched off
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge
-rw-r--r--src/Db.zig157
1 files changed, 149 insertions, 8 deletions
diff --git a/src/Db.zig b/src/Db.zig
index 684604c7..b79bf315 100644
--- a/src/Db.zig
+++ b/src/Db.zig
@@ -12,11 +12,13 @@
//! this session's changes first (`recent.List.merge`). Held by another
//! session, the lock is tried again at the next step: a frame never waits
//! for one. Recent's commits are not fsynced, as its file never was: a crash
-//! keeps every one committed, a power loss may lose the last.
+//! keeps every one committed, a power loss may tear the last (its root
+//! header on disk, its bytes not), and the file is then opened at the commit
+//! before it (`mend`): the dumps outlive it.
//!
-//! A file there that is no database of pardes's (another program's, one a
-//! power loss tore) is kept beside it as `pardes.xitdb.broken` and a new one
-//! started: pardes always starts. The test host's is a `.memory` database,
+//! A file there that is no database of pardes's (another program's, one
+//! with no commit left whole) is kept beside it as `pardes.xitdb.broken` and
+//! a new one started: pardes always starts. The test host's is a `.memory` database,
//! the same code (`Kind.memory`).
const Db = @This();
@@ -128,8 +130,9 @@ fn opened(db: *Db, p: *Pardes) ?*Open {
}
/// The state file at `path`, its directory made: a new one is written under
-/// the file's lock (two sessions starting at once write one header), and a
-/// file that is no database of pardes's is put aside for a new one.
+/// the file's lock (two sessions starting at once write one header), one a
+/// power loss tore is opened at its last whole commit (`mend`), and a file
+/// that is no database of pardes's is put aside for a new one.
fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
pardes.turn.yield();
defer pardes.turn.back();
@@ -157,11 +160,18 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
o.where = .{ .disk = .{ .db = d, .file = file, .path = own } };
return o;
} else |err| {
- file.close(io);
switch (err) {
error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {},
- else => return err,
+ else => {
+ file.close(io);
+ return err;
+ },
}
+ if (!fresh) if (mend(io, file, &o.buffer)) |d| {
+ o.where = .{ .disk = .{ .db = d, .file = file, .path = own } };
+ return o;
+ };
+ file.close(io);
if (tried_aside) return err;
tried_aside = true;
var aside_buf: [4300]u8 = undefined;
@@ -170,6 +180,64 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
}
}
+// xitdb's layout, which a mend reads (zig-pkg's xitdb src/lib.zig): the
+// database header (its file id the last 8 of its 20 bytes, big endian),
+// the root header (the root slot's 9 bytes, then the committed size's 8),
+// and the trailer ending every commit (the root slot's 9 bytes, the commit
+// id's 8, and their CRC-64/XZ's 8, with the file id and its own position).
+const db_header_len = 20;
+const root_header_end = db_header_len + 17;
+const trailer_len = 25;
+const hash_map_tag = 4;
+
+/// A file xitdb refused, a power loss having torn its last commit (its
+/// root header written, the commit's bytes not all): opened at the last
+/// commit before it whose trailer verifies and whose maps read whole,
+/// scanned back from the file's end, its root header put back at it (what
+/// came after, a Recent's change or two, is lost; the dumps before it are
+/// not). Null, the file as it was, when none does: it is no database of
+/// pardes's, or none of its commits outlived the tear.
+fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk {
+ file.lock(io, .exclusive) catch return null;
+ defer file.unlock(io);
+ // Another session mended it first.
+ if (Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id })) |d| return d else |_| {}
+ var head: [root_header_end]u8 = undefined;
+ if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null;
+ const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big);
+ var chunk: [64 << 10]u8 = undefined;
+ var end = file.length(io) catch return null;
+ while (end >= root_header_end + trailer_len) {
+ const start = @max(root_header_end, end - @min(end, chunk.len));
+ const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null;
+ if (got != end - start) return null;
+ var i = got;
+ while (i >= trailer_len) : (i -= 1) {
+ const t = chunk[i - trailer_len ..][0..trailer_len];
+ const at = start + i - trailer_len;
+ // A pardes root is a map, at a slot written before its trailer.
+ if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue;
+ var sum: [33]u8 = undefined;
+ std.mem.writeInt(u64, sum[0..8], file_id, .big);
+ std.mem.writeInt(u64, sum[8..16], at, .big);
+ @memcpy(sum[16..33], t[0..17]);
+ if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue;
+ var root: [17]u8 = undefined;
+ @memcpy(root[0..9], t[0..9]);
+ std.mem.writeInt(u64, root[9..17], at + trailer_len, .big);
+ file.writePositionalAll(io, &root, db_header_len) catch return null;
+ var d = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id }) catch continue;
+ _ = liveSize(&d) catch continue;
+ file.sync(io) catch {};
+ return d;
+ }
+ if (start == root_header_end) break;
+ end = start + trailer_len - 1;
+ }
+ file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {};
+ return null;
+}
+
/// Whether the file at the open one's path is another now (a compaction's).
fn replaced(o: *Open) bool {
const d = o.where.disk;
@@ -1177,6 +1245,79 @@ test "state file: one that is no database of pardes's never stops pardes: it is
try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path);
}
+test "state file: one a power loss tore is opened at its last whole commit, the dumps before it kept; it is set aside only when none is whole" {
+ if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest;
+ var tmp = testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const dir = try testDir(&tmp, &dir_buf);
+ var label_buf: [64]u8 = undefined;
+ var label: []const u8 = undefined;
+ var dumped: u64 = undefined;
+ {
+ // A Dump, then a Recent's change committed after it.
+ const a = try testWindow(dir);
+ defer a.deinit();
+ _ = try a.setTestFile("kept through a tear\n");
+ try pardes.dump.dumpState(a);
+ pardes.host_io.writeDump(a, a.dump_out.?);
+ label = try std.fmt.bufPrint(&label_buf, "{s}", .{a.last_dump.?});
+ dumped = fileSize(dir);
+ a.recent.opened(a.recentGpa(), "/w/after.zig");
+ a.saveRecent();
+ }
+ const whole = try tmp.dir.readFileAlloc(testing.io, file_name, testing.allocator, .limited(64 << 20));
+ defer testing.allocator.free(whole);
+ try testing.expect(whole.len > dumped);
+ const torn = try testing.allocator.alloc(u8, whole.len + 4096);
+ defer testing.allocator.free(torn);
+ const cases = [_]struct { what: []const u8, len: usize, after: ?bool }{
+ // Its last commit cut short: the file ends before its root header says.
+ .{ .what = "cut short", .len = whole.len - 10, .after = null },
+ // A commit after the last whole one, its root header written, its
+ // bytes garbage: the last whole one's Recent is there.
+ .{ .what = "garbage after", .len = whole.len + 4096, .after = true },
+ // Recent's commit after the Dump never reached the disk (zeros where
+ // it would be) but its root header did: the Dump is there, the
+ // change gone.
+ .{ .what = "Recent's commit torn", .len = whole.len, .after = false },
+ };
+ for (cases) |case| {
+ @memcpy(torn[0..whole.len], whole);
+ @memset(torn[whole.len..], 0x5a);
+ if (std.mem.eql(u8, case.what, "garbage after")) std.mem.writeInt(u64, torn[root_header_end - 8 .. root_header_end], torn.len, .big);
+ if (std.mem.eql(u8, case.what, "Recent's commit torn")) @memset(torn[dumped..whole.len], 0);
+ try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = torn[0..case.len] });
+ const b = try testWindow(dir);
+ defer b.deinit();
+ errdefer std.debug.print("case: {s}\n", .{case.what});
+ try testing.expectError(error.FileNotFound, tmp.dir.statFile(testing.io, file_name ++ ".broken", .{}));
+ var buf: [64]u8 = undefined;
+ try testing.expect(b.db.labelled(b, label, &buf) != null);
+ if (case.after) |after| try testing.expectEqual(after, b.recent.get("/w/after.zig") != null);
+ // The Dump read back whole, its text with it.
+ const img = (try b.db.getDump(b, testing.allocator, label)).?;
+ defer testing.allocator.free(img);
+ const q = try pardes.dump.initFromDump(testing.allocator, .{ .tty_only = true }, img);
+ defer q.deinit();
+ const text = for (q.panes) |slot| {
+ if (slot) |pane| if (pane.file) |f| if (f.output == null) break f.content;
+ } else return error.FileLost;
+ try testing.expectEqualStrings("kept through a tear\n", text);
+ }
+ // No commit whole: set aside, the bytes as they were, and a new one started.
+ @memcpy(torn[0..whole.len], whole);
+ @memset(torn[root_header_end..whole.len], 0);
+ try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = torn[0..whole.len] });
+ const c = try testWindow(dir);
+ defer c.deinit();
+ var buf: [64]u8 = undefined;
+ try testing.expect(c.db.labelled(c, label, &buf) == null);
+ const aside = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(64 << 20));
+ defer testing.allocator.free(aside);
+ try testing.expectEqualSlices(u8, torn[0..whole.len], aside);
+}
+
test "state file: two dumps with one label get labels of their own" {
const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .state = .memory });
defer p.deinit();