summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/Db.zig157
1 files changed, 149 insertions, 8 deletions
diff --git a/src/Db.zig b/src/Db.zig
index 684604c7..b79bf315 100644
--- a/src/Db.zig
+++ b/src/Db.zig
@@ -12,11 +12,13 @@
//! this session's changes first (`recent.List.merge`). Held by another
//! session, the lock is tried again at the next step: a frame never waits
//! for one. Recent's commits are not fsynced, as its file never was: a crash
-//! keeps every one committed, a power loss may lose the last.
+//! keeps every one committed, a power loss may tear the last (its root
+//! header on disk, its bytes not), and the file is then opened at the commit
+//! before it (`mend`): the dumps outlive it.
//!
-//! A file there that is no database of pardes's (another program's, one a
-//! power loss tore) is kept beside it as `pardes.xitdb.broken` and a new one
-//! started: pardes always starts. The test host's is a `.memory` database,
+//! A file there that is no database of pardes's (another program's, one
+//! with no commit left whole) is kept beside it as `pardes.xitdb.broken` and
+//! a new one started: pardes always starts. The test host's is a `.memory` database,
//! the same code (`Kind.memory`).
const Db = @This();
@@ -128,8 +130,9 @@ fn opened(db: *Db, p: *Pardes) ?*Open {
}
/// The state file at `path`, its directory made: a new one is written under
-/// the file's lock (two sessions starting at once write one header), and a
-/// file that is no database of pardes's is put aside for a new one.
+/// the file's lock (two sessions starting at once write one header), one a
+/// power loss tore is opened at its last whole commit (`mend`), and a file
+/// that is no database of pardes's is put aside for a new one.
fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
pardes.turn.yield();
defer pardes.turn.back();
@@ -157,11 +160,18 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
o.where = .{ .disk = .{ .db = d, .file = file, .path = own } };
return o;
} else |err| {
- file.close(io);
switch (err) {
error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {},
- else => return err,
+ else => {
+ file.close(io);
+ return err;
+ },
}
+ if (!fresh) if (mend(io, file, &o.buffer)) |d| {
+ o.where = .{ .disk = .{ .db = d, .file = file, .path = own } };
+ return o;
+ };
+ file.close(io);
if (tried_aside) return err;
tried_aside = true;
var aside_buf: [4300]u8 = undefined;
@@ -170,6 +180,64 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
}
}
+// xitdb's layout, which a mend reads (zig-pkg's xitdb src/lib.zig): the
+// database header (its file id the last 8 of its 20 bytes, big endian),
+// the root header (the root slot's 9 bytes, then the committed size's 8),
+// and the trailer ending every commit (the root slot's 9 bytes, the commit
+// id's 8, and their CRC-64/XZ's 8, with the file id and its own position).
+const db_header_len = 20;
+const root_header_end = db_header_len + 17;
+const trailer_len = 25;
+const hash_map_tag = 4;
+
+/// A file xitdb refused, a power loss having torn its last commit (its
+/// root header written, the commit's bytes not all): opened at the last
+/// commit before it whose trailer verifies and whose maps read whole,
+/// scanned back from the file's end, its root header put back at it (what
+/// came after, a Recent's change or two, is lost; the dumps before it are
+/// not). Null, the file as it was, when none does: it is no database of
+/// pardes's, or none of its commits outlived the tear.
+fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk {
+ file.lock(io, .exclusive) catch return null;
+ defer file.unlock(io);
+ // Another session mended it first.
+ if (Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id })) |d| return d else |_| {}
+ var head: [root_header_end]u8 = undefined;
+ if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null;
+ const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big);
+ var chunk: [64 << 10]u8 = undefined;
+ var end = file.length(io) catch return null;
+ while (end >= root_header_end + trailer_len) {
+ const start = @max(root_header_end, end - @min(end, chunk.len));
+ const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null;
+ if (got != end - start) return null;
+ var i = got;
+ while (i >= trailer_len) : (i -= 1) {
+ const t = chunk[i - trailer_len ..][0..trailer_len];
+ const at = start + i - trailer_len;
+ // A pardes root is a map, at a slot written before its trailer.
+ if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue;
+ var sum: [33]u8 = undefined;
+ std.mem.writeInt(u64, sum[0..8], file_id, .big);
+ std.mem.writeInt(u64, sum[8..16], at, .big);
+ @memcpy(sum[16..33], t[0..17]);
+ if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue;
+ var root: [17]u8 = undefined;
+ @memcpy(root[0..9], t[0..9]);
+ std.mem.writeInt(u64, root[9..17], at + trailer_len, .big);
+ file.writePositionalAll(io, &root, db_header_len) catch return null;
+ var d = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id }) catch continue;
+ _ = liveSize(&d) catch continue;
+ file.sync(io) catch {};
+ return d;
+ }
+ if (start == root_header_end) break;
+ end = start + trailer_len - 1;
+ }
+ file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {};
+ return null;
+}
+
/// Whether the file at the open one's path is another now (a compaction's).
fn replaced(o: *Open) bool {
const d = o.where.disk;
@@ -1177,6 +1245,79 @@ test "state file: one that is no database of pardes's never stops pardes: it is
try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path);
}
+test "state file: one a power loss tore is opened at its last whole commit, the dumps before it kept; it is set aside only when none is whole" {
+ if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest;
+ var tmp = testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const dir = try testDir(&tmp, &dir_buf);
+ var label_buf: [64]u8 = undefined;
+ var label: []const u8 = undefined;
+ var dumped: u64 = undefined;
+ {
+ // A Dump, then a Recent's change committed after it.
+ const a = try testWindow(dir);
+ defer a.deinit();
+ _ = try a.setTestFile("kept through a tear\n");
+ try pardes.dump.dumpState(a);
+ pardes.host_io.writeDump(a, a.dump_out.?);
+ label = try std.fmt.bufPrint(&label_buf, "{s}", .{a.last_dump.?});
+ dumped = fileSize(dir);
+ a.recent.opened(a.recentGpa(), "/w/after.zig");
+ a.saveRecent();
+ }
+ const whole = try tmp.dir.readFileAlloc(testing.io, file_name, testing.allocator, .limited(64 << 20));
+ defer testing.allocator.free(whole);
+ try testing.expect(whole.len > dumped);
+ const torn = try testing.allocator.alloc(u8, whole.len + 4096);
+ defer testing.allocator.free(torn);
+ const cases = [_]struct { what: []const u8, len: usize, after: ?bool }{
+ // Its last commit cut short: the file ends before its root header says.
+ .{ .what = "cut short", .len = whole.len - 10, .after = null },
+ // A commit after the last whole one, its root header written, its
+ // bytes garbage: the last whole one's Recent is there.
+ .{ .what = "garbage after", .len = whole.len + 4096, .after = true },
+ // Recent's commit after the Dump never reached the disk (zeros where
+ // it would be) but its root header did: the Dump is there, the
+ // change gone.
+ .{ .what = "Recent's commit torn", .len = whole.len, .after = false },
+ };
+ for (cases) |case| {
+ @memcpy(torn[0..whole.len], whole);
+ @memset(torn[whole.len..], 0x5a);
+ if (std.mem.eql(u8, case.what, "garbage after")) std.mem.writeInt(u64, torn[root_header_end - 8 .. root_header_end], torn.len, .big);
+ if (std.mem.eql(u8, case.what, "Recent's commit torn")) @memset(torn[dumped..whole.len], 0);
+ try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = torn[0..case.len] });
+ const b = try testWindow(dir);
+ defer b.deinit();
+ errdefer std.debug.print("case: {s}\n", .{case.what});
+ try testing.expectError(error.FileNotFound, tmp.dir.statFile(testing.io, file_name ++ ".broken", .{}));
+ var buf: [64]u8 = undefined;
+ try testing.expect(b.db.labelled(b, label, &buf) != null);
+ if (case.after) |after| try testing.expectEqual(after, b.recent.get("/w/after.zig") != null);
+ // The Dump read back whole, its text with it.
+ const img = (try b.db.getDump(b, testing.allocator, label)).?;
+ defer testing.allocator.free(img);
+ const q = try pardes.dump.initFromDump(testing.allocator, .{ .tty_only = true }, img);
+ defer q.deinit();
+ const text = for (q.panes) |slot| {
+ if (slot) |pane| if (pane.file) |f| if (f.output == null) break f.content;
+ } else return error.FileLost;
+ try testing.expectEqualStrings("kept through a tear\n", text);
+ }
+ // No commit whole: set aside, the bytes as they were, and a new one started.
+ @memcpy(torn[0..whole.len], whole);
+ @memset(torn[root_header_end..whole.len], 0);
+ try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = torn[0..whole.len] });
+ const c = try testWindow(dir);
+ defer c.deinit();
+ var buf: [64]u8 = undefined;
+ try testing.expect(c.db.labelled(c, label, &buf) == null);
+ const aside = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(64 << 20));
+ defer testing.allocator.free(aside);
+ try testing.expectEqualSlices(u8, torn[0..whole.len], aside);
+}
+
test "state file: two dumps with one label get labels of their own" {
const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .state = .memory });
defer p.deinit();