diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Db.zig | 157 |
1 files changed, 149 insertions, 8 deletions
@@ -12,11 +12,13 @@ //! this session's changes first (`recent.List.merge`). Held by another //! session, the lock is tried again at the next step: a frame never waits //! for one. Recent's commits are not fsynced, as its file never was: a crash -//! keeps every one committed, a power loss may lose the last. +//! keeps every one committed, a power loss may tear the last (its root +//! header on disk, its bytes not), and the file is then opened at the commit +//! before it (`mend`): the dumps outlive it. //! -//! A file there that is no database of pardes's (another program's, one a -//! power loss tore) is kept beside it as `pardes.xitdb.broken` and a new one -//! started: pardes always starts. The test host's is a `.memory` database, +//! A file there that is no database of pardes's (another program's, one +//! with no commit left whole) is kept beside it as `pardes.xitdb.broken` and +//! a new one started: pardes always starts. The test host's is a `.memory` database, //! the same code (`Kind.memory`). const Db = @This(); @@ -128,8 +130,9 @@ fn opened(db: *Db, p: *Pardes) ?*Open { } /// The state file at `path`, its directory made: a new one is written under -/// the file's lock (two sessions starting at once write one header), and a -/// file that is no database of pardes's is put aside for a new one. +/// the file's lock (two sessions starting at once write one header), one a +/// power loss tore is opened at its last whole commit (`mend`), and a file +/// that is no database of pardes's is put aside for a new one. fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { pardes.turn.yield(); defer pardes.turn.back(); @@ -157,11 +160,18 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { o.where = .{ .disk = .{ .db = d, .file = file, .path = own } }; return o; } else |err| { - file.close(io); switch (err) { error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, - else => return err, + else => { + file.close(io); + return err; + }, } + if (!fresh) if (mend(io, file, &o.buffer)) |d| { + o.where = .{ .disk = .{ .db = d, .file = file, .path = own } }; + return o; + }; + file.close(io); if (tried_aside) return err; tried_aside = true; var aside_buf: [4300]u8 = undefined; @@ -170,6 +180,64 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { } } +// xitdb's layout, which a mend reads (zig-pkg's xitdb src/lib.zig): the +// database header (its file id the last 8 of its 20 bytes, big endian), +// the root header (the root slot's 9 bytes, then the committed size's 8), +// and the trailer ending every commit (the root slot's 9 bytes, the commit +// id's 8, and their CRC-64/XZ's 8, with the file id and its own position). +const db_header_len = 20; +const root_header_end = db_header_len + 17; +const trailer_len = 25; +const hash_map_tag = 4; + +/// A file xitdb refused, a power loss having torn its last commit (its +/// root header written, the commit's bytes not all): opened at the last +/// commit before it whose trailer verifies and whose maps read whole, +/// scanned back from the file's end, its root header put back at it (what +/// came after, a Recent's change or two, is lost; the dumps before it are +/// not). Null, the file as it was, when none does: it is no database of +/// pardes's, or none of its commits outlived the tear. +fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk { + file.lock(io, .exclusive) catch return null; + defer file.unlock(io); + // Another session mended it first. + if (Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id })) |d| return d else |_| {} + var head: [root_header_end]u8 = undefined; + if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null; + const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big); + var chunk: [64 << 10]u8 = undefined; + var end = file.length(io) catch return null; + while (end >= root_header_end + trailer_len) { + const start = @max(root_header_end, end - @min(end, chunk.len)); + const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null; + if (got != end - start) return null; + var i = got; + while (i >= trailer_len) : (i -= 1) { + const t = chunk[i - trailer_len ..][0..trailer_len]; + const at = start + i - trailer_len; + // A pardes root is a map, at a slot written before its trailer. + if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue; + var sum: [33]u8 = undefined; + std.mem.writeInt(u64, sum[0..8], file_id, .big); + std.mem.writeInt(u64, sum[8..16], at, .big); + @memcpy(sum[16..33], t[0..17]); + if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue; + var root: [17]u8 = undefined; + @memcpy(root[0..9], t[0..9]); + std.mem.writeInt(u64, root[9..17], at + trailer_len, .big); + file.writePositionalAll(io, &root, db_header_len) catch return null; + var d = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id }) catch continue; + _ = liveSize(&d) catch continue; + file.sync(io) catch {}; + return d; + } + if (start == root_header_end) break; + end = start + trailer_len - 1; + } + file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {}; + return null; +} + /// Whether the file at the open one's path is another now (a compaction's). fn replaced(o: *Open) bool { const d = o.where.disk; @@ -1177,6 +1245,79 @@ test "state file: one that is no database of pardes's never stops pardes: it is try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path); } +test "state file: one a power loss tore is opened at its last whole commit, the dumps before it kept; it is set aside only when none is whole" { + if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir = try testDir(&tmp, &dir_buf); + var label_buf: [64]u8 = undefined; + var label: []const u8 = undefined; + var dumped: u64 = undefined; + { + // A Dump, then a Recent's change committed after it. + const a = try testWindow(dir); + defer a.deinit(); + _ = try a.setTestFile("kept through a tear\n"); + try pardes.dump.dumpState(a); + pardes.host_io.writeDump(a, a.dump_out.?); + label = try std.fmt.bufPrint(&label_buf, "{s}", .{a.last_dump.?}); + dumped = fileSize(dir); + a.recent.opened(a.recentGpa(), "/w/after.zig"); + a.saveRecent(); + } + const whole = try tmp.dir.readFileAlloc(testing.io, file_name, testing.allocator, .limited(64 << 20)); + defer testing.allocator.free(whole); + try testing.expect(whole.len > dumped); + const torn = try testing.allocator.alloc(u8, whole.len + 4096); + defer testing.allocator.free(torn); + const cases = [_]struct { what: []const u8, len: usize, after: ?bool }{ + // Its last commit cut short: the file ends before its root header says. + .{ .what = "cut short", .len = whole.len - 10, .after = null }, + // A commit after the last whole one, its root header written, its + // bytes garbage: the last whole one's Recent is there. + .{ .what = "garbage after", .len = whole.len + 4096, .after = true }, + // Recent's commit after the Dump never reached the disk (zeros where + // it would be) but its root header did: the Dump is there, the + // change gone. + .{ .what = "Recent's commit torn", .len = whole.len, .after = false }, + }; + for (cases) |case| { + @memcpy(torn[0..whole.len], whole); + @memset(torn[whole.len..], 0x5a); + if (std.mem.eql(u8, case.what, "garbage after")) std.mem.writeInt(u64, torn[root_header_end - 8 .. root_header_end], torn.len, .big); + if (std.mem.eql(u8, case.what, "Recent's commit torn")) @memset(torn[dumped..whole.len], 0); + try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = torn[0..case.len] }); + const b = try testWindow(dir); + defer b.deinit(); + errdefer std.debug.print("case: {s}\n", .{case.what}); + try testing.expectError(error.FileNotFound, tmp.dir.statFile(testing.io, file_name ++ ".broken", .{})); + var buf: [64]u8 = undefined; + try testing.expect(b.db.labelled(b, label, &buf) != null); + if (case.after) |after| try testing.expectEqual(after, b.recent.get("/w/after.zig") != null); + // The Dump read back whole, its text with it. + const img = (try b.db.getDump(b, testing.allocator, label)).?; + defer testing.allocator.free(img); + const q = try pardes.dump.initFromDump(testing.allocator, .{ .tty_only = true }, img); + defer q.deinit(); + const text = for (q.panes) |slot| { + if (slot) |pane| if (pane.file) |f| if (f.output == null) break f.content; + } else return error.FileLost; + try testing.expectEqualStrings("kept through a tear\n", text); + } + // No commit whole: set aside, the bytes as they were, and a new one started. + @memcpy(torn[0..whole.len], whole); + @memset(torn[root_header_end..whole.len], 0); + try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = torn[0..whole.len] }); + const c = try testWindow(dir); + defer c.deinit(); + var buf: [64]u8 = undefined; + try testing.expect(c.db.labelled(c, label, &buf) == null); + const aside = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(64 << 20)); + defer testing.allocator.free(aside); + try testing.expectEqualSlices(u8, torn[0..whole.len], aside); +} + test "state file: two dumps with one label get labels of their own" { const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .state = .memory }); defer p.deinit(); |
