//! What pardes keeps on disk, through xitdb (docs/next-store.md): the //! user's state file, `pardes.xitdb` in the DumpDir ($XDG_DATA_HOME/pardes, //! else ~/.local/share/pardes), which every session of the user's shares. //! Its root is a map: //! //! recent Recent's list (recent.zig), a line a file, the newest first //! //! A session reads Recent at its start, and again on use (Recent, /recent) //! when the file's committed size says another session committed since. A //! step that changed it writes it at its end in one transaction under the //! file's lock, what another session committed since read and merged under //! this session's changes first (`recent.List.merge`). Held by another //! session, the lock is tried again at the next step: a frame never waits //! for one. Recent's commits are not fsynced, as its file never was: a crash //! keeps every one committed, a power loss may tear the last (its root //! header on disk, its bytes not), and the file is then opened at the commit //! before it (`mend`): the dumps outlive it. //! //! A file there that is no database of pardes's (another program's, one //! with no commit left whole) is kept beside it as `pardes.xitdb.broken` and //! a new one started: pardes always starts. The test host's is a `.memory` database, //! the same code (`Kind.memory`). const Db = @This(); const std = @import("std"); const xitdb = @import("xitdb"); const pardes = @import("pardes.zig"); const recent = @import("recent.zig"); const Pardes = pardes.Pardes; pub const Disk = xitdb.Database(.buffered_file, u256); pub const Memory = xitdb.Database(.memory, u256); /// Where a session keeps what outlives it: nowhere (a unit test, a replica, /// a replay), its user's state file, or memory (the test host's). pub const Kind = enum { off, disk, memory }; pub const file_name = "pardes.xitdb"; /// The hash every key of these databases is: a field's or a label's name /// hashed, a blob keyed by its content's. const hash_id = xitdb.HashId.fromBytes("blk3"); /// Recent's lines are read this far at most (200 of the longest paths). const recent_max = 1 << 20; kind: Kind = .off, open: ?*Open = null, /// The state file's committed size when this session last read or wrote /// Recent: another session's commit moves it. seen: u64 = 0, /// Recent was read (or there was none): only then is it written, so a /// session never writes over a list it did not read. loaded: bool = false, /// The open database. Its own allocation: a database points at its buffer, /// and a Restore's new core takes it over from the old one. pub const Open = struct { gpa: std.mem.Allocator, buffer: std.Io.Writer.Allocating, where: union(enum) { disk: struct { db: Disk, file: std.Io.File, path: []u8 }, memory: struct { db: Memory }, }, }; fn ioOf() std.Io { return std.Io.Threaded.global_single_threaded.io(); } /// A name's key: its BLAKE3 hash. pub fn key(name: []const u8) u256 { var out: [32]u8 = undefined; std.crypto.hash.Blake3.hash(name, &out, .{}); return std.mem.readInt(u256, &out, .big); } pub fn deinit(db: *Db) void { const o = db.open orelse return; db.open = null; switch (o.where) { .disk => |*d| { d.file.close(ioOf()); o.gpa.free(d.path); }, .memory => {}, } o.buffer.deinit(); o.gpa.destroy(o); } /// The state file's path for the DumpDir setting `dir_setting`, made in /// `buf`; null with no home to keep it in. pub fn statePath(buf: []u8, dir_setting: []const u8) ?[]const u8 { var dir_buf: [4096]u8 = undefined; const dir = pardes.dump.directory(&dir_buf, dir_setting) orelse return null; return std.fmt.bufPrint(buf, "{s}/" ++ file_name, .{dir}) catch null; } /// The database, opened (made, when its file is not there) at its first use /// and again once the DumpDir names another: null when this session keeps /// none, or it cannot be had now (the next use tries again). fn opened(db: *Db, p: *Pardes) ?*Open { switch (db.kind) { .off => return null, .disk => { var buf: [4200]u8 = undefined; const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null; if (db.open) |o| { // The same file still: another session's compaction renames // a new one over it, which this one then opens. if (std.mem.eql(u8, o.where.disk.path, path) and !replaced(o)) return o; // The DumpDir moved: its own file, read afresh. db.deinit(); db.seen = 0; } db.open = openDisk(p.recentGpa(), path) catch return null; }, .memory => if (db.open == null) { const gpa = p.recentGpa(); const o = gpa.create(Open) catch return null; o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined }; o.where = .{ .memory = .{ .db = Memory.init(ioOf(), .{ .buffer = &o.buffer, .hash_id = hash_id }) catch { o.buffer.deinit(); gpa.destroy(o); return null; } } }; db.open = o; }, } return db.open; } /// The state file at `path`, its directory made: a new one is written under /// the file's lock (two sessions starting at once write one header), one a /// power loss tore is opened at its last whole commit (`mend`), and a file /// that is no database of pardes's is put aside for a new one. fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { pardes.turn.yield(); defer pardes.turn.back(); const io = ioOf(); const cwd = std.Io.Dir.cwd(); if (std.fs.path.dirname(path)) |dir| try cwd.createDirPath(io, dir); const o = try gpa.create(Open); errdefer gpa.destroy(o); o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined }; errdefer o.buffer.deinit(); const own = try gpa.dupe(u8, path); errdefer gpa.free(own); var tried_aside = false; while (true) { const file = try cwd.createFile(io, path, .{ .read = true, .truncate = false }); const fresh = (file.length(io) catch 0) == 0; // Another session making it now: it has its header in a moment. if (fresh and !(file.tryLock(io, .exclusive) catch false)) { file.close(io); return error.WouldBlock; } const made = Disk.init(io, .{ .file = file, .buffer = &o.buffer, .fsync = false, .hash_id = hash_id }); if (fresh) file.unlock(io); if (made) |d| { o.where = .{ .disk = .{ .db = d, .file = file, .path = own } }; return o; } else |err| { switch (err) { error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, else => { file.close(io); return err; }, } if (!fresh) if (mend(io, file, &o.buffer)) |d| { o.where = .{ .disk = .{ .db = d, .file = file, .path = own } }; return o; }; file.close(io); if (tried_aside) return err; tried_aside = true; var aside_buf: [4300]u8 = undefined; try cwd.rename(path, cwd, try std.fmt.bufPrint(&aside_buf, "{s}.broken", .{path}), io); } } } // xitdb's layout, which a mend reads (zig-pkg's xitdb src/lib.zig): the // database header (its file id the last 8 of its 20 bytes, big endian), // the root header (the root slot's 9 bytes, then the committed size's 8), // and the trailer ending every commit (the root slot's 9 bytes, the commit // id's 8, and their CRC-64/XZ's 8, with the file id and its own position). const db_header_len = 20; const root_header_end = db_header_len + 17; const trailer_len = 25; const hash_map_tag = 4; /// A file xitdb refused, a power loss having torn its last commit (its /// root header written, the commit's bytes not all): opened at the last /// commit before it whose trailer verifies and whose maps read whole, /// scanned back from the file's end, its root header put back at it (what /// came after, a Recent's change or two, is lost; the dumps before it are /// not). Null, the file as it was, when none does: it is no database of /// pardes's, or none of its commits outlived the tear. fn mend(io: std.Io, file: std.Io.File, buffer: *std.Io.Writer.Allocating) ?Disk { file.lock(io, .exclusive) catch return null; defer file.unlock(io); // Another session mended it first. if (Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id })) |d| return d else |_| {} var head: [root_header_end]u8 = undefined; if ((file.readPositionalAll(io, &head, 0) catch return null) != head.len or !std.mem.eql(u8, head[0..3], "xit")) return null; const file_id = std.mem.readInt(u64, head[db_header_len - 8 .. db_header_len], .big); var chunk: [64 << 10]u8 = undefined; var end = file.length(io) catch return null; while (end >= root_header_end + trailer_len) { const start = @max(root_header_end, end - @min(end, chunk.len)); const got = file.readPositionalAll(io, chunk[0 .. end - start], start) catch return null; if (got != end - start) return null; var i = got; while (i >= trailer_len) : (i -= 1) { const t = chunk[i - trailer_len ..][0..trailer_len]; const at = start + i - trailer_len; // A pardes root is a map, at a slot written before its trailer. if (t[0] & 0x7f != hash_map_tag or std.mem.readInt(u64, t[1..9], .big) >= at) continue; var sum: [33]u8 = undefined; std.mem.writeInt(u64, sum[0..8], file_id, .big); std.mem.writeInt(u64, sum[8..16], at, .big); @memcpy(sum[16..33], t[0..17]); if (std.hash.crc.@"CRC-64/XZ".hash(&sum) != std.mem.readInt(u64, t[17..25], .big)) continue; var root: [17]u8 = undefined; @memcpy(root[0..9], t[0..9]); std.mem.writeInt(u64, root[9..17], at + trailer_len, .big); file.writePositionalAll(io, &root, db_header_len) catch return null; var d = Disk.init(io, .{ .file = file, .buffer = buffer, .fsync = false, .hash_id = hash_id }) catch continue; _ = liveSize(&d) catch continue; file.sync(io) catch {}; return d; } if (start == root_header_end) break; end = start + trailer_len - 1; } file.writePositionalAll(io, head[db_header_len..], db_header_len) catch {}; return null; } /// Whether the file at the open one's path is another now (a compaction's). fn replaced(o: *Open) bool { const d = o.where.disk; pardes.turn.yield(); defer pardes.turn.back(); const io = ioOf(); const open_st = d.file.stat(io) catch return false; const path_st = std.Io.Dir.cwd().statFile(io, d.path, .{}) catch return false; return open_st.inode != path_st.inode; } fn committed(o: *Open) !u64 { pardes.turn.yield(); defer pardes.turn.back(); return switch (o.where) { inline else => |*w| w.db.committedSize(), }; } /// The file's lock, taken (`wait`: waited for): false when another session /// holds it. A database in memory is this process's alone. fn lock(o: *Open, wait: bool) bool { const file = switch (o.where) { .disk => |d| d.file, .memory => return true, }; pardes.turn.yield(); defer pardes.turn.back(); if (wait) { file.lock(ioOf(), .exclusive) catch return false; return true; } return file.tryLock(ioOf(), .exclusive) catch false; } /// The database, its file locked for a transaction (`wait`: waited for). A /// file a compaction replaced while this waited is let go, and the new one /// opened and locked. fn locked(db: *Db, p: *Pardes, wait: bool) error{ NoStateFile, Busy }!*Open { for (0..2) |_| { const o = db.opened(p) orelse return error.NoStateFile; if (!lock(o, wait)) return error.Busy; if (o.where == .memory or !replaced(o)) return o; unlock(o); db.deinit(); db.seen = 0; } return error.Busy; } fn unlock(o: *Open) void { switch (o.where) { .disk => |d| d.file.unlock(ioOf()), .memory => {}, } } // ---- Recent ---- /// Recent's lines, `gpa`'s; null when the database holds none. fn recentText(o: *Open, gpa: std.mem.Allocator) !?[]const u8 { pardes.turn.yield(); defer pardes.turn.back(); switch (o.where) { inline else => |*w| { const DB = @TypeOf(w.db); const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); const c = (try root.getCursor(key("recent"))) orelse return null; return try c.readBytesAlloc(gpa, recent_max); }, } } fn putRecent(o: *Open, text: []const u8) !void { pardes.turn.yield(); defer pardes.turn.back(); switch (o.where) { inline else => |*w| { const DB = @TypeOf(w.db); try w.db.runContext(struct { text: []const u8, pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { const root = try DB.HashMap(.read_write).init(cursor.*); try root.put(key("recent"), .{ .bytes = ctx.text }); } }{ .text = text }); }, } } /// What other sessions committed to Recent since this one last read or /// wrote it, read under this session's changes: at the start, and on use. pub fn readRecent(db: *Db, p: *Pardes) void { const o = db.opened(p) orelse return; const first = !db.loaded; db.loaded = true; const held = db.mergeRecent(p, o); // A state file that holds no Recent yet: the files before it, once. if (first and held != null and !held.?) if (p.opts.recent_import) |dir| importRecent(p, dir); } /// Whether the database holds Recent; null when nothing was read (nothing /// committed since the last read, or it could not be). fn mergeRecent(db: *Db, p: *Pardes, o: *Open) ?bool { const size = committed(o) catch return null; if (size == db.seen) return null; const gpa = p.recentGpa(); const text = recentText(o, gpa) catch return null; defer if (text) |t| gpa.free(t); p.recent.merge(gpa, text orelse ""); db.seen = size; return text != null; } /// Recent as the files before the state file kept it, in `dir`: 0.24's /// `recent` (200 lines, the newest first), else the `store` before it (a /// log whose `@r` records are such lines, the oldest first). Both are left /// where they are. ponytail: for a release or two, while those files are /// about. fn importRecent(p: *Pardes, dir: []const u8) void { const gpa = p.recentGpa(); const io = ioOf(); var path_buf: [4200]u8 = undefined; pardes.turn.yield(); const listed = std.Io.Dir.cwd().readFileAlloc(io, std.fmt.bufPrint(&path_buf, "{s}/recent", .{dir}) catch "", gpa, .limited(recent_max)) catch null; const logged = if (listed == null) std.Io.Dir.cwd().readFileAlloc(io, std.fmt.bufPrint(&path_buf, "{s}/store", .{dir}) catch "", gpa, .limited(128 << 20)) catch null else null; pardes.turn.back(); if (listed) |text| { defer gpa.free(text); p.recent.merge(gpa, text); // Written to the state file at the next step: read once. for (p.recent.entries.items) |*e| e.changed = true; p.recent.dirty = p.recent.entries.items.len > 0; } else if (logged) |text| { defer gpa.free(text); var lines = std.mem.splitScalar(u8, text, '\n'); while (lines.next()) |line| { const e = recent.List.parseLine(line) orelse continue; const open = if (p.recent.get(e.path)) |here| here.open else false; p.recent.opened(gpa, e.path); p.recent.closed(gpa, e.path, e.nums[0], e.nums[1], e.nums[2], e.nums[3]); // Open here, it stays open: the cap never drops it. if (open) p.recent.opened(gpa, e.path); } } } /// Recent written when it changed, at the end of a step and at the end /// (`wait`: the lock is waited for there, never in a step). pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { if (!p.recent.dirty) return; if (db.kind != .off) written: { if (!db.loaded) return; const o = db.locked(p, wait) catch |err| switch (err) { // Another session's lock: the next step tries again. error.Busy => return, error.NoStateFile => break :written, }; defer unlock(o); _ = db.mergeRecent(p, o); p.recent.changedFirst(); const gpa = p.recentGpa(); var text: std.Io.Writer.Allocating = .init(gpa); defer text.deinit(); const entries = p.recent.entries.items; for (entries[0..@min(entries.len, recent.cap)]) |e| recent.List.writeEntry(e, &text.writer) catch break :written; // The disk refused it: kept for no one, as before. putRecent(o, text.written()) catch break :written; db.seen = committed(o) catch 0; } for (p.recent.entries.items) |*e| e.changed = false; p.recent.dirty = false; } // ---- the dumps ---- // // Each Dump is a labelled entry of the state file's `dumps`, written in one // transaction, fsynced (a dump is worth the wait; Recent's commits are not): // a full disk or a kill in the middle leaves the dump before it whole, as // xitdb writes its header last. // // dumps/