//! What pardes keeps on disk, through xitdb (docs/next-store.md): the //! user's state file, `pardes.xitdb` in the DumpDir ($XDG_DATA_HOME/pardes, //! else ~/.local/share/pardes), which every session of the user's shares. //! Its root is a map: //! //! recent Recent's list (recent.zig), a line a file, the newest first //! //! A session reads Recent at its start, and again on use (Recent, /recent) //! when the file's committed size says another session committed since. A //! step that changed it writes it at its end in one transaction under the //! file's lock, what another session committed since read and merged under //! this session's changes first (`recent.List.merge`). Held by another //! session, the lock is tried again at the next step: a frame never waits //! for one. Recent's commits are not fsynced, as its file never was: a crash //! keeps every one committed, a power loss may lose the last. //! //! A file there that is no database of pardes's (another program's, one a //! power loss tore) is kept beside it as `pardes.xitdb.broken` and a new one //! started: pardes always starts. The test host's is a `.memory` database, //! the same code (`Kind.memory`). const Db = @This(); const std = @import("std"); const xitdb = @import("xitdb"); const pardes = @import("pardes.zig"); const recent = @import("recent.zig"); const Pardes = pardes.Pardes; pub const Disk = xitdb.Database(.buffered_file, u256); pub const Memory = xitdb.Database(.memory, u256); /// Where a session keeps what outlives it: nowhere (a unit test, a replica, /// a replay), its user's state file, or memory (the test host's). pub const Kind = enum { off, disk, memory }; pub const file_name = "pardes.xitdb"; /// The hash every key of these databases is: a field's or a label's name /// hashed, a blob keyed by its content's. const hash_id = xitdb.HashId.fromBytes("blk3"); /// Recent's lines are read this far at most (200 of the longest paths). const recent_max = 1 << 20; kind: Kind = .off, open: ?*Open = null, /// The state file's committed size when this session last read or wrote /// Recent: another session's commit moves it. seen: u64 = 0, /// Recent was read (or there was none): only then is it written, so a /// session never writes over a list it did not read. loaded: bool = false, /// The open database. Its own allocation: a database points at its buffer, /// and a Restore's new core takes it over from the old one. pub const Open = struct { gpa: std.mem.Allocator, buffer: std.Io.Writer.Allocating, where: union(enum) { disk: struct { db: Disk, file: std.Io.File, path: []u8 }, memory: struct { db: Memory }, }, }; fn ioOf() std.Io { return std.Io.Threaded.global_single_threaded.io(); } /// A name's key: its BLAKE3 hash. pub fn key(name: []const u8) u256 { var out: [32]u8 = undefined; std.crypto.hash.Blake3.hash(name, &out, .{}); return std.mem.readInt(u256, &out, .big); } pub fn deinit(db: *Db) void { const o = db.open orelse return; db.open = null; switch (o.where) { .disk => |*d| { d.file.close(ioOf()); o.gpa.free(d.path); }, .memory => {}, } o.buffer.deinit(); o.gpa.destroy(o); } /// The state file's path for the DumpDir setting `dir_setting`, made in /// `buf`; null with no home to keep it in. pub fn statePath(buf: []u8, dir_setting: []const u8) ?[]const u8 { var dir_buf: [4096]u8 = undefined; const dir = pardes.dump.directory(&dir_buf, dir_setting) orelse return null; return std.fmt.bufPrint(buf, "{s}/" ++ file_name, .{dir}) catch null; } /// The database, opened (made, when its file is not there) at its first use /// and again once the DumpDir names another: null when this session keeps /// none, or it cannot be had now (the next use tries again). fn opened(db: *Db, p: *Pardes) ?*Open { switch (db.kind) { .off => return null, .disk => { var buf: [4200]u8 = undefined; const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null; if (db.open) |o| { if (std.mem.eql(u8, o.where.disk.path, path)) return o; // The DumpDir moved: its own file, read afresh. db.deinit(); db.seen = 0; } db.open = openDisk(p.recentGpa(), path) catch return null; }, .memory => if (db.open == null) { const gpa = p.recentGpa(); const o = gpa.create(Open) catch return null; o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined }; o.where = .{ .memory = .{ .db = Memory.init(ioOf(), .{ .buffer = &o.buffer, .hash_id = hash_id }) catch { o.buffer.deinit(); gpa.destroy(o); return null; } } }; db.open = o; }, } return db.open; } /// The state file at `path`, its directory made: a new one is written under /// the file's lock (two sessions starting at once write one header), and a /// file that is no database of pardes's is put aside for a new one. fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { pardes.turn.yield(); defer pardes.turn.back(); const io = ioOf(); const cwd = std.Io.Dir.cwd(); if (std.fs.path.dirname(path)) |dir| try cwd.createDirPath(io, dir); const o = try gpa.create(Open); errdefer gpa.destroy(o); o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined }; errdefer o.buffer.deinit(); const own = try gpa.dupe(u8, path); errdefer gpa.free(own); var tried_aside = false; while (true) { const file = try cwd.createFile(io, path, .{ .read = true, .truncate = false }); const fresh = (file.length(io) catch 0) == 0; // Another session making it now: it has its header in a moment. if (fresh and !(file.tryLock(io, .exclusive) catch false)) { file.close(io); return error.WouldBlock; } const made = Disk.init(io, .{ .file = file, .buffer = &o.buffer, .fsync = false, .hash_id = hash_id }); if (fresh) file.unlock(io); if (made) |d| { o.where = .{ .disk = .{ .db = d, .file = file, .path = own } }; return o; } else |err| { file.close(io); switch (err) { error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, else => return err, } if (tried_aside) return err; tried_aside = true; var aside_buf: [4300]u8 = undefined; try cwd.rename(path, cwd, try std.fmt.bufPrint(&aside_buf, "{s}.broken", .{path}), io); } } } fn committed(o: *Open) !u64 { pardes.turn.yield(); defer pardes.turn.back(); return switch (o.where) { inline else => |*w| w.db.committedSize(), }; } /// The file's lock, taken (`wait`: waited for): false when another session /// holds it. A database in memory is this process's alone. fn lock(o: *Open, wait: bool) bool { const file = switch (o.where) { .disk => |d| d.file, .memory => return true, }; pardes.turn.yield(); defer pardes.turn.back(); if (wait) { file.lock(ioOf(), .exclusive) catch return false; return true; } return file.tryLock(ioOf(), .exclusive) catch false; } fn unlock(o: *Open) void { switch (o.where) { .disk => |d| d.file.unlock(ioOf()), .memory => {}, } } // ---- Recent ---- /// Recent's lines, `gpa`'s; null when the database holds none. fn recentText(o: *Open, gpa: std.mem.Allocator) !?[]const u8 { pardes.turn.yield(); defer pardes.turn.back(); switch (o.where) { inline else => |*w| { const DB = @TypeOf(w.db); const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); const c = (try root.getCursor(key("recent"))) orelse return null; return try c.readBytesAlloc(gpa, recent_max); }, } } fn putRecent(o: *Open, text: []const u8) !void { pardes.turn.yield(); defer pardes.turn.back(); switch (o.where) { inline else => |*w| { const DB = @TypeOf(w.db); try w.db.runContext(struct { text: []const u8, pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { const root = try DB.HashMap(.read_write).init(cursor.*); try root.put(key("recent"), .{ .bytes = ctx.text }); } }{ .text = text }); }, } } /// What other sessions committed to Recent since this one last read or /// wrote it, read under this session's changes: at the start, and on use. pub fn readRecent(db: *Db, p: *Pardes) void { const o = db.opened(p) orelse return; db.loaded = true; db.mergeRecent(p, o); } fn mergeRecent(db: *Db, p: *Pardes, o: *Open) void { const size = committed(o) catch return; if (size == db.seen) return; const gpa = p.recentGpa(); const text = recentText(o, gpa) catch return; defer if (text) |t| gpa.free(t); p.recent.merge(gpa, text orelse ""); db.seen = size; } /// Recent written when it changed, at the end of a step and at the end /// (`wait`: the lock is waited for there, never in a step). pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { if (!p.recent.dirty) return; if (db.kind != .off) written: { if (!db.loaded) return; const o = db.opened(p) orelse break :written; // Another session's lock: the next step tries again. if (!lock(o, wait)) return; defer unlock(o); db.mergeRecent(p, o); p.recent.changedFirst(); const gpa = p.recentGpa(); var text: std.Io.Writer.Allocating = .init(gpa); defer text.deinit(); const entries = p.recent.entries.items; for (entries[0..@min(entries.len, recent.cap)]) |e| recent.List.writeEntry(e, &text.writer) catch break :written; // The disk refused it: kept for no one, as before. putRecent(o, text.written()) catch break :written; db.seen = committed(o) catch 0; } for (p.recent.entries.items) |*e| e.changed = false; p.recent.dirty = false; } // ---- tests ---- const testing = std.testing; /// A window keeping its state in `dir` (never the user's DumpDir). fn testWindow(dir: []const u8) !*Pardes { var config: [4200]u8 = undefined; return Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 48, .state = .disk, .startup_config = try std.fmt.bufPrint(&config, "DumpDir {s}\n", .{dir}) }); } fn testDir(tmp: *std.testing.TmpDir, buf: *[std.fs.max_path_bytes]u8) ![]const u8 { return buf[0..try tmp.dir.realPath(testing.io, buf)]; } fn fileSize(dir: []const u8) u64 { var buf: [4200]u8 = undefined; const path = std.fmt.bufPrint(&buf, "{s}/" ++ file_name, .{dir}) catch return 0; const st = std.Io.Dir.cwd().statFile(testing.io, path, .{}) catch return 0; return st.size; } test "state file: two windows share Recent, each reading the other's commits on use; a busy lock delays, never loses" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); var dir_buf: [std.fs.max_path_bytes]u8 = undefined; const dir = try testDir(&tmp, &dir_buf); const a = try testWindow(dir); defer a.deinit(); const b = try testWindow(dir); defer b.deinit(); a.recent.opened(a.recentGpa(), "/w/a.zig"); a.saveRecent(); b.recent.opened(b.recentGpa(), "/w/b.zig"); b.saveRecent(); a.recent.closed(a.recentGpa(), "/w/a.zig", 7, 3, 0, 0); a.saveRecent(); // Each has both, in the order they were written: a closed last, then b. a.db.readRecent(a); b.db.readRecent(b); for ([_]*Pardes{ a, b }) |w| { try testing.expectEqual(@as(usize, 2), w.recent.entries.items.len); try testing.expectEqualStrings("/w/a.zig", w.recent.entries.items[0].path); try testing.expectEqualStrings("/w/b.zig", w.recent.entries.items[1].path); try testing.expectEqual(@as(u32, 7), w.recent.get("/w/a.zig").?.line); } // Another window holding the file's lock: b's change waits in memory... var path_buf: [4200]u8 = undefined; const lock_file = try std.Io.Dir.cwd().openFile(testing.io, try std.fmt.bufPrint(&path_buf, "{s}/" ++ file_name, .{dir}), .{ .mode = .read_write }); try lock_file.lock(testing.io, .exclusive); b.recent.opened(b.recentGpa(), "/w/c.zig"); const size = fileSize(dir); b.saveRecent(); try testing.expectEqual(size, fileSize(dir)); try testing.expect(b.recent.dirty); // ...and is written at the next step after it is let go. lock_file.close(testing.io); b.saveRecent(); try testing.expect(fileSize(dir) > size); a.db.readRecent(a); try testing.expectEqualStrings("/w/c.zig", a.recent.entries.items[0].path); // A window started now reads the same list. const c = try testWindow(dir); defer c.deinit(); try testing.expectEqual(@as(usize, 3), c.recent.entries.items.len); for (a.recent.entries.items, c.recent.entries.items) |x, y| try testing.expectEqualStrings(x.path, y.path); } test "state file: Recent holds the newest 200, the newest first, and a window started on it reads them so" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); var dir_buf: [std.fs.max_path_bytes]u8 = undefined; const dir = try testDir(&tmp, &dir_buf); const a = try testWindow(dir); defer a.deinit(); var name: [32]u8 = undefined; for (0..recent.cap + 100) |i| { const file = std.fmt.bufPrint(&name, "/w/file{d}.zig", .{i}) catch unreachable; a.recent.opened(a.recentGpa(), file); a.recent.closed(a.recentGpa(), file, 1, 1, 0, 0); a.saveRecent(); } try testing.expectEqual(@as(usize, recent.cap), a.recent.entries.items.len); const text = (try recentText(a.db.open.?, testing.allocator)).?; defer testing.allocator.free(text); try testing.expectEqual(@as(usize, recent.cap), std.mem.count(u8, text, "\n")); try testing.expect(std.mem.startsWith(u8, text, "1 1 0 0 /w/file299.zig\n")); const b = try testWindow(dir); defer b.deinit(); try testing.expectEqual(@as(usize, recent.cap), b.recent.entries.items.len); try testing.expectEqualStrings("/w/file299.zig", b.recent.entries.items[0].path); try testing.expectEqualStrings("/w/file100.zig", b.recent.entries.items[recent.cap - 1].path); } test "state file: one that is no database of pardes's never stops pardes: it is kept aside, and a new one started" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); var dir_buf: [std.fs.max_path_bytes]u8 = undefined; const dir = try testDir(&tmp, &dir_buf); try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = "\x00\xffnot a database at all\n7 3 0 0 /w/ok.zig\n" }); const a = try testWindow(dir); defer a.deinit(); try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len); a.recent.opened(a.recentGpa(), "/w/a.zig"); a.saveRecent(); const kept = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20)); defer testing.allocator.free(kept); try testing.expect(std.mem.startsWith(u8, kept, "\x00\xffnot a database")); const b = try testWindow(dir); defer b.deinit(); try testing.expectEqual(@as(usize, 1), b.recent.entries.items.len); try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path); }