diff options
Diffstat (limited to 'src/Db.zig')
| -rw-r--r-- | src/Db.zig | 389 |
1 files changed, 389 insertions, 0 deletions
diff --git a/src/Db.zig b/src/Db.zig new file mode 100644 index 00000000..e496945f --- /dev/null +++ b/src/Db.zig @@ -0,0 +1,389 @@ +//! What pardes keeps on disk, through xitdb (docs/next-store.md): the +//! user's state file, `pardes.xitdb` in the DumpDir ($XDG_DATA_HOME/pardes, +//! else ~/.local/share/pardes), which every session of the user's shares. +//! Its root is a map: +//! +//! recent Recent's list (recent.zig), a line a file, the newest first +//! +//! A session reads Recent at its start, and again on use (Recent, /recent) +//! when the file's committed size says another session committed since. A +//! step that changed it writes it at its end in one transaction under the +//! file's lock, what another session committed since read and merged under +//! this session's changes first (`recent.List.merge`). Held by another +//! session, the lock is tried again at the next step: a frame never waits +//! for one. Recent's commits are not fsynced, as its file never was: a crash +//! keeps every one committed, a power loss may lose the last. +//! +//! A file there that is no database of pardes's (another program's, one a +//! power loss tore) is kept beside it as `pardes.xitdb.broken` and a new one +//! started: pardes always starts. The test host's is a `.memory` database, +//! the same code (`Kind.memory`). +const Db = @This(); + +const std = @import("std"); +const xitdb = @import("xitdb"); +const pardes = @import("pardes.zig"); +const recent = @import("recent.zig"); +const Pardes = pardes.Pardes; + +pub const Disk = xitdb.Database(.buffered_file, u256); +pub const Memory = xitdb.Database(.memory, u256); + +/// Where a session keeps what outlives it: nowhere (a unit test, a replica, +/// a replay), its user's state file, or memory (the test host's). +pub const Kind = enum { off, disk, memory }; + +pub const file_name = "pardes.xitdb"; +/// The hash every key of these databases is: a field's or a label's name +/// hashed, a blob keyed by its content's. +const hash_id = xitdb.HashId.fromBytes("blk3"); +/// Recent's lines are read this far at most (200 of the longest paths). +const recent_max = 1 << 20; + +kind: Kind = .off, +open: ?*Open = null, +/// The state file's committed size when this session last read or wrote +/// Recent: another session's commit moves it. +seen: u64 = 0, +/// Recent was read (or there was none): only then is it written, so a +/// session never writes over a list it did not read. +loaded: bool = false, + +/// The open database. Its own allocation: a database points at its buffer, +/// and a Restore's new core takes it over from the old one. +pub const Open = struct { + gpa: std.mem.Allocator, + buffer: std.Io.Writer.Allocating, + where: union(enum) { + disk: struct { db: Disk, file: std.Io.File, path: []u8 }, + memory: struct { db: Memory }, + }, +}; + +fn ioOf() std.Io { + return std.Io.Threaded.global_single_threaded.io(); +} + +/// A name's key: its BLAKE3 hash. +pub fn key(name: []const u8) u256 { + var out: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(name, &out, .{}); + return std.mem.readInt(u256, &out, .big); +} + +pub fn deinit(db: *Db) void { + const o = db.open orelse return; + db.open = null; + switch (o.where) { + .disk => |*d| { + d.file.close(ioOf()); + o.gpa.free(d.path); + }, + .memory => {}, + } + o.buffer.deinit(); + o.gpa.destroy(o); +} + +/// The state file's path for the DumpDir setting `dir_setting`, made in +/// `buf`; null with no home to keep it in. +pub fn statePath(buf: []u8, dir_setting: []const u8) ?[]const u8 { + var dir_buf: [4096]u8 = undefined; + const dir = pardes.dump.directory(&dir_buf, dir_setting) orelse return null; + return std.fmt.bufPrint(buf, "{s}/" ++ file_name, .{dir}) catch null; +} + +/// The database, opened (made, when its file is not there) at its first use +/// and again once the DumpDir names another: null when this session keeps +/// none, or it cannot be had now (the next use tries again). +fn opened(db: *Db, p: *Pardes) ?*Open { + switch (db.kind) { + .off => return null, + .disk => { + var buf: [4200]u8 = undefined; + const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null; + if (db.open) |o| { + if (std.mem.eql(u8, o.where.disk.path, path)) return o; + // The DumpDir moved: its own file, read afresh. + db.deinit(); + db.seen = 0; + } + db.open = openDisk(p.recentGpa(), path) catch return null; + }, + .memory => if (db.open == null) { + const gpa = p.recentGpa(); + const o = gpa.create(Open) catch return null; + o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined }; + o.where = .{ .memory = .{ .db = Memory.init(ioOf(), .{ .buffer = &o.buffer, .hash_id = hash_id }) catch { + o.buffer.deinit(); + gpa.destroy(o); + return null; + } } }; + db.open = o; + }, + } + return db.open; +} + +/// The state file at `path`, its directory made: a new one is written under +/// the file's lock (two sessions starting at once write one header), and a +/// file that is no database of pardes's is put aside for a new one. +fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { + pardes.turn.yield(); + defer pardes.turn.back(); + const io = ioOf(); + const cwd = std.Io.Dir.cwd(); + if (std.fs.path.dirname(path)) |dir| try cwd.createDirPath(io, dir); + const o = try gpa.create(Open); + errdefer gpa.destroy(o); + o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined }; + errdefer o.buffer.deinit(); + const own = try gpa.dupe(u8, path); + errdefer gpa.free(own); + var tried_aside = false; + while (true) { + const file = try cwd.createFile(io, path, .{ .read = true, .truncate = false }); + const fresh = (file.length(io) catch 0) == 0; + // Another session making it now: it has its header in a moment. + if (fresh and !(file.tryLock(io, .exclusive) catch false)) { + file.close(io); + return error.WouldBlock; + } + const made = Disk.init(io, .{ .file = file, .buffer = &o.buffer, .fsync = false, .hash_id = hash_id }); + if (fresh) file.unlock(io); + if (made) |d| { + o.where = .{ .disk = .{ .db = d, .file = file, .path = own } }; + return o; + } else |err| { + file.close(io); + switch (err) { + error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {}, + else => return err, + } + if (tried_aside) return err; + tried_aside = true; + var aside_buf: [4300]u8 = undefined; + try cwd.rename(path, cwd, try std.fmt.bufPrint(&aside_buf, "{s}.broken", .{path}), io); + } + } +} + +fn committed(o: *Open) !u64 { + pardes.turn.yield(); + defer pardes.turn.back(); + return switch (o.where) { + inline else => |*w| w.db.committedSize(), + }; +} + +/// The file's lock, taken (`wait`: waited for): false when another session +/// holds it. A database in memory is this process's alone. +fn lock(o: *Open, wait: bool) bool { + const file = switch (o.where) { + .disk => |d| d.file, + .memory => return true, + }; + pardes.turn.yield(); + defer pardes.turn.back(); + if (wait) { + file.lock(ioOf(), .exclusive) catch return false; + return true; + } + return file.tryLock(ioOf(), .exclusive) catch false; +} + +fn unlock(o: *Open) void { + switch (o.where) { + .disk => |d| d.file.unlock(ioOf()), + .memory => {}, + } +} + +// ---- Recent ---- + +/// Recent's lines, `gpa`'s; null when the database holds none. +fn recentText(o: *Open, gpa: std.mem.Allocator) !?[]const u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); + const c = (try root.getCursor(key("recent"))) orelse return null; + return try c.readBytesAlloc(gpa, recent_max); + }, + } +} + +fn putRecent(o: *Open, text: []const u8) !void { + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + try w.db.runContext(struct { + text: []const u8, + pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { + const root = try DB.HashMap(.read_write).init(cursor.*); + try root.put(key("recent"), .{ .bytes = ctx.text }); + } + }{ .text = text }); + }, + } +} + +/// What other sessions committed to Recent since this one last read or +/// wrote it, read under this session's changes: at the start, and on use. +pub fn readRecent(db: *Db, p: *Pardes) void { + const o = db.opened(p) orelse return; + db.loaded = true; + db.mergeRecent(p, o); +} + +fn mergeRecent(db: *Db, p: *Pardes, o: *Open) void { + const size = committed(o) catch return; + if (size == db.seen) return; + const gpa = p.recentGpa(); + const text = recentText(o, gpa) catch return; + defer if (text) |t| gpa.free(t); + p.recent.merge(gpa, text orelse ""); + db.seen = size; +} + +/// Recent written when it changed, at the end of a step and at the end +/// (`wait`: the lock is waited for there, never in a step). +pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { + if (!p.recent.dirty) return; + if (db.kind != .off) written: { + if (!db.loaded) return; + const o = db.opened(p) orelse break :written; + // Another session's lock: the next step tries again. + if (!lock(o, wait)) return; + defer unlock(o); + db.mergeRecent(p, o); + p.recent.changedFirst(); + const gpa = p.recentGpa(); + var text: std.Io.Writer.Allocating = .init(gpa); + defer text.deinit(); + const entries = p.recent.entries.items; + for (entries[0..@min(entries.len, recent.cap)]) |e| recent.List.writeEntry(e, &text.writer) catch break :written; + // The disk refused it: kept for no one, as before. + putRecent(o, text.written()) catch break :written; + db.seen = committed(o) catch 0; + } + for (p.recent.entries.items) |*e| e.changed = false; + p.recent.dirty = false; +} + +// ---- tests ---- + +const testing = std.testing; + +/// A window keeping its state in `dir` (never the user's DumpDir). +fn testWindow(dir: []const u8) !*Pardes { + var config: [4200]u8 = undefined; + return Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 48, .state = .disk, .startup_config = try std.fmt.bufPrint(&config, "DumpDir {s}\n", .{dir}) }); +} + +fn testDir(tmp: *std.testing.TmpDir, buf: *[std.fs.max_path_bytes]u8) ![]const u8 { + return buf[0..try tmp.dir.realPath(testing.io, buf)]; +} + +fn fileSize(dir: []const u8) u64 { + var buf: [4200]u8 = undefined; + const path = std.fmt.bufPrint(&buf, "{s}/" ++ file_name, .{dir}) catch return 0; + const st = std.Io.Dir.cwd().statFile(testing.io, path, .{}) catch return 0; + return st.size; +} + +test "state file: two windows share Recent, each reading the other's commits on use; a busy lock delays, never loses" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir = try testDir(&tmp, &dir_buf); + const a = try testWindow(dir); + defer a.deinit(); + const b = try testWindow(dir); + defer b.deinit(); + a.recent.opened(a.recentGpa(), "/w/a.zig"); + a.saveRecent(); + b.recent.opened(b.recentGpa(), "/w/b.zig"); + b.saveRecent(); + a.recent.closed(a.recentGpa(), "/w/a.zig", 7, 3, 0, 0); + a.saveRecent(); + // Each has both, in the order they were written: a closed last, then b. + a.db.readRecent(a); + b.db.readRecent(b); + for ([_]*Pardes{ a, b }) |w| { + try testing.expectEqual(@as(usize, 2), w.recent.entries.items.len); + try testing.expectEqualStrings("/w/a.zig", w.recent.entries.items[0].path); + try testing.expectEqualStrings("/w/b.zig", w.recent.entries.items[1].path); + try testing.expectEqual(@as(u32, 7), w.recent.get("/w/a.zig").?.line); + } + // Another window holding the file's lock: b's change waits in memory... + var path_buf: [4200]u8 = undefined; + const lock_file = try std.Io.Dir.cwd().openFile(testing.io, try std.fmt.bufPrint(&path_buf, "{s}/" ++ file_name, .{dir}), .{ .mode = .read_write }); + try lock_file.lock(testing.io, .exclusive); + b.recent.opened(b.recentGpa(), "/w/c.zig"); + const size = fileSize(dir); + b.saveRecent(); + try testing.expectEqual(size, fileSize(dir)); + try testing.expect(b.recent.dirty); + // ...and is written at the next step after it is let go. + lock_file.close(testing.io); + b.saveRecent(); + try testing.expect(fileSize(dir) > size); + a.db.readRecent(a); + try testing.expectEqualStrings("/w/c.zig", a.recent.entries.items[0].path); + // A window started now reads the same list. + const c = try testWindow(dir); + defer c.deinit(); + try testing.expectEqual(@as(usize, 3), c.recent.entries.items.len); + for (a.recent.entries.items, c.recent.entries.items) |x, y| try testing.expectEqualStrings(x.path, y.path); +} + +test "state file: Recent holds the newest 200, the newest first, and a window started on it reads them so" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir = try testDir(&tmp, &dir_buf); + const a = try testWindow(dir); + defer a.deinit(); + var name: [32]u8 = undefined; + for (0..recent.cap + 100) |i| { + const file = std.fmt.bufPrint(&name, "/w/file{d}.zig", .{i}) catch unreachable; + a.recent.opened(a.recentGpa(), file); + a.recent.closed(a.recentGpa(), file, 1, 1, 0, 0); + a.saveRecent(); + } + try testing.expectEqual(@as(usize, recent.cap), a.recent.entries.items.len); + const text = (try recentText(a.db.open.?, testing.allocator)).?; + defer testing.allocator.free(text); + try testing.expectEqual(@as(usize, recent.cap), std.mem.count(u8, text, "\n")); + try testing.expect(std.mem.startsWith(u8, text, "1 1 0 0 /w/file299.zig\n")); + const b = try testWindow(dir); + defer b.deinit(); + try testing.expectEqual(@as(usize, recent.cap), b.recent.entries.items.len); + try testing.expectEqualStrings("/w/file299.zig", b.recent.entries.items[0].path); + try testing.expectEqualStrings("/w/file100.zig", b.recent.entries.items[recent.cap - 1].path); +} + +test "state file: one that is no database of pardes's never stops pardes: it is kept aside, and a new one started" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir = try testDir(&tmp, &dir_buf); + try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = "\x00\xffnot a database at all\n7 3 0 0 /w/ok.zig\n" }); + const a = try testWindow(dir); + defer a.deinit(); + try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len); + a.recent.opened(a.recentGpa(), "/w/a.zig"); + a.saveRecent(); + const kept = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20)); + defer testing.allocator.free(kept); + try testing.expect(std.mem.startsWith(u8, kept, "\x00\xffnot a database")); + const b = try testWindow(dir); + defer b.deinit(); + try testing.expectEqual(@as(usize, 1), b.recent.entries.items.len); + try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path); +} |
