summaryrefslogtreecommitdiff
path: root/src/Db.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/Db.zig')
-rw-r--r--src/Db.zig389
1 files changed, 389 insertions, 0 deletions
diff --git a/src/Db.zig b/src/Db.zig
new file mode 100644
index 00000000..e496945f
--- /dev/null
+++ b/src/Db.zig
@@ -0,0 +1,389 @@
+//! What pardes keeps on disk, through xitdb (docs/next-store.md): the
+//! user's state file, `pardes.xitdb` in the DumpDir ($XDG_DATA_HOME/pardes,
+//! else ~/.local/share/pardes), which every session of the user's shares.
+//! Its root is a map:
+//!
+//! recent Recent's list (recent.zig), a line a file, the newest first
+//!
+//! A session reads Recent at its start, and again on use (Recent, /recent)
+//! when the file's committed size says another session committed since. A
+//! step that changed it writes it at its end in one transaction under the
+//! file's lock, what another session committed since read and merged under
+//! this session's changes first (`recent.List.merge`). Held by another
+//! session, the lock is tried again at the next step: a frame never waits
+//! for one. Recent's commits are not fsynced, as its file never was: a crash
+//! keeps every one committed, a power loss may lose the last.
+//!
+//! A file there that is no database of pardes's (another program's, one a
+//! power loss tore) is kept beside it as `pardes.xitdb.broken` and a new one
+//! started: pardes always starts. The test host's is a `.memory` database,
+//! the same code (`Kind.memory`).
+const Db = @This();
+
+const std = @import("std");
+const xitdb = @import("xitdb");
+const pardes = @import("pardes.zig");
+const recent = @import("recent.zig");
+const Pardes = pardes.Pardes;
+
+pub const Disk = xitdb.Database(.buffered_file, u256);
+pub const Memory = xitdb.Database(.memory, u256);
+
+/// Where a session keeps what outlives it: nowhere (a unit test, a replica,
+/// a replay), its user's state file, or memory (the test host's).
+pub const Kind = enum { off, disk, memory };
+
+pub const file_name = "pardes.xitdb";
+/// The hash every key of these databases is: a field's or a label's name
+/// hashed, a blob keyed by its content's.
+const hash_id = xitdb.HashId.fromBytes("blk3");
+/// Recent's lines are read this far at most (200 of the longest paths).
+const recent_max = 1 << 20;
+
+kind: Kind = .off,
+open: ?*Open = null,
+/// The state file's committed size when this session last read or wrote
+/// Recent: another session's commit moves it.
+seen: u64 = 0,
+/// Recent was read (or there was none): only then is it written, so a
+/// session never writes over a list it did not read.
+loaded: bool = false,
+
+/// The open database. Its own allocation: a database points at its buffer,
+/// and a Restore's new core takes it over from the old one.
+pub const Open = struct {
+ gpa: std.mem.Allocator,
+ buffer: std.Io.Writer.Allocating,
+ where: union(enum) {
+ disk: struct { db: Disk, file: std.Io.File, path: []u8 },
+ memory: struct { db: Memory },
+ },
+};
+
+fn ioOf() std.Io {
+ return std.Io.Threaded.global_single_threaded.io();
+}
+
+/// A name's key: its BLAKE3 hash.
+pub fn key(name: []const u8) u256 {
+ var out: [32]u8 = undefined;
+ std.crypto.hash.Blake3.hash(name, &out, .{});
+ return std.mem.readInt(u256, &out, .big);
+}
+
+pub fn deinit(db: *Db) void {
+ const o = db.open orelse return;
+ db.open = null;
+ switch (o.where) {
+ .disk => |*d| {
+ d.file.close(ioOf());
+ o.gpa.free(d.path);
+ },
+ .memory => {},
+ }
+ o.buffer.deinit();
+ o.gpa.destroy(o);
+}
+
+/// The state file's path for the DumpDir setting `dir_setting`, made in
+/// `buf`; null with no home to keep it in.
+pub fn statePath(buf: []u8, dir_setting: []const u8) ?[]const u8 {
+ var dir_buf: [4096]u8 = undefined;
+ const dir = pardes.dump.directory(&dir_buf, dir_setting) orelse return null;
+ return std.fmt.bufPrint(buf, "{s}/" ++ file_name, .{dir}) catch null;
+}
+
+/// The database, opened (made, when its file is not there) at its first use
+/// and again once the DumpDir names another: null when this session keeps
+/// none, or it cannot be had now (the next use tries again).
+fn opened(db: *Db, p: *Pardes) ?*Open {
+ switch (db.kind) {
+ .off => return null,
+ .disk => {
+ var buf: [4200]u8 = undefined;
+ const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null;
+ if (db.open) |o| {
+ if (std.mem.eql(u8, o.where.disk.path, path)) return o;
+ // The DumpDir moved: its own file, read afresh.
+ db.deinit();
+ db.seen = 0;
+ }
+ db.open = openDisk(p.recentGpa(), path) catch return null;
+ },
+ .memory => if (db.open == null) {
+ const gpa = p.recentGpa();
+ const o = gpa.create(Open) catch return null;
+ o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined };
+ o.where = .{ .memory = .{ .db = Memory.init(ioOf(), .{ .buffer = &o.buffer, .hash_id = hash_id }) catch {
+ o.buffer.deinit();
+ gpa.destroy(o);
+ return null;
+ } } };
+ db.open = o;
+ },
+ }
+ return db.open;
+}
+
+/// The state file at `path`, its directory made: a new one is written under
+/// the file's lock (two sessions starting at once write one header), and a
+/// file that is no database of pardes's is put aside for a new one.
+fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ const io = ioOf();
+ const cwd = std.Io.Dir.cwd();
+ if (std.fs.path.dirname(path)) |dir| try cwd.createDirPath(io, dir);
+ const o = try gpa.create(Open);
+ errdefer gpa.destroy(o);
+ o.* = .{ .gpa = gpa, .buffer = .init(gpa), .where = undefined };
+ errdefer o.buffer.deinit();
+ const own = try gpa.dupe(u8, path);
+ errdefer gpa.free(own);
+ var tried_aside = false;
+ while (true) {
+ const file = try cwd.createFile(io, path, .{ .read = true, .truncate = false });
+ const fresh = (file.length(io) catch 0) == 0;
+ // Another session making it now: it has its header in a moment.
+ if (fresh and !(file.tryLock(io, .exclusive) catch false)) {
+ file.close(io);
+ return error.WouldBlock;
+ }
+ const made = Disk.init(io, .{ .file = file, .buffer = &o.buffer, .fsync = false, .hash_id = hash_id });
+ if (fresh) file.unlock(io);
+ if (made) |d| {
+ o.where = .{ .disk = .{ .db = d, .file = file, .path = own } };
+ return o;
+ } else |err| {
+ file.close(io);
+ switch (err) {
+ error.InvalidDatabase, error.InvalidVersion, error.InvalidHashSize, error.TruncatedDatabase, error.InvalidEnumTag, error.EndOfStream => {},
+ else => return err,
+ }
+ if (tried_aside) return err;
+ tried_aside = true;
+ var aside_buf: [4300]u8 = undefined;
+ try cwd.rename(path, cwd, try std.fmt.bufPrint(&aside_buf, "{s}.broken", .{path}), io);
+ }
+ }
+}
+
+fn committed(o: *Open) !u64 {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ return switch (o.where) {
+ inline else => |*w| w.db.committedSize(),
+ };
+}
+
+/// The file's lock, taken (`wait`: waited for): false when another session
+/// holds it. A database in memory is this process's alone.
+fn lock(o: *Open, wait: bool) bool {
+ const file = switch (o.where) {
+ .disk => |d| d.file,
+ .memory => return true,
+ };
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ if (wait) {
+ file.lock(ioOf(), .exclusive) catch return false;
+ return true;
+ }
+ return file.tryLock(ioOf(), .exclusive) catch false;
+}
+
+fn unlock(o: *Open) void {
+ switch (o.where) {
+ .disk => |d| d.file.unlock(ioOf()),
+ .memory => {},
+ }
+}
+
+// ---- Recent ----
+
+/// Recent's lines, `gpa`'s; null when the database holds none.
+fn recentText(o: *Open, gpa: std.mem.Allocator) !?[]const u8 {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ switch (o.where) {
+ inline else => |*w| {
+ const DB = @TypeOf(w.db);
+ const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly());
+ const c = (try root.getCursor(key("recent"))) orelse return null;
+ return try c.readBytesAlloc(gpa, recent_max);
+ },
+ }
+}
+
+fn putRecent(o: *Open, text: []const u8) !void {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ switch (o.where) {
+ inline else => |*w| {
+ const DB = @TypeOf(w.db);
+ try w.db.runContext(struct {
+ text: []const u8,
+ pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void {
+ const root = try DB.HashMap(.read_write).init(cursor.*);
+ try root.put(key("recent"), .{ .bytes = ctx.text });
+ }
+ }{ .text = text });
+ },
+ }
+}
+
+/// What other sessions committed to Recent since this one last read or
+/// wrote it, read under this session's changes: at the start, and on use.
+pub fn readRecent(db: *Db, p: *Pardes) void {
+ const o = db.opened(p) orelse return;
+ db.loaded = true;
+ db.mergeRecent(p, o);
+}
+
+fn mergeRecent(db: *Db, p: *Pardes, o: *Open) void {
+ const size = committed(o) catch return;
+ if (size == db.seen) return;
+ const gpa = p.recentGpa();
+ const text = recentText(o, gpa) catch return;
+ defer if (text) |t| gpa.free(t);
+ p.recent.merge(gpa, text orelse "");
+ db.seen = size;
+}
+
+/// Recent written when it changed, at the end of a step and at the end
+/// (`wait`: the lock is waited for there, never in a step).
+pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void {
+ if (!p.recent.dirty) return;
+ if (db.kind != .off) written: {
+ if (!db.loaded) return;
+ const o = db.opened(p) orelse break :written;
+ // Another session's lock: the next step tries again.
+ if (!lock(o, wait)) return;
+ defer unlock(o);
+ db.mergeRecent(p, o);
+ p.recent.changedFirst();
+ const gpa = p.recentGpa();
+ var text: std.Io.Writer.Allocating = .init(gpa);
+ defer text.deinit();
+ const entries = p.recent.entries.items;
+ for (entries[0..@min(entries.len, recent.cap)]) |e| recent.List.writeEntry(e, &text.writer) catch break :written;
+ // The disk refused it: kept for no one, as before.
+ putRecent(o, text.written()) catch break :written;
+ db.seen = committed(o) catch 0;
+ }
+ for (p.recent.entries.items) |*e| e.changed = false;
+ p.recent.dirty = false;
+}
+
+// ---- tests ----
+
+const testing = std.testing;
+
+/// A window keeping its state in `dir` (never the user's DumpDir).
+fn testWindow(dir: []const u8) !*Pardes {
+ var config: [4200]u8 = undefined;
+ return Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 48, .state = .disk, .startup_config = try std.fmt.bufPrint(&config, "DumpDir {s}\n", .{dir}) });
+}
+
+fn testDir(tmp: *std.testing.TmpDir, buf: *[std.fs.max_path_bytes]u8) ![]const u8 {
+ return buf[0..try tmp.dir.realPath(testing.io, buf)];
+}
+
+fn fileSize(dir: []const u8) u64 {
+ var buf: [4200]u8 = undefined;
+ const path = std.fmt.bufPrint(&buf, "{s}/" ++ file_name, .{dir}) catch return 0;
+ const st = std.Io.Dir.cwd().statFile(testing.io, path, .{}) catch return 0;
+ return st.size;
+}
+
+test "state file: two windows share Recent, each reading the other's commits on use; a busy lock delays, never loses" {
+ var tmp = testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const dir = try testDir(&tmp, &dir_buf);
+ const a = try testWindow(dir);
+ defer a.deinit();
+ const b = try testWindow(dir);
+ defer b.deinit();
+ a.recent.opened(a.recentGpa(), "/w/a.zig");
+ a.saveRecent();
+ b.recent.opened(b.recentGpa(), "/w/b.zig");
+ b.saveRecent();
+ a.recent.closed(a.recentGpa(), "/w/a.zig", 7, 3, 0, 0);
+ a.saveRecent();
+ // Each has both, in the order they were written: a closed last, then b.
+ a.db.readRecent(a);
+ b.db.readRecent(b);
+ for ([_]*Pardes{ a, b }) |w| {
+ try testing.expectEqual(@as(usize, 2), w.recent.entries.items.len);
+ try testing.expectEqualStrings("/w/a.zig", w.recent.entries.items[0].path);
+ try testing.expectEqualStrings("/w/b.zig", w.recent.entries.items[1].path);
+ try testing.expectEqual(@as(u32, 7), w.recent.get("/w/a.zig").?.line);
+ }
+ // Another window holding the file's lock: b's change waits in memory...
+ var path_buf: [4200]u8 = undefined;
+ const lock_file = try std.Io.Dir.cwd().openFile(testing.io, try std.fmt.bufPrint(&path_buf, "{s}/" ++ file_name, .{dir}), .{ .mode = .read_write });
+ try lock_file.lock(testing.io, .exclusive);
+ b.recent.opened(b.recentGpa(), "/w/c.zig");
+ const size = fileSize(dir);
+ b.saveRecent();
+ try testing.expectEqual(size, fileSize(dir));
+ try testing.expect(b.recent.dirty);
+ // ...and is written at the next step after it is let go.
+ lock_file.close(testing.io);
+ b.saveRecent();
+ try testing.expect(fileSize(dir) > size);
+ a.db.readRecent(a);
+ try testing.expectEqualStrings("/w/c.zig", a.recent.entries.items[0].path);
+ // A window started now reads the same list.
+ const c = try testWindow(dir);
+ defer c.deinit();
+ try testing.expectEqual(@as(usize, 3), c.recent.entries.items.len);
+ for (a.recent.entries.items, c.recent.entries.items) |x, y| try testing.expectEqualStrings(x.path, y.path);
+}
+
+test "state file: Recent holds the newest 200, the newest first, and a window started on it reads them so" {
+ var tmp = testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const dir = try testDir(&tmp, &dir_buf);
+ const a = try testWindow(dir);
+ defer a.deinit();
+ var name: [32]u8 = undefined;
+ for (0..recent.cap + 100) |i| {
+ const file = std.fmt.bufPrint(&name, "/w/file{d}.zig", .{i}) catch unreachable;
+ a.recent.opened(a.recentGpa(), file);
+ a.recent.closed(a.recentGpa(), file, 1, 1, 0, 0);
+ a.saveRecent();
+ }
+ try testing.expectEqual(@as(usize, recent.cap), a.recent.entries.items.len);
+ const text = (try recentText(a.db.open.?, testing.allocator)).?;
+ defer testing.allocator.free(text);
+ try testing.expectEqual(@as(usize, recent.cap), std.mem.count(u8, text, "\n"));
+ try testing.expect(std.mem.startsWith(u8, text, "1 1 0 0 /w/file299.zig\n"));
+ const b = try testWindow(dir);
+ defer b.deinit();
+ try testing.expectEqual(@as(usize, recent.cap), b.recent.entries.items.len);
+ try testing.expectEqualStrings("/w/file299.zig", b.recent.entries.items[0].path);
+ try testing.expectEqualStrings("/w/file100.zig", b.recent.entries.items[recent.cap - 1].path);
+}
+
+test "state file: one that is no database of pardes's never stops pardes: it is kept aside, and a new one started" {
+ var tmp = testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const dir = try testDir(&tmp, &dir_buf);
+ try tmp.dir.writeFile(testing.io, .{ .sub_path = file_name, .data = "\x00\xffnot a database at all\n7 3 0 0 /w/ok.zig\n" });
+ const a = try testWindow(dir);
+ defer a.deinit();
+ try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len);
+ a.recent.opened(a.recentGpa(), "/w/a.zig");
+ a.saveRecent();
+ const kept = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20));
+ defer testing.allocator.free(kept);
+ try testing.expect(std.mem.startsWith(u8, kept, "\x00\xffnot a database"));
+ const b = try testWindow(dir);
+ defer b.deinit();
+ try testing.expectEqual(@as(usize, 1), b.recent.entries.items.len);
+ try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path);
+}