diff options
| author | Gabriel Schneider <[email protected]> | 2026-11-05 14:42:45 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-11-05 19:16:37 -0300 |
| commit | 15a1b94eaa73747afaa299262a2d6e4fd6af42fd (patch) | |
| tree | 46bff4f35b642521eead62bc3b5657902e5fee0b /src/Db.zig | |
| parent | 4e096f6dc6db043caa630514fa75bd18438cb15b (diff) | |
| download | pardes-15a1b94eaa73747afaa299262a2d6e4fd6af42fd.tar.gz pardes-15a1b94eaa73747afaa299262a2d6e4fd6af42fd.zip | |
dumps: a Dump is a labelled entry of the state file (src/Db.zig, architecture §14.3), one fsynced transaction: `dumps/<label>` holds the session as a database of its own, which the walk writes by field name into xitdb's maps (walk.Put: a struct a map keyed by its fields' names, a union a map of its one field, records a list, plain numbers one bytes value tagged with their kind and width, an enum its name, a number a uint or an int, an optional an absent key, a type with dumpWrite its bytes, a .blob text of 4 KiB and up its BLAKE3 hash) and another build reads by name (walk.Get: defaults, notes, a pane it cannot read left out, as the byte loader did), with the hashes of the texts it keeps once under `blobs/`, so an unchanged text is written once across dumps; labels are `pardes-<date>-<time>` (`-2` and on in one second), `last` names the newest, the newest 32 are kept and the texts no kept one names are dropped in the same transaction, and at a session's end the file is compacted into `pardes.xitdb.new` and renamed over once it is past twice what it holds plus 64 MiB (another session notices the new file by its inode); a dump travels as an image, a database in memory holding it alone, deterministic (std.Io.failing's zero ids, so the replay's fixpoint holds): what Dump hands its host, what `Dump export <path>` and $PARDES_DUMP write whole (the snapshot goldens' `Restore <path>` unchanged), what a Restore of a label is read into, what a handover carries (`handover-<seq>`), and what `-l` and `Restore <path>` read; Restore takes a label or a path (bare: this session's last dump, else the newest), a label the file keeps none of refused in words (`Restore: no dump labelled X`, asked of the authority as the closed panes are); `/dumps` lists them, `<label> <date> <size> <panes>`, the newest first; Upgrade commits `upgrade-<pid>` before the exec, its record naming the state file and the label (a record without one, an older build's, names its `.dump` file, read as before), and the label stays; the byte dump stays for the join, the closed panes and the old `.dump` files (dump.byteDump), and dump.outPath, uniqueIn and upgradePath go for dump.label; measured (Debug, a session with a 3.2 MB file and three scratch panes, 20 dumps): the image on the loop 64-66 ms (59 ms of it BLAKE3 of the file, 0.65 ms in ReleaseFast) where the byte dump was 0.7 ms, the commit 9-16 ms with its fsyncs where the file was written in 2, a Restore 1.8 ms to read and 11.4 to load, and the file 3.55 MB after the first dump and 379 KB a dump after it with the big text unchanged (3.58 MB a dump with it edited each time, as twenty `.dump` files would be); tests: dump.zig's 'two dumps in one second get names of their own' and 'a dump makes every directory missing on the way to its DumpDir' become Db.zig's 'two dumps with one label get labels of their own' and 'a Dump makes every directory missing on the way to its DumpDir, and the file there', 'a session read by name, field by field as another build reads it, is the same session' becomes 'a session's byte dump read by name...' beside the new 'a session's image, read by name as every load reads it, is the same session, and dumps to the same bytes again', two replica tests compare the join's stream with dump.byteDump, the Handover tests read the carried image, upgrade's record test carries a label, the root listing tests and the monkey list `dumps`, the behaviour suite gains five root-ctl tests (a Dump and Restore round trip through 9P with `/dumps` and the tag's word, a label kept none of refused in words, the 33rd dump dropping the first, an unchanged text written once across two dumps by the file's size, a killed Dump leaving the one before it whole), the test host performs a Dump's write and a Restore as a host does, the monkey lets go of its log follower at a Restore, and daemon.py's upgrade case finds the kept dump in `/dumps`
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge
Diffstat (limited to 'src/Db.zig')
| -rw-r--r-- | src/Db.zig | 532 |
1 files changed, 525 insertions, 7 deletions
@@ -103,7 +103,9 @@ fn opened(db: *Db, p: *Pardes) ?*Open { var buf: [4200]u8 = undefined; const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null; if (db.open) |o| { - if (std.mem.eql(u8, o.where.disk.path, path)) return o; + // The same file still: another session's compaction renames + // a new one over it, which this one then opens. + if (std.mem.eql(u8, o.where.disk.path, path) and !replaced(o)) return o; // The DumpDir moved: its own file, read afresh. db.deinit(); db.seen = 0; @@ -168,6 +170,17 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { } } +/// Whether the file at the open one's path is another now (a compaction's). +fn replaced(o: *Open) bool { + const d = o.where.disk; + pardes.turn.yield(); + defer pardes.turn.back(); + const io = ioOf(); + const open_st = d.file.stat(io) catch return false; + const path_st = std.Io.Dir.cwd().statFile(io, d.path, .{}) catch return false; + return open_st.inode != path_st.inode; +} + fn committed(o: *Open) !u64 { pardes.turn.yield(); defer pardes.turn.back(); @@ -192,6 +205,21 @@ fn lock(o: *Open, wait: bool) bool { return file.tryLock(ioOf(), .exclusive) catch false; } +/// The database, its file locked for a transaction (`wait`: waited for). A +/// file a compaction replaced while this waited is let go, and the new one +/// opened and locked. +fn locked(db: *Db, p: *Pardes, wait: bool) error{ NoStateFile, Busy }!*Open { + for (0..2) |_| { + const o = db.opened(p) orelse return error.NoStateFile; + if (!lock(o, wait)) return error.Busy; + if (o.where == .memory or !replaced(o)) return o; + unlock(o); + db.deinit(); + db.seen = 0; + } + return error.Busy; +} + fn unlock(o: *Open) void { switch (o.where) { .disk => |d| d.file.unlock(ioOf()), @@ -256,9 +284,11 @@ pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { if (!p.recent.dirty) return; if (db.kind != .off) written: { if (!db.loaded) return; - const o = db.opened(p) orelse break :written; - // Another session's lock: the next step tries again. - if (!lock(o, wait)) return; + const o = db.locked(p, wait) catch |err| switch (err) { + // Another session's lock: the next step tries again. + error.Busy => return, + error.NoStateFile => break :written, + }; defer unlock(o); db.mergeRecent(p, o); p.recent.changedFirst(); @@ -275,6 +305,463 @@ pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { p.recent.dirty = false; } +// ---- the dumps ---- +// +// Each Dump is a labelled entry of the state file's `dumps`, written in one +// transaction, fsynced (a dump is worth the wait; Recent's commits are not): +// a full disk or a kill in the middle leaves the dump before it whole, as +// xitdb writes its header last. +// +// dumps/<label> seq, date (seconds, UTC), panes, size, the session (a +// database of its own: walk.Put's maps, its bytes one +// value) and the hashes of the texts it keeps beside it +// blobs/<hash> each such text once, by its BLAKE3 hash: an unchanged +// text is written once, however many dumps keep it +// last the newest dump's label +// seq the next dump's number +// +// A Dump is labelled `pardes-<date>-<time>` (UTC; `-2` and on for another +// in the same second), an Upgrade's `upgrade-<pid>`. The newest `kept` are +// kept: one more drops the oldest, and the texts no kept dump names, in +// the same transaction. A dump travels as an image, a database in memory +// holding it alone in the same shape (`image`): what a Dump hands its host, +// what a Restore of a label is read into (`imageOf`), what `Dump export` +// and a handover write as a file, and what `-l` and `Restore <path>` read. + +/// How many dumps the state file keeps, the newest. +pub const kept = 32; +/// The text of a dump's blob is its BLAKE3 hash's key. +fn blobKey(hash: *const [32]u8) u256 { + return std.mem.readInt(u256, hash, .big); +} + +const session_tag = "db".*; + +/// A dump's image: the one dump `label` (its session's bytes, the texts it +/// keeps beside it and its pane count) in a database in memory, `last` +/// naming it, `gpa`'s. Of the same session, the same bytes: its ids are +/// std.Io.failing's zeros, never random ones. +pub fn image(gpa: std.mem.Allocator, label: []const u8, session: []const u8, blobs: []const pardes.walk.Blob, panes: u64) ![]u8 { + var buf: std.Io.Writer.Allocating = .init(gpa); + errdefer buf.deinit(); + var db = try Memory.init(std.Io.failing, .{ .buffer = &buf, .hash_id = hash_id, .file_id = 0 }); + try db.runContext(struct { + label: []const u8, + session: []const u8, + blobs: []const pardes.walk.Blob, + panes: u64, + pub fn run(ctx: @This(), cursor: *Memory.Cursor(.read_write)) !void { + const root = try Memory.HashMap(.read_write).init(cursor.*); + try root.put(key("last"), .{ .bytes = ctx.label }); + const dumps = try Memory.HashMap(.read_write).init(try root.putCursor(key("dumps"))); + try dumps.putKey(key(ctx.label), .{ .bytes = ctx.label }); + const entry = try Memory.HashMap(.read_write).init(try dumps.putCursor(key(ctx.label))); + try entry.put(key("session"), .{ .bytes_object = .{ .value = ctx.session, .format_tag = session_tag } }); + try entry.put(key("panes"), .{ .uint = ctx.panes }); + const hashes = try Memory.ArrayList(.read_write).init(try entry.putCursor(key("blobs"))); + const texts = try Memory.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + for (ctx.blobs) |*b| { + try hashes.append(.{ .bytes = &b.hash }); + try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes }); + } + } + }{ .label = label, .session = session, .blobs = blobs, .panes = panes }); + return buf.toOwnedSlice(); +} + +/// Whether `bytes` are a database of xitdb's (an image, a state file): +/// what `-l` and `Restore <path>` tell from an old dump by. +pub fn isImage(bytes: []const u8) bool { + return std.mem.startsWith(u8, bytes, "xit"); +} + +/// A dump's image opened for a load: the session's database, and the +/// texts its blobs are. The image's bytes are the caller's while it is open. +pub const Read = struct { + gpa: std.mem.Allocator, + image_buf: std.Io.Writer.Allocating, + image: Memory, + session_buf: std.Io.Writer.Allocating, + session: Memory, + label: []u8, + panes: u64, + + /// The image `bytes` at its dump `label`, else its last one. + pub fn open(gpa: std.mem.Allocator, bytes: []const u8, label: ?[]const u8) !*Read { + if (!isImage(bytes)) return error.NotADump; + const r = try gpa.create(Read); + errdefer gpa.destroy(r); + // Read, never written: the bytes are borrowed, never freed here. + var borrowed: std.ArrayList(u8) = .fromOwnedSlice(@constCast(bytes)); + r.* = .{ .gpa = gpa, .image_buf = .fromArrayList(gpa, &borrowed), .image = undefined, .session_buf = undefined, .session = undefined, .label = &.{}, .panes = 0 }; + r.image = Memory.init(std.Io.failing, .{ .buffer = &r.image_buf }) catch return error.NotADump; + const top = Memory.HashMap(.read_only).init(r.image.rootCursor().readOnly()) catch return error.NotADump; + r.label = if (label) |l| try gpa.dupe(u8, l) else last: { + const c = (top.getCursor(key("last")) catch return error.NotADump) orelse return error.NoDump; + break :last @constCast(c.readBytesAlloc(gpa, 4096) catch return error.NotADump); + }; + errdefer gpa.free(r.label); + const entry = try entryOf(Memory, top, r.label) orelse return error.NoDump; + r.panes = if (entry.getCursor(key("panes")) catch null) |c| c.readUint() catch 0 else 0; + const at = (entry.getCursor(key("session")) catch return error.NotADump) orelse return error.NotADump; + const session: []u8 = @constCast(at.readBytesAlloc(gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.NotADump); + var owned: std.ArrayList(u8) = .fromOwnedSlice(session); + r.session_buf = .fromArrayList(gpa, &owned); + errdefer r.session_buf.deinit(); + r.session = Memory.init(std.Io.failing, .{ .buffer = &r.session_buf }) catch return error.NotADump; + return r; + } + + pub fn close(r: *Read) void { + r.session_buf.deinit(); + r.gpa.free(r.label); + r.gpa.destroy(r); + } + + /// The session's roots. + pub fn root(r: *Read) Memory.Cursor(.read_only) { + return r.session.rootCursor().readOnly(); + } + + /// The texts the session's blobs are, by hash. + pub fn blobs(r: *Read) !Memory.HashMap(.read_only) { + const root_map = try Memory.HashMap(.read_only).init(r.image.rootCursor().readOnly()); + return Memory.HashMap(.read_only).init((try root_map.getCursor(key("blobs"))) orelse return error.NotADump); + } +}; + +/// The entry of dump `label` in the root map `root`; null when there is none. +fn entryOf(comptime DB: type, root: DB.HashMap(.read_only), label: []const u8) !?DB.HashMap(.read_only) { + const dumps = (try root.getCursor(key("dumps"))) orelse return null; + const at = (try (try DB.HashMap(.read_only).init(dumps)).getCursor(key(label))) orelse return null; + return try DB.HashMap(.read_only).init(at); +} + +/// The dump `label` of `o` as an image, `gpa`'s; null when it keeps none +/// of that label. +fn imageOf(o: *Open, gpa: std.mem.Allocator, label: []const u8) !?[]u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); + const entry = (try entryOf(DB, root, label)) orelse return null; + const session = try (try entry.getCursor(key("session")) orelse return error.NotADump).readBytesAlloc(gpa, null); + defer gpa.free(session); + const panes = if (try entry.getCursor(key("panes"))) |c| try c.readUint() else 0; + var blobs: std.ArrayList(pardes.walk.Blob) = .empty; + defer { + for (blobs.items) |b| gpa.free(b.bytes); + blobs.deinit(gpa); + } + const texts = try DB.HashMap(.read_only).init((try root.getCursor(key("blobs"))) orelse return error.NotADump); + if (try entry.getCursor(key("blobs"))) |list_at| { + var it = try (try DB.ArrayList(.read_only).init(list_at)).iterator(); + while (try it.next()) |h| { + var hash: [32]u8 = undefined; + if ((try h.readBytes(&hash)).len != 32) return error.NotADump; + const text = try (try texts.getCursor(blobKey(&hash)) orelse return error.NotADump).readBytesAlloc(gpa, null); + errdefer gpa.free(text); + try blobs.append(gpa, .{ .hash = hash, .bytes = text }); + } + } + return try image(gpa, label, session, blobs.items, panes); + }, + } +} + +/// The dump `label` (bare: the newest) of this session's state file, as +/// an image, `gpa`'s; null when it keeps none of that label. +pub fn getDump(db: *Db, p: *Pardes, gpa: std.mem.Allocator, label: ?[]const u8) !?[]u8 { + const o = db.opened(p) orelse return error.NoStateFile; + var last_buf: [4096]u8 = undefined; + const name = label orelse (try lastLabel(o, &last_buf)) orelse return null; + return imageOf(o, gpa, name); +} + +/// The dump `label` of the state file at `path` as an image, `gpa`'s: an +/// upgraded process's (its record names both), read before it has a core. +pub fn readDumpAt(gpa: std.mem.Allocator, path: []const u8, label: []const u8) ![]u8 { + var db: Db = .{ .kind = .disk, .open = try openDisk(gpa, path) }; + defer db.deinit(); + return (try imageOf(db.open.?, gpa, label)) orelse error.NoDump; +} + +/// The newest dump's label, in `buf`; null when there is none. +fn lastLabel(o: *Open, buf: []u8) !?[]const u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); + const c = (try root.getCursor(key("last"))) orelse return null; + return try c.readBytes(buf); + }, + } +} + +/// The label of the state file's dump `label` (bare: its newest), in `buf`; +/// null when it keeps none of that label. A step asks it of the authority +/// (tape.authority): a replica keeps no state file. +pub fn labelled(db: *Db, p: *Pardes, label: ?[]const u8, buf: []u8) ?[]const u8 { + const o = db.opened(p) orelse return null; + const name = if (label) |l| (std.fmt.bufPrint(buf, "{s}", .{l}) catch return null) else ((lastLabel(o, buf) catch return null) orelse return null); + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()) catch return null; + return if ((entryOf(DB, root, name) catch return null) != null) name else null; + }, + } +} + +/// The dump in the image `bytes` put in this session's state file under a +/// label of its own (`label`, else with `-2` and on), in one fsynced +/// transaction that also drops the oldest past `kept` and the texts no kept +/// dump names; waits for another session's transaction. The label, `buf`'s. +pub fn putDump(db: *Db, p: *Pardes, bytes: []const u8, label: []const u8, buf: []u8) ![]const u8 { + const r = try Read.open(p.recentGpa(), bytes, null); + defer r.close(); + const o = try db.locked(p, true); + defer unlock(o); + const before = committed(o) catch 0; + const name = try putLocked(o, r, label, buf); + if (db.seen == before) db.seen = committed(o) catch 0; + return name; +} + +fn putLocked(o: *Open, r: *Read, label: []const u8, buf: []u8) ![]const u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + var ts: std.c.timespec = undefined; + _ = std.c.clock_gettime(.REALTIME, &ts); + switch (o.where) { + inline else => |*w, kind| { + const DB = @TypeOf(w.db); + if (kind == .disk) w.db.core.file.fsync = true; + defer if (kind == .disk) { + w.db.core.file.fsync = false; + }; + var name: []const u8 = label; + try w.db.runContext(struct { + r: *Read, + label: []const u8, + buf: []u8, + name: *[]const u8, + date: u64, + gpa: std.mem.Allocator, + pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { + const root = try DB.HashMap(.read_write).init(cursor.*); + const seq = if (try root.getCursor(key("seq"))) |c| try c.readUint() else 0; + try root.put(key("seq"), .{ .uint = seq + 1 }); + const dumps = try DB.HashMap(.read_write).init(try root.putCursor(key("dumps"))); + // A label of its own: a second dump in a second is `-2`. + var n: usize = 1; + var label_now: []const u8 = ctx.label; + while (try dumps.getCursor(key(label_now)) != null) { + n += 1; + label_now = try std.fmt.bufPrint(ctx.buf, "{s}-{d}", .{ ctx.label, n }); + } + if (n == 1) label_now = try std.fmt.bufPrint(ctx.buf, "{s}", .{ctx.label}); + ctx.name.* = label_now; + // The image's one entry, copied: its session, its texts + // the state file does not have yet, the numbers. + const image_root = try Memory.HashMap(.read_only).init(ctx.r.image.rootCursor().readOnly()); + const from = (try entryOf(Memory, image_root, ctx.r.label)) orelse return error.NoDump; + const image_texts = try ctx.r.blobs(); + try dumps.putKey(key(label_now), .{ .bytes = label_now }); + const entry = try DB.HashMap(.read_write).init(try dumps.putCursor(key(label_now))); + try entry.put(key("seq"), .{ .uint = seq }); + try entry.put(key("date"), .{ .uint = ctx.date }); + try entry.put(key("panes"), .{ .uint = ctx.r.panes }); + var session_at = (try from.getCursor(key("session"))) orelse return error.NoDump; + var size = try session_at.count(); + try copyBytes(DB, &session_at, try entry.putCursor(key("session")), session_tag); + const texts = try DB.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const hashes = try DB.ArrayList(.read_write).init(try entry.putCursor(key("blobs"))); + if (try from.getCursor(key("blobs"))) |list_at| { + var it = try (try Memory.ArrayList(.read_only).init(list_at)).iterator(); + while (try it.next()) |h| { + var hash: [32]u8 = undefined; + if ((try h.readBytes(&hash)).len != 32) return error.NotADump; + try hashes.append(.{ .bytes = &hash }); + var text_at = (try image_texts.getCursor(blobKey(&hash))) orelse return error.NotADump; + size += try text_at.count(); + if (try texts.getCursor(blobKey(&hash)) == null) try copyBytes(DB, &text_at, try texts.putCursor(blobKey(&hash)), null); + } + } + try entry.put(key("size"), .{ .uint = size }); + try root.put(key("last"), .{ .bytes = label_now }); + try trim(DB, ctx.gpa, dumps, texts); + } + }{ .r = r, .label = label, .buf = buf, .name = &name, .date = @intCast(@max(0, ts.sec)), .gpa = r.gpa }); + return name; + }, + } +} + +/// The bytes at `from` (an image's, read while it is open) written at `to`, +/// streamed. +fn copyBytes(comptime DB: type, from: *Memory.Cursor(.read_only), to: DB.Cursor(.read_write), tag: ?[2]u8) !void { + var at = to; + var in_buf: [64 << 10]u8 = undefined; + var reader = try from.reader(&in_buf); + var out_buf: [64 << 10]u8 = undefined; + var writer = try at.writer(&out_buf); + _ = try reader.interface.streamRemaining(&writer.interface); + writer.format_tag = tag; + try writer.finish(); +} + +/// The newest `kept` dumps kept, the older dropped, and every text no kept +/// dump names (one replaced or dropped) dropped with them. +fn trim(comptime DB: type, gpa: std.mem.Allocator, dumps: DB.HashMap(.read_write), texts: DB.HashMap(.read_write)) !void { + const Dumped = struct { seq: u64, hash: u256 }; + var all: std.ArrayList(Dumped) = .empty; + defer all.deinit(gpa); + var it = try dumps.iterator(); + while (try it.next()) |kv_at| { + const kv = try kv_at.readKeyValuePair(); + const entry = try DB.HashMap(.read_only).init(kv.value_cursor); + const seq = if (try entry.getCursor(key("seq"))) |c| try c.readUint() else 0; + try all.append(gpa, .{ .seq = seq, .hash = kv.hash }); + } + std.mem.sort(Dumped, all.items, {}, struct { + fn newer(_: void, a: Dumped, b: Dumped) bool { + return a.seq > b.seq; + } + }.newer); + if (all.items.len > kept) for (all.items[kept..]) |d| { + _ = try dumps.remove(d.hash); + }; + // The texts the kept ones name; every other one goes. + var named: std.AutoHashMapUnmanaged(u256, void) = .empty; + defer named.deinit(gpa); + for (all.items[0..@min(all.items.len, kept)]) |d| { + const entry = try DB.HashMap(.read_only).init((try dumps.getCursor(d.hash)) orelse continue); + const list_at = (try entry.getCursor(key("blobs"))) orelse continue; + var hashes = try (try DB.ArrayList(.read_only).init(list_at)).iterator(); + while (try hashes.next()) |h| { + var hash: [32]u8 = undefined; + if ((try h.readBytes(&hash)).len == 32) try named.put(gpa, blobKey(&hash), {}); + } + } + var gone: std.ArrayList(u256) = .empty; + defer gone.deinit(gpa); + var texts_it = try texts.iterator(); + while (try texts_it.next()) |kv_at| { + const kv = try kv_at.readKeyValuePair(); + if (!named.contains(kv.hash)) try gone.append(gpa, kv.hash); + } + for (gone.items) |h| _ = try texts.remove(h); +} + +/// The dumps the state file keeps, a line each, the newest first: its +/// label, its date (UTC), its size in bytes and its pane count. +pub fn listDumps(db: *Db, p: *Pardes, w: *std.Io.Writer) !void { + const o = db.opened(p) orelse return; + pardes.turn.yield(); + defer pardes.turn.back(); + const gpa = p.recentGpa(); + const Row = struct { seq: u64, date: u64, size: u64, panes: u64, label: []u8 }; + var rows: std.ArrayList(Row) = .empty; + defer { + for (rows.items) |r| gpa.free(r.label); + rows.deinit(gpa); + } + switch (o.where) { + inline else => |*d| { + const DB = @TypeOf(d.db); + const root = try DB.HashMap(.read_only).init(d.db.rootCursor().readOnly()); + const dumps_at = (try root.getCursor(key("dumps"))) orelse return; + var it = try (try DB.HashMap(.read_only).init(dumps_at)).iterator(); + while (try it.next()) |kv_at| { + const kv = try kv_at.readKeyValuePair(); + const entry = try DB.HashMap(.read_only).init(kv.value_cursor); + var row: Row = .{ .seq = 0, .date = 0, .size = 0, .panes = 0, .label = @constCast(try kv.key_cursor.readBytesAlloc(gpa, 4096)) }; + inline for (.{ "seq", "date", "size", "panes" }) |field| if (try entry.getCursor(key(field))) |c| { + @field(row, field) = c.readUint() catch 0; + }; + rows.append(gpa, row) catch |err| { + gpa.free(row.label); + return err; + }; + } + }, + } + std.mem.sort(Row, rows.items, {}, struct { + fn newer(_: void, a: Row, b: Row) bool { + return a.seq > b.seq; + } + }.newer); + for (rows.items) |r| { + const es: std.time.epoch.EpochSeconds = .{ .secs = r.date }; + const yd = es.getEpochDay().calculateYearDay(); + const md = yd.calculateMonthDay(); + const ds = es.getDaySeconds(); + try w.print("{s} {d:0>4}-{d:0>2}-{d:0>2}T{d:0>2}:{d:0>2}:{d:0>2}Z {d} {d}\n", .{ + r.label, yd.year, md.month.numeric(), @as(u8, md.day_index) + 1, + ds.getHoursIntoDay(), ds.getMinutesIntoHour(), ds.getSecondsIntoMinute(), r.size, + r.panes, + }); + } +} + +/// At a session's end: the state file compacted (into `<path>.new`, +/// renamed over it) once it is past twice what its dumps and Recent hold, +/// plus 64 MiB. Under the file's lock: no session commits to the old file +/// meanwhile, and one that opened it reopens the new one (`opened`). +pub fn compact(db: *Db, p: *Pardes) void { + if (db.kind != .disk or db.open == null) return; + const size = db.open.?.where.disk.file.length(ioOf()) catch return; + if (size < 64 << 20) return; + const o = db.locked(p, true) catch return; + defer unlock(o); + const d = &o.where.disk; + const io = ioOf(); + const live = liveSize(&d.db) catch return; + if (size <= 2 * live + (64 << 20)) return; + var buf: [4300]u8 = undefined; + const new = std.fmt.bufPrint(&buf, "{s}.new", .{d.path}) catch return; + const cwd = std.Io.Dir.cwd(); + const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return; + var new_buf: std.Io.Writer.Allocating = .init(p.recentGpa()); + defer new_buf.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(p.recentGpa()); + defer offsets.deinit(); + _ = d.db.compact(.buffered_file, .{ .file = file, .buffer = &new_buf, .hash_id = hash_id }, &offsets) catch { + file.close(io); + cwd.deleteFile(io, new) catch {}; + return; + }; + file.close(io); + cwd.rename(new, cwd, d.path, io) catch cwd.deleteFile(io, new) catch {}; +} + +/// What the state file's latest moment holds: Recent, each dump's +/// session, each text once. +fn liveSize(db: *Disk) !u64 { + const root = try Disk.HashMap(.read_only).init(db.rootCursor().readOnly()); + var live: u64 = 0; + if (try root.getCursor(key("recent"))) |c| live += try c.count(); + if (try root.getCursor(key("dumps"))) |at| { + var it = try (try Disk.HashMap(.read_only).init(at)).iterator(); + while (try it.next()) |kv_at| { + const entry = try Disk.HashMap(.read_only).init((try kv_at.readKeyValuePair()).value_cursor); + if (try entry.getCursor(key("session"))) |c| live += try c.count(); + } + } + if (try root.getCursor(key("blobs"))) |at| { + var it = try (try Disk.HashMap(.read_only).init(at)).iterator(); + while (try it.next()) |kv_at| live += try (try kv_at.readKeyValuePair()).value_cursor.count(); + } + return live; +} + // ---- tests ---- const testing = std.testing; @@ -379,11 +866,42 @@ test "state file: one that is no database of pardes's never stops pardes: it is try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len); a.recent.opened(a.recentGpa(), "/w/a.zig"); a.saveRecent(); - const kept = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20)); - defer testing.allocator.free(kept); - try testing.expect(std.mem.startsWith(u8, kept, "\x00\xffnot a database")); + const aside = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20)); + defer testing.allocator.free(aside); + try testing.expect(std.mem.startsWith(u8, aside, "\x00\xffnot a database")); const b = try testWindow(dir); defer b.deinit(); try testing.expectEqual(@as(usize, 1), b.recent.entries.items.len); try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path); } + +test "state file: two dumps with one label get labels of their own" { + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .state = .memory }); + defer p.deinit(); + const bytes = try pardes.dump.dumpBytes(p); + defer testing.allocator.free(bytes); + var buf: [64]u8 = undefined; + try testing.expectEqualStrings("pardes-x", try p.db.putDump(p, bytes, "pardes-x", &buf)); + try testing.expectEqualStrings("pardes-x-2", try p.db.putDump(p, bytes, "pardes-x", &buf)); + try testing.expectEqualStrings("pardes-x-3", try p.db.putDump(p, bytes, "pardes-x", &buf)); + var label_buf: [64]u8 = undefined; + try testing.expectEqualStrings("pardes-x-3", p.db.labelled(p, null, &label_buf).?); +} + +test "state file: a Dump makes every directory missing on the way to its DumpDir, and the file there" { + if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const base = try testDir(&tmp, &dir_buf); + var setting: [4200]u8 = undefined; + const dir = try std.fmt.bufPrint(&setting, "{s}/fresh/.local/share/pardes", .{base}); + const p = try testWindow(dir); + defer p.deinit(); + try pardes.dump.dumpState(p); + pardes.host_io.writeDump(p, p.dump_out.?); + try testing.expect(std.mem.startsWith(u8, p.last_dump.?, "pardes-")); + try testing.expect(fileSize(dir) > 0); + var label_buf: [64]u8 = undefined; + try testing.expect(p.db.labelled(p, p.last_dump.?, &label_buf) != null); +} |
