summaryrefslogtreecommitdiff
path: root/src/Db.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/Db.zig')
-rw-r--r--src/Db.zig532
1 files changed, 525 insertions, 7 deletions
diff --git a/src/Db.zig b/src/Db.zig
index e496945f..a91a5054 100644
--- a/src/Db.zig
+++ b/src/Db.zig
@@ -103,7 +103,9 @@ fn opened(db: *Db, p: *Pardes) ?*Open {
var buf: [4200]u8 = undefined;
const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null;
if (db.open) |o| {
- if (std.mem.eql(u8, o.where.disk.path, path)) return o;
+ // The same file still: another session's compaction renames
+ // a new one over it, which this one then opens.
+ if (std.mem.eql(u8, o.where.disk.path, path) and !replaced(o)) return o;
// The DumpDir moved: its own file, read afresh.
db.deinit();
db.seen = 0;
@@ -168,6 +170,17 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open {
}
}
+/// Whether the file at the open one's path is another now (a compaction's).
+fn replaced(o: *Open) bool {
+ const d = o.where.disk;
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ const io = ioOf();
+ const open_st = d.file.stat(io) catch return false;
+ const path_st = std.Io.Dir.cwd().statFile(io, d.path, .{}) catch return false;
+ return open_st.inode != path_st.inode;
+}
+
fn committed(o: *Open) !u64 {
pardes.turn.yield();
defer pardes.turn.back();
@@ -192,6 +205,21 @@ fn lock(o: *Open, wait: bool) bool {
return file.tryLock(ioOf(), .exclusive) catch false;
}
+/// The database, its file locked for a transaction (`wait`: waited for). A
+/// file a compaction replaced while this waited is let go, and the new one
+/// opened and locked.
+fn locked(db: *Db, p: *Pardes, wait: bool) error{ NoStateFile, Busy }!*Open {
+ for (0..2) |_| {
+ const o = db.opened(p) orelse return error.NoStateFile;
+ if (!lock(o, wait)) return error.Busy;
+ if (o.where == .memory or !replaced(o)) return o;
+ unlock(o);
+ db.deinit();
+ db.seen = 0;
+ }
+ return error.Busy;
+}
+
fn unlock(o: *Open) void {
switch (o.where) {
.disk => |d| d.file.unlock(ioOf()),
@@ -256,9 +284,11 @@ pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void {
if (!p.recent.dirty) return;
if (db.kind != .off) written: {
if (!db.loaded) return;
- const o = db.opened(p) orelse break :written;
- // Another session's lock: the next step tries again.
- if (!lock(o, wait)) return;
+ const o = db.locked(p, wait) catch |err| switch (err) {
+ // Another session's lock: the next step tries again.
+ error.Busy => return,
+ error.NoStateFile => break :written,
+ };
defer unlock(o);
db.mergeRecent(p, o);
p.recent.changedFirst();
@@ -275,6 +305,463 @@ pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void {
p.recent.dirty = false;
}
+// ---- the dumps ----
+//
+// Each Dump is a labelled entry of the state file's `dumps`, written in one
+// transaction, fsynced (a dump is worth the wait; Recent's commits are not):
+// a full disk or a kill in the middle leaves the dump before it whole, as
+// xitdb writes its header last.
+//
+// dumps/<label> seq, date (seconds, UTC), panes, size, the session (a
+// database of its own: walk.Put's maps, its bytes one
+// value) and the hashes of the texts it keeps beside it
+// blobs/<hash> each such text once, by its BLAKE3 hash: an unchanged
+// text is written once, however many dumps keep it
+// last the newest dump's label
+// seq the next dump's number
+//
+// A Dump is labelled `pardes-<date>-<time>` (UTC; `-2` and on for another
+// in the same second), an Upgrade's `upgrade-<pid>`. The newest `kept` are
+// kept: one more drops the oldest, and the texts no kept dump names, in
+// the same transaction. A dump travels as an image, a database in memory
+// holding it alone in the same shape (`image`): what a Dump hands its host,
+// what a Restore of a label is read into (`imageOf`), what `Dump export`
+// and a handover write as a file, and what `-l` and `Restore <path>` read.
+
+/// How many dumps the state file keeps, the newest.
+pub const kept = 32;
+/// The text of a dump's blob is its BLAKE3 hash's key.
+fn blobKey(hash: *const [32]u8) u256 {
+ return std.mem.readInt(u256, hash, .big);
+}
+
+const session_tag = "db".*;
+
+/// A dump's image: the one dump `label` (its session's bytes, the texts it
+/// keeps beside it and its pane count) in a database in memory, `last`
+/// naming it, `gpa`'s. Of the same session, the same bytes: its ids are
+/// std.Io.failing's zeros, never random ones.
+pub fn image(gpa: std.mem.Allocator, label: []const u8, session: []const u8, blobs: []const pardes.walk.Blob, panes: u64) ![]u8 {
+ var buf: std.Io.Writer.Allocating = .init(gpa);
+ errdefer buf.deinit();
+ var db = try Memory.init(std.Io.failing, .{ .buffer = &buf, .hash_id = hash_id, .file_id = 0 });
+ try db.runContext(struct {
+ label: []const u8,
+ session: []const u8,
+ blobs: []const pardes.walk.Blob,
+ panes: u64,
+ pub fn run(ctx: @This(), cursor: *Memory.Cursor(.read_write)) !void {
+ const root = try Memory.HashMap(.read_write).init(cursor.*);
+ try root.put(key("last"), .{ .bytes = ctx.label });
+ const dumps = try Memory.HashMap(.read_write).init(try root.putCursor(key("dumps")));
+ try dumps.putKey(key(ctx.label), .{ .bytes = ctx.label });
+ const entry = try Memory.HashMap(.read_write).init(try dumps.putCursor(key(ctx.label)));
+ try entry.put(key("session"), .{ .bytes_object = .{ .value = ctx.session, .format_tag = session_tag } });
+ try entry.put(key("panes"), .{ .uint = ctx.panes });
+ const hashes = try Memory.ArrayList(.read_write).init(try entry.putCursor(key("blobs")));
+ const texts = try Memory.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ for (ctx.blobs) |*b| {
+ try hashes.append(.{ .bytes = &b.hash });
+ try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes });
+ }
+ }
+ }{ .label = label, .session = session, .blobs = blobs, .panes = panes });
+ return buf.toOwnedSlice();
+}
+
+/// Whether `bytes` are a database of xitdb's (an image, a state file):
+/// what `-l` and `Restore <path>` tell from an old dump by.
+pub fn isImage(bytes: []const u8) bool {
+ return std.mem.startsWith(u8, bytes, "xit");
+}
+
+/// A dump's image opened for a load: the session's database, and the
+/// texts its blobs are. The image's bytes are the caller's while it is open.
+pub const Read = struct {
+ gpa: std.mem.Allocator,
+ image_buf: std.Io.Writer.Allocating,
+ image: Memory,
+ session_buf: std.Io.Writer.Allocating,
+ session: Memory,
+ label: []u8,
+ panes: u64,
+
+ /// The image `bytes` at its dump `label`, else its last one.
+ pub fn open(gpa: std.mem.Allocator, bytes: []const u8, label: ?[]const u8) !*Read {
+ if (!isImage(bytes)) return error.NotADump;
+ const r = try gpa.create(Read);
+ errdefer gpa.destroy(r);
+ // Read, never written: the bytes are borrowed, never freed here.
+ var borrowed: std.ArrayList(u8) = .fromOwnedSlice(@constCast(bytes));
+ r.* = .{ .gpa = gpa, .image_buf = .fromArrayList(gpa, &borrowed), .image = undefined, .session_buf = undefined, .session = undefined, .label = &.{}, .panes = 0 };
+ r.image = Memory.init(std.Io.failing, .{ .buffer = &r.image_buf }) catch return error.NotADump;
+ const top = Memory.HashMap(.read_only).init(r.image.rootCursor().readOnly()) catch return error.NotADump;
+ r.label = if (label) |l| try gpa.dupe(u8, l) else last: {
+ const c = (top.getCursor(key("last")) catch return error.NotADump) orelse return error.NoDump;
+ break :last @constCast(c.readBytesAlloc(gpa, 4096) catch return error.NotADump);
+ };
+ errdefer gpa.free(r.label);
+ const entry = try entryOf(Memory, top, r.label) orelse return error.NoDump;
+ r.panes = if (entry.getCursor(key("panes")) catch null) |c| c.readUint() catch 0 else 0;
+ const at = (entry.getCursor(key("session")) catch return error.NotADump) orelse return error.NotADump;
+ const session: []u8 = @constCast(at.readBytesAlloc(gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.NotADump);
+ var owned: std.ArrayList(u8) = .fromOwnedSlice(session);
+ r.session_buf = .fromArrayList(gpa, &owned);
+ errdefer r.session_buf.deinit();
+ r.session = Memory.init(std.Io.failing, .{ .buffer = &r.session_buf }) catch return error.NotADump;
+ return r;
+ }
+
+ pub fn close(r: *Read) void {
+ r.session_buf.deinit();
+ r.gpa.free(r.label);
+ r.gpa.destroy(r);
+ }
+
+ /// The session's roots.
+ pub fn root(r: *Read) Memory.Cursor(.read_only) {
+ return r.session.rootCursor().readOnly();
+ }
+
+ /// The texts the session's blobs are, by hash.
+ pub fn blobs(r: *Read) !Memory.HashMap(.read_only) {
+ const root_map = try Memory.HashMap(.read_only).init(r.image.rootCursor().readOnly());
+ return Memory.HashMap(.read_only).init((try root_map.getCursor(key("blobs"))) orelse return error.NotADump);
+ }
+};
+
+/// The entry of dump `label` in the root map `root`; null when there is none.
+fn entryOf(comptime DB: type, root: DB.HashMap(.read_only), label: []const u8) !?DB.HashMap(.read_only) {
+ const dumps = (try root.getCursor(key("dumps"))) orelse return null;
+ const at = (try (try DB.HashMap(.read_only).init(dumps)).getCursor(key(label))) orelse return null;
+ return try DB.HashMap(.read_only).init(at);
+}
+
+/// The dump `label` of `o` as an image, `gpa`'s; null when it keeps none
+/// of that label.
+fn imageOf(o: *Open, gpa: std.mem.Allocator, label: []const u8) !?[]u8 {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ switch (o.where) {
+ inline else => |*w| {
+ const DB = @TypeOf(w.db);
+ const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly());
+ const entry = (try entryOf(DB, root, label)) orelse return null;
+ const session = try (try entry.getCursor(key("session")) orelse return error.NotADump).readBytesAlloc(gpa, null);
+ defer gpa.free(session);
+ const panes = if (try entry.getCursor(key("panes"))) |c| try c.readUint() else 0;
+ var blobs: std.ArrayList(pardes.walk.Blob) = .empty;
+ defer {
+ for (blobs.items) |b| gpa.free(b.bytes);
+ blobs.deinit(gpa);
+ }
+ const texts = try DB.HashMap(.read_only).init((try root.getCursor(key("blobs"))) orelse return error.NotADump);
+ if (try entry.getCursor(key("blobs"))) |list_at| {
+ var it = try (try DB.ArrayList(.read_only).init(list_at)).iterator();
+ while (try it.next()) |h| {
+ var hash: [32]u8 = undefined;
+ if ((try h.readBytes(&hash)).len != 32) return error.NotADump;
+ const text = try (try texts.getCursor(blobKey(&hash)) orelse return error.NotADump).readBytesAlloc(gpa, null);
+ errdefer gpa.free(text);
+ try blobs.append(gpa, .{ .hash = hash, .bytes = text });
+ }
+ }
+ return try image(gpa, label, session, blobs.items, panes);
+ },
+ }
+}
+
+/// The dump `label` (bare: the newest) of this session's state file, as
+/// an image, `gpa`'s; null when it keeps none of that label.
+pub fn getDump(db: *Db, p: *Pardes, gpa: std.mem.Allocator, label: ?[]const u8) !?[]u8 {
+ const o = db.opened(p) orelse return error.NoStateFile;
+ var last_buf: [4096]u8 = undefined;
+ const name = label orelse (try lastLabel(o, &last_buf)) orelse return null;
+ return imageOf(o, gpa, name);
+}
+
+/// The dump `label` of the state file at `path` as an image, `gpa`'s: an
+/// upgraded process's (its record names both), read before it has a core.
+pub fn readDumpAt(gpa: std.mem.Allocator, path: []const u8, label: []const u8) ![]u8 {
+ var db: Db = .{ .kind = .disk, .open = try openDisk(gpa, path) };
+ defer db.deinit();
+ return (try imageOf(db.open.?, gpa, label)) orelse error.NoDump;
+}
+
+/// The newest dump's label, in `buf`; null when there is none.
+fn lastLabel(o: *Open, buf: []u8) !?[]const u8 {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ switch (o.where) {
+ inline else => |*w| {
+ const DB = @TypeOf(w.db);
+ const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly());
+ const c = (try root.getCursor(key("last"))) orelse return null;
+ return try c.readBytes(buf);
+ },
+ }
+}
+
+/// The label of the state file's dump `label` (bare: its newest), in `buf`;
+/// null when it keeps none of that label. A step asks it of the authority
+/// (tape.authority): a replica keeps no state file.
+pub fn labelled(db: *Db, p: *Pardes, label: ?[]const u8, buf: []u8) ?[]const u8 {
+ const o = db.opened(p) orelse return null;
+ const name = if (label) |l| (std.fmt.bufPrint(buf, "{s}", .{l}) catch return null) else ((lastLabel(o, buf) catch return null) orelse return null);
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ switch (o.where) {
+ inline else => |*w| {
+ const DB = @TypeOf(w.db);
+ const root = DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()) catch return null;
+ return if ((entryOf(DB, root, name) catch return null) != null) name else null;
+ },
+ }
+}
+
+/// The dump in the image `bytes` put in this session's state file under a
+/// label of its own (`label`, else with `-2` and on), in one fsynced
+/// transaction that also drops the oldest past `kept` and the texts no kept
+/// dump names; waits for another session's transaction. The label, `buf`'s.
+pub fn putDump(db: *Db, p: *Pardes, bytes: []const u8, label: []const u8, buf: []u8) ![]const u8 {
+ const r = try Read.open(p.recentGpa(), bytes, null);
+ defer r.close();
+ const o = try db.locked(p, true);
+ defer unlock(o);
+ const before = committed(o) catch 0;
+ const name = try putLocked(o, r, label, buf);
+ if (db.seen == before) db.seen = committed(o) catch 0;
+ return name;
+}
+
+fn putLocked(o: *Open, r: *Read, label: []const u8, buf: []u8) ![]const u8 {
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ var ts: std.c.timespec = undefined;
+ _ = std.c.clock_gettime(.REALTIME, &ts);
+ switch (o.where) {
+ inline else => |*w, kind| {
+ const DB = @TypeOf(w.db);
+ if (kind == .disk) w.db.core.file.fsync = true;
+ defer if (kind == .disk) {
+ w.db.core.file.fsync = false;
+ };
+ var name: []const u8 = label;
+ try w.db.runContext(struct {
+ r: *Read,
+ label: []const u8,
+ buf: []u8,
+ name: *[]const u8,
+ date: u64,
+ gpa: std.mem.Allocator,
+ pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void {
+ const root = try DB.HashMap(.read_write).init(cursor.*);
+ const seq = if (try root.getCursor(key("seq"))) |c| try c.readUint() else 0;
+ try root.put(key("seq"), .{ .uint = seq + 1 });
+ const dumps = try DB.HashMap(.read_write).init(try root.putCursor(key("dumps")));
+ // A label of its own: a second dump in a second is `-2`.
+ var n: usize = 1;
+ var label_now: []const u8 = ctx.label;
+ while (try dumps.getCursor(key(label_now)) != null) {
+ n += 1;
+ label_now = try std.fmt.bufPrint(ctx.buf, "{s}-{d}", .{ ctx.label, n });
+ }
+ if (n == 1) label_now = try std.fmt.bufPrint(ctx.buf, "{s}", .{ctx.label});
+ ctx.name.* = label_now;
+ // The image's one entry, copied: its session, its texts
+ // the state file does not have yet, the numbers.
+ const image_root = try Memory.HashMap(.read_only).init(ctx.r.image.rootCursor().readOnly());
+ const from = (try entryOf(Memory, image_root, ctx.r.label)) orelse return error.NoDump;
+ const image_texts = try ctx.r.blobs();
+ try dumps.putKey(key(label_now), .{ .bytes = label_now });
+ const entry = try DB.HashMap(.read_write).init(try dumps.putCursor(key(label_now)));
+ try entry.put(key("seq"), .{ .uint = seq });
+ try entry.put(key("date"), .{ .uint = ctx.date });
+ try entry.put(key("panes"), .{ .uint = ctx.r.panes });
+ var session_at = (try from.getCursor(key("session"))) orelse return error.NoDump;
+ var size = try session_at.count();
+ try copyBytes(DB, &session_at, try entry.putCursor(key("session")), session_tag);
+ const texts = try DB.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const hashes = try DB.ArrayList(.read_write).init(try entry.putCursor(key("blobs")));
+ if (try from.getCursor(key("blobs"))) |list_at| {
+ var it = try (try Memory.ArrayList(.read_only).init(list_at)).iterator();
+ while (try it.next()) |h| {
+ var hash: [32]u8 = undefined;
+ if ((try h.readBytes(&hash)).len != 32) return error.NotADump;
+ try hashes.append(.{ .bytes = &hash });
+ var text_at = (try image_texts.getCursor(blobKey(&hash))) orelse return error.NotADump;
+ size += try text_at.count();
+ if (try texts.getCursor(blobKey(&hash)) == null) try copyBytes(DB, &text_at, try texts.putCursor(blobKey(&hash)), null);
+ }
+ }
+ try entry.put(key("size"), .{ .uint = size });
+ try root.put(key("last"), .{ .bytes = label_now });
+ try trim(DB, ctx.gpa, dumps, texts);
+ }
+ }{ .r = r, .label = label, .buf = buf, .name = &name, .date = @intCast(@max(0, ts.sec)), .gpa = r.gpa });
+ return name;
+ },
+ }
+}
+
+/// The bytes at `from` (an image's, read while it is open) written at `to`,
+/// streamed.
+fn copyBytes(comptime DB: type, from: *Memory.Cursor(.read_only), to: DB.Cursor(.read_write), tag: ?[2]u8) !void {
+ var at = to;
+ var in_buf: [64 << 10]u8 = undefined;
+ var reader = try from.reader(&in_buf);
+ var out_buf: [64 << 10]u8 = undefined;
+ var writer = try at.writer(&out_buf);
+ _ = try reader.interface.streamRemaining(&writer.interface);
+ writer.format_tag = tag;
+ try writer.finish();
+}
+
+/// The newest `kept` dumps kept, the older dropped, and every text no kept
+/// dump names (one replaced or dropped) dropped with them.
+fn trim(comptime DB: type, gpa: std.mem.Allocator, dumps: DB.HashMap(.read_write), texts: DB.HashMap(.read_write)) !void {
+ const Dumped = struct { seq: u64, hash: u256 };
+ var all: std.ArrayList(Dumped) = .empty;
+ defer all.deinit(gpa);
+ var it = try dumps.iterator();
+ while (try it.next()) |kv_at| {
+ const kv = try kv_at.readKeyValuePair();
+ const entry = try DB.HashMap(.read_only).init(kv.value_cursor);
+ const seq = if (try entry.getCursor(key("seq"))) |c| try c.readUint() else 0;
+ try all.append(gpa, .{ .seq = seq, .hash = kv.hash });
+ }
+ std.mem.sort(Dumped, all.items, {}, struct {
+ fn newer(_: void, a: Dumped, b: Dumped) bool {
+ return a.seq > b.seq;
+ }
+ }.newer);
+ if (all.items.len > kept) for (all.items[kept..]) |d| {
+ _ = try dumps.remove(d.hash);
+ };
+ // The texts the kept ones name; every other one goes.
+ var named: std.AutoHashMapUnmanaged(u256, void) = .empty;
+ defer named.deinit(gpa);
+ for (all.items[0..@min(all.items.len, kept)]) |d| {
+ const entry = try DB.HashMap(.read_only).init((try dumps.getCursor(d.hash)) orelse continue);
+ const list_at = (try entry.getCursor(key("blobs"))) orelse continue;
+ var hashes = try (try DB.ArrayList(.read_only).init(list_at)).iterator();
+ while (try hashes.next()) |h| {
+ var hash: [32]u8 = undefined;
+ if ((try h.readBytes(&hash)).len == 32) try named.put(gpa, blobKey(&hash), {});
+ }
+ }
+ var gone: std.ArrayList(u256) = .empty;
+ defer gone.deinit(gpa);
+ var texts_it = try texts.iterator();
+ while (try texts_it.next()) |kv_at| {
+ const kv = try kv_at.readKeyValuePair();
+ if (!named.contains(kv.hash)) try gone.append(gpa, kv.hash);
+ }
+ for (gone.items) |h| _ = try texts.remove(h);
+}
+
+/// The dumps the state file keeps, a line each, the newest first: its
+/// label, its date (UTC), its size in bytes and its pane count.
+pub fn listDumps(db: *Db, p: *Pardes, w: *std.Io.Writer) !void {
+ const o = db.opened(p) orelse return;
+ pardes.turn.yield();
+ defer pardes.turn.back();
+ const gpa = p.recentGpa();
+ const Row = struct { seq: u64, date: u64, size: u64, panes: u64, label: []u8 };
+ var rows: std.ArrayList(Row) = .empty;
+ defer {
+ for (rows.items) |r| gpa.free(r.label);
+ rows.deinit(gpa);
+ }
+ switch (o.where) {
+ inline else => |*d| {
+ const DB = @TypeOf(d.db);
+ const root = try DB.HashMap(.read_only).init(d.db.rootCursor().readOnly());
+ const dumps_at = (try root.getCursor(key("dumps"))) orelse return;
+ var it = try (try DB.HashMap(.read_only).init(dumps_at)).iterator();
+ while (try it.next()) |kv_at| {
+ const kv = try kv_at.readKeyValuePair();
+ const entry = try DB.HashMap(.read_only).init(kv.value_cursor);
+ var row: Row = .{ .seq = 0, .date = 0, .size = 0, .panes = 0, .label = @constCast(try kv.key_cursor.readBytesAlloc(gpa, 4096)) };
+ inline for (.{ "seq", "date", "size", "panes" }) |field| if (try entry.getCursor(key(field))) |c| {
+ @field(row, field) = c.readUint() catch 0;
+ };
+ rows.append(gpa, row) catch |err| {
+ gpa.free(row.label);
+ return err;
+ };
+ }
+ },
+ }
+ std.mem.sort(Row, rows.items, {}, struct {
+ fn newer(_: void, a: Row, b: Row) bool {
+ return a.seq > b.seq;
+ }
+ }.newer);
+ for (rows.items) |r| {
+ const es: std.time.epoch.EpochSeconds = .{ .secs = r.date };
+ const yd = es.getEpochDay().calculateYearDay();
+ const md = yd.calculateMonthDay();
+ const ds = es.getDaySeconds();
+ try w.print("{s} {d:0>4}-{d:0>2}-{d:0>2}T{d:0>2}:{d:0>2}:{d:0>2}Z {d} {d}\n", .{
+ r.label, yd.year, md.month.numeric(), @as(u8, md.day_index) + 1,
+ ds.getHoursIntoDay(), ds.getMinutesIntoHour(), ds.getSecondsIntoMinute(), r.size,
+ r.panes,
+ });
+ }
+}
+
+/// At a session's end: the state file compacted (into `<path>.new`,
+/// renamed over it) once it is past twice what its dumps and Recent hold,
+/// plus 64 MiB. Under the file's lock: no session commits to the old file
+/// meanwhile, and one that opened it reopens the new one (`opened`).
+pub fn compact(db: *Db, p: *Pardes) void {
+ if (db.kind != .disk or db.open == null) return;
+ const size = db.open.?.where.disk.file.length(ioOf()) catch return;
+ if (size < 64 << 20) return;
+ const o = db.locked(p, true) catch return;
+ defer unlock(o);
+ const d = &o.where.disk;
+ const io = ioOf();
+ const live = liveSize(&d.db) catch return;
+ if (size <= 2 * live + (64 << 20)) return;
+ var buf: [4300]u8 = undefined;
+ const new = std.fmt.bufPrint(&buf, "{s}.new", .{d.path}) catch return;
+ const cwd = std.Io.Dir.cwd();
+ const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return;
+ var new_buf: std.Io.Writer.Allocating = .init(p.recentGpa());
+ defer new_buf.deinit();
+ var offsets: std.AutoHashMap(u64, u64) = .init(p.recentGpa());
+ defer offsets.deinit();
+ _ = d.db.compact(.buffered_file, .{ .file = file, .buffer = &new_buf, .hash_id = hash_id }, &offsets) catch {
+ file.close(io);
+ cwd.deleteFile(io, new) catch {};
+ return;
+ };
+ file.close(io);
+ cwd.rename(new, cwd, d.path, io) catch cwd.deleteFile(io, new) catch {};
+}
+
+/// What the state file's latest moment holds: Recent, each dump's
+/// session, each text once.
+fn liveSize(db: *Disk) !u64 {
+ const root = try Disk.HashMap(.read_only).init(db.rootCursor().readOnly());
+ var live: u64 = 0;
+ if (try root.getCursor(key("recent"))) |c| live += try c.count();
+ if (try root.getCursor(key("dumps"))) |at| {
+ var it = try (try Disk.HashMap(.read_only).init(at)).iterator();
+ while (try it.next()) |kv_at| {
+ const entry = try Disk.HashMap(.read_only).init((try kv_at.readKeyValuePair()).value_cursor);
+ if (try entry.getCursor(key("session"))) |c| live += try c.count();
+ }
+ }
+ if (try root.getCursor(key("blobs"))) |at| {
+ var it = try (try Disk.HashMap(.read_only).init(at)).iterator();
+ while (try it.next()) |kv_at| live += try (try kv_at.readKeyValuePair()).value_cursor.count();
+ }
+ return live;
+}
+
// ---- tests ----
const testing = std.testing;
@@ -379,11 +866,42 @@ test "state file: one that is no database of pardes's never stops pardes: it is
try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len);
a.recent.opened(a.recentGpa(), "/w/a.zig");
a.saveRecent();
- const kept = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20));
- defer testing.allocator.free(kept);
- try testing.expect(std.mem.startsWith(u8, kept, "\x00\xffnot a database"));
+ const aside = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20));
+ defer testing.allocator.free(aside);
+ try testing.expect(std.mem.startsWith(u8, aside, "\x00\xffnot a database"));
const b = try testWindow(dir);
defer b.deinit();
try testing.expectEqual(@as(usize, 1), b.recent.entries.items.len);
try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path);
}
+
+test "state file: two dumps with one label get labels of their own" {
+ const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .state = .memory });
+ defer p.deinit();
+ const bytes = try pardes.dump.dumpBytes(p);
+ defer testing.allocator.free(bytes);
+ var buf: [64]u8 = undefined;
+ try testing.expectEqualStrings("pardes-x", try p.db.putDump(p, bytes, "pardes-x", &buf));
+ try testing.expectEqualStrings("pardes-x-2", try p.db.putDump(p, bytes, "pardes-x", &buf));
+ try testing.expectEqualStrings("pardes-x-3", try p.db.putDump(p, bytes, "pardes-x", &buf));
+ var label_buf: [64]u8 = undefined;
+ try testing.expectEqualStrings("pardes-x-3", p.db.labelled(p, null, &label_buf).?);
+}
+
+test "state file: a Dump makes every directory missing on the way to its DumpDir, and the file there" {
+ if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest;
+ var tmp = testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const base = try testDir(&tmp, &dir_buf);
+ var setting: [4200]u8 = undefined;
+ const dir = try std.fmt.bufPrint(&setting, "{s}/fresh/.local/share/pardes", .{base});
+ const p = try testWindow(dir);
+ defer p.deinit();
+ try pardes.dump.dumpState(p);
+ pardes.host_io.writeDump(p, p.dump_out.?);
+ try testing.expect(std.mem.startsWith(u8, p.last_dump.?, "pardes-"));
+ try testing.expect(fileSize(dir) > 0);
+ var label_buf: [64]u8 = undefined;
+ try testing.expect(p.db.labelled(p, p.last_dump.?, &label_buf) != null);
+}