diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Db.zig | 532 | ||||
| -rw-r--r-- | src/Handover.zig | 61 | ||||
| -rw-r--r-- | src/builtins.zig | 47 | ||||
| -rw-r--r-- | src/config.zig | 2 | ||||
| -rw-r--r-- | src/dump.zig | 275 | ||||
| -rw-r--r-- | src/fs-help.txt | 3 | ||||
| -rw-r--r-- | src/host_io.zig | 31 | ||||
| -rw-r--r-- | src/macos.zig | 8 | ||||
| -rw-r--r-- | src/ninep/tree.zig | 32 | ||||
| -rw-r--r-- | src/pardes.zig | 10 | ||||
| -rw-r--r-- | src/replica.zig | 4 | ||||
| -rw-r--r-- | src/upgrade.zig | 77 | ||||
| -rw-r--r-- | src/walk.zig | 808 |
13 files changed, 1700 insertions, 190 deletions
@@ -103,7 +103,9 @@ fn opened(db: *Db, p: *Pardes) ?*Open { var buf: [4200]u8 = undefined; const path = statePath(&buf, p.settings.dump_dir.get()) orelse return null; if (db.open) |o| { - if (std.mem.eql(u8, o.where.disk.path, path)) return o; + // The same file still: another session's compaction renames + // a new one over it, which this one then opens. + if (std.mem.eql(u8, o.where.disk.path, path) and !replaced(o)) return o; // The DumpDir moved: its own file, read afresh. db.deinit(); db.seen = 0; @@ -168,6 +170,17 @@ fn openDisk(gpa: std.mem.Allocator, path: []const u8) !*Open { } } +/// Whether the file at the open one's path is another now (a compaction's). +fn replaced(o: *Open) bool { + const d = o.where.disk; + pardes.turn.yield(); + defer pardes.turn.back(); + const io = ioOf(); + const open_st = d.file.stat(io) catch return false; + const path_st = std.Io.Dir.cwd().statFile(io, d.path, .{}) catch return false; + return open_st.inode != path_st.inode; +} + fn committed(o: *Open) !u64 { pardes.turn.yield(); defer pardes.turn.back(); @@ -192,6 +205,21 @@ fn lock(o: *Open, wait: bool) bool { return file.tryLock(ioOf(), .exclusive) catch false; } +/// The database, its file locked for a transaction (`wait`: waited for). A +/// file a compaction replaced while this waited is let go, and the new one +/// opened and locked. +fn locked(db: *Db, p: *Pardes, wait: bool) error{ NoStateFile, Busy }!*Open { + for (0..2) |_| { + const o = db.opened(p) orelse return error.NoStateFile; + if (!lock(o, wait)) return error.Busy; + if (o.where == .memory or !replaced(o)) return o; + unlock(o); + db.deinit(); + db.seen = 0; + } + return error.Busy; +} + fn unlock(o: *Open) void { switch (o.where) { .disk => |d| d.file.unlock(ioOf()), @@ -256,9 +284,11 @@ pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { if (!p.recent.dirty) return; if (db.kind != .off) written: { if (!db.loaded) return; - const o = db.opened(p) orelse break :written; - // Another session's lock: the next step tries again. - if (!lock(o, wait)) return; + const o = db.locked(p, wait) catch |err| switch (err) { + // Another session's lock: the next step tries again. + error.Busy => return, + error.NoStateFile => break :written, + }; defer unlock(o); db.mergeRecent(p, o); p.recent.changedFirst(); @@ -275,6 +305,463 @@ pub fn writeRecent(db: *Db, p: *Pardes, wait: bool) void { p.recent.dirty = false; } +// ---- the dumps ---- +// +// Each Dump is a labelled entry of the state file's `dumps`, written in one +// transaction, fsynced (a dump is worth the wait; Recent's commits are not): +// a full disk or a kill in the middle leaves the dump before it whole, as +// xitdb writes its header last. +// +// dumps/<label> seq, date (seconds, UTC), panes, size, the session (a +// database of its own: walk.Put's maps, its bytes one +// value) and the hashes of the texts it keeps beside it +// blobs/<hash> each such text once, by its BLAKE3 hash: an unchanged +// text is written once, however many dumps keep it +// last the newest dump's label +// seq the next dump's number +// +// A Dump is labelled `pardes-<date>-<time>` (UTC; `-2` and on for another +// in the same second), an Upgrade's `upgrade-<pid>`. The newest `kept` are +// kept: one more drops the oldest, and the texts no kept dump names, in +// the same transaction. A dump travels as an image, a database in memory +// holding it alone in the same shape (`image`): what a Dump hands its host, +// what a Restore of a label is read into (`imageOf`), what `Dump export` +// and a handover write as a file, and what `-l` and `Restore <path>` read. + +/// How many dumps the state file keeps, the newest. +pub const kept = 32; +/// The text of a dump's blob is its BLAKE3 hash's key. +fn blobKey(hash: *const [32]u8) u256 { + return std.mem.readInt(u256, hash, .big); +} + +const session_tag = "db".*; + +/// A dump's image: the one dump `label` (its session's bytes, the texts it +/// keeps beside it and its pane count) in a database in memory, `last` +/// naming it, `gpa`'s. Of the same session, the same bytes: its ids are +/// std.Io.failing's zeros, never random ones. +pub fn image(gpa: std.mem.Allocator, label: []const u8, session: []const u8, blobs: []const pardes.walk.Blob, panes: u64) ![]u8 { + var buf: std.Io.Writer.Allocating = .init(gpa); + errdefer buf.deinit(); + var db = try Memory.init(std.Io.failing, .{ .buffer = &buf, .hash_id = hash_id, .file_id = 0 }); + try db.runContext(struct { + label: []const u8, + session: []const u8, + blobs: []const pardes.walk.Blob, + panes: u64, + pub fn run(ctx: @This(), cursor: *Memory.Cursor(.read_write)) !void { + const root = try Memory.HashMap(.read_write).init(cursor.*); + try root.put(key("last"), .{ .bytes = ctx.label }); + const dumps = try Memory.HashMap(.read_write).init(try root.putCursor(key("dumps"))); + try dumps.putKey(key(ctx.label), .{ .bytes = ctx.label }); + const entry = try Memory.HashMap(.read_write).init(try dumps.putCursor(key(ctx.label))); + try entry.put(key("session"), .{ .bytes_object = .{ .value = ctx.session, .format_tag = session_tag } }); + try entry.put(key("panes"), .{ .uint = ctx.panes }); + const hashes = try Memory.ArrayList(.read_write).init(try entry.putCursor(key("blobs"))); + const texts = try Memory.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + for (ctx.blobs) |*b| { + try hashes.append(.{ .bytes = &b.hash }); + try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes }); + } + } + }{ .label = label, .session = session, .blobs = blobs, .panes = panes }); + return buf.toOwnedSlice(); +} + +/// Whether `bytes` are a database of xitdb's (an image, a state file): +/// what `-l` and `Restore <path>` tell from an old dump by. +pub fn isImage(bytes: []const u8) bool { + return std.mem.startsWith(u8, bytes, "xit"); +} + +/// A dump's image opened for a load: the session's database, and the +/// texts its blobs are. The image's bytes are the caller's while it is open. +pub const Read = struct { + gpa: std.mem.Allocator, + image_buf: std.Io.Writer.Allocating, + image: Memory, + session_buf: std.Io.Writer.Allocating, + session: Memory, + label: []u8, + panes: u64, + + /// The image `bytes` at its dump `label`, else its last one. + pub fn open(gpa: std.mem.Allocator, bytes: []const u8, label: ?[]const u8) !*Read { + if (!isImage(bytes)) return error.NotADump; + const r = try gpa.create(Read); + errdefer gpa.destroy(r); + // Read, never written: the bytes are borrowed, never freed here. + var borrowed: std.ArrayList(u8) = .fromOwnedSlice(@constCast(bytes)); + r.* = .{ .gpa = gpa, .image_buf = .fromArrayList(gpa, &borrowed), .image = undefined, .session_buf = undefined, .session = undefined, .label = &.{}, .panes = 0 }; + r.image = Memory.init(std.Io.failing, .{ .buffer = &r.image_buf }) catch return error.NotADump; + const top = Memory.HashMap(.read_only).init(r.image.rootCursor().readOnly()) catch return error.NotADump; + r.label = if (label) |l| try gpa.dupe(u8, l) else last: { + const c = (top.getCursor(key("last")) catch return error.NotADump) orelse return error.NoDump; + break :last @constCast(c.readBytesAlloc(gpa, 4096) catch return error.NotADump); + }; + errdefer gpa.free(r.label); + const entry = try entryOf(Memory, top, r.label) orelse return error.NoDump; + r.panes = if (entry.getCursor(key("panes")) catch null) |c| c.readUint() catch 0 else 0; + const at = (entry.getCursor(key("session")) catch return error.NotADump) orelse return error.NotADump; + const session: []u8 = @constCast(at.readBytesAlloc(gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.NotADump); + var owned: std.ArrayList(u8) = .fromOwnedSlice(session); + r.session_buf = .fromArrayList(gpa, &owned); + errdefer r.session_buf.deinit(); + r.session = Memory.init(std.Io.failing, .{ .buffer = &r.session_buf }) catch return error.NotADump; + return r; + } + + pub fn close(r: *Read) void { + r.session_buf.deinit(); + r.gpa.free(r.label); + r.gpa.destroy(r); + } + + /// The session's roots. + pub fn root(r: *Read) Memory.Cursor(.read_only) { + return r.session.rootCursor().readOnly(); + } + + /// The texts the session's blobs are, by hash. + pub fn blobs(r: *Read) !Memory.HashMap(.read_only) { + const root_map = try Memory.HashMap(.read_only).init(r.image.rootCursor().readOnly()); + return Memory.HashMap(.read_only).init((try root_map.getCursor(key("blobs"))) orelse return error.NotADump); + } +}; + +/// The entry of dump `label` in the root map `root`; null when there is none. +fn entryOf(comptime DB: type, root: DB.HashMap(.read_only), label: []const u8) !?DB.HashMap(.read_only) { + const dumps = (try root.getCursor(key("dumps"))) orelse return null; + const at = (try (try DB.HashMap(.read_only).init(dumps)).getCursor(key(label))) orelse return null; + return try DB.HashMap(.read_only).init(at); +} + +/// The dump `label` of `o` as an image, `gpa`'s; null when it keeps none +/// of that label. +fn imageOf(o: *Open, gpa: std.mem.Allocator, label: []const u8) !?[]u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); + const entry = (try entryOf(DB, root, label)) orelse return null; + const session = try (try entry.getCursor(key("session")) orelse return error.NotADump).readBytesAlloc(gpa, null); + defer gpa.free(session); + const panes = if (try entry.getCursor(key("panes"))) |c| try c.readUint() else 0; + var blobs: std.ArrayList(pardes.walk.Blob) = .empty; + defer { + for (blobs.items) |b| gpa.free(b.bytes); + blobs.deinit(gpa); + } + const texts = try DB.HashMap(.read_only).init((try root.getCursor(key("blobs"))) orelse return error.NotADump); + if (try entry.getCursor(key("blobs"))) |list_at| { + var it = try (try DB.ArrayList(.read_only).init(list_at)).iterator(); + while (try it.next()) |h| { + var hash: [32]u8 = undefined; + if ((try h.readBytes(&hash)).len != 32) return error.NotADump; + const text = try (try texts.getCursor(blobKey(&hash)) orelse return error.NotADump).readBytesAlloc(gpa, null); + errdefer gpa.free(text); + try blobs.append(gpa, .{ .hash = hash, .bytes = text }); + } + } + return try image(gpa, label, session, blobs.items, panes); + }, + } +} + +/// The dump `label` (bare: the newest) of this session's state file, as +/// an image, `gpa`'s; null when it keeps none of that label. +pub fn getDump(db: *Db, p: *Pardes, gpa: std.mem.Allocator, label: ?[]const u8) !?[]u8 { + const o = db.opened(p) orelse return error.NoStateFile; + var last_buf: [4096]u8 = undefined; + const name = label orelse (try lastLabel(o, &last_buf)) orelse return null; + return imageOf(o, gpa, name); +} + +/// The dump `label` of the state file at `path` as an image, `gpa`'s: an +/// upgraded process's (its record names both), read before it has a core. +pub fn readDumpAt(gpa: std.mem.Allocator, path: []const u8, label: []const u8) ![]u8 { + var db: Db = .{ .kind = .disk, .open = try openDisk(gpa, path) }; + defer db.deinit(); + return (try imageOf(db.open.?, gpa, label)) orelse error.NoDump; +} + +/// The newest dump's label, in `buf`; null when there is none. +fn lastLabel(o: *Open, buf: []u8) !?[]const u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = try DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()); + const c = (try root.getCursor(key("last"))) orelse return null; + return try c.readBytes(buf); + }, + } +} + +/// The label of the state file's dump `label` (bare: its newest), in `buf`; +/// null when it keeps none of that label. A step asks it of the authority +/// (tape.authority): a replica keeps no state file. +pub fn labelled(db: *Db, p: *Pardes, label: ?[]const u8, buf: []u8) ?[]const u8 { + const o = db.opened(p) orelse return null; + const name = if (label) |l| (std.fmt.bufPrint(buf, "{s}", .{l}) catch return null) else ((lastLabel(o, buf) catch return null) orelse return null); + pardes.turn.yield(); + defer pardes.turn.back(); + switch (o.where) { + inline else => |*w| { + const DB = @TypeOf(w.db); + const root = DB.HashMap(.read_only).init(w.db.rootCursor().readOnly()) catch return null; + return if ((entryOf(DB, root, name) catch return null) != null) name else null; + }, + } +} + +/// The dump in the image `bytes` put in this session's state file under a +/// label of its own (`label`, else with `-2` and on), in one fsynced +/// transaction that also drops the oldest past `kept` and the texts no kept +/// dump names; waits for another session's transaction. The label, `buf`'s. +pub fn putDump(db: *Db, p: *Pardes, bytes: []const u8, label: []const u8, buf: []u8) ![]const u8 { + const r = try Read.open(p.recentGpa(), bytes, null); + defer r.close(); + const o = try db.locked(p, true); + defer unlock(o); + const before = committed(o) catch 0; + const name = try putLocked(o, r, label, buf); + if (db.seen == before) db.seen = committed(o) catch 0; + return name; +} + +fn putLocked(o: *Open, r: *Read, label: []const u8, buf: []u8) ![]const u8 { + pardes.turn.yield(); + defer pardes.turn.back(); + var ts: std.c.timespec = undefined; + _ = std.c.clock_gettime(.REALTIME, &ts); + switch (o.where) { + inline else => |*w, kind| { + const DB = @TypeOf(w.db); + if (kind == .disk) w.db.core.file.fsync = true; + defer if (kind == .disk) { + w.db.core.file.fsync = false; + }; + var name: []const u8 = label; + try w.db.runContext(struct { + r: *Read, + label: []const u8, + buf: []u8, + name: *[]const u8, + date: u64, + gpa: std.mem.Allocator, + pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { + const root = try DB.HashMap(.read_write).init(cursor.*); + const seq = if (try root.getCursor(key("seq"))) |c| try c.readUint() else 0; + try root.put(key("seq"), .{ .uint = seq + 1 }); + const dumps = try DB.HashMap(.read_write).init(try root.putCursor(key("dumps"))); + // A label of its own: a second dump in a second is `-2`. + var n: usize = 1; + var label_now: []const u8 = ctx.label; + while (try dumps.getCursor(key(label_now)) != null) { + n += 1; + label_now = try std.fmt.bufPrint(ctx.buf, "{s}-{d}", .{ ctx.label, n }); + } + if (n == 1) label_now = try std.fmt.bufPrint(ctx.buf, "{s}", .{ctx.label}); + ctx.name.* = label_now; + // The image's one entry, copied: its session, its texts + // the state file does not have yet, the numbers. + const image_root = try Memory.HashMap(.read_only).init(ctx.r.image.rootCursor().readOnly()); + const from = (try entryOf(Memory, image_root, ctx.r.label)) orelse return error.NoDump; + const image_texts = try ctx.r.blobs(); + try dumps.putKey(key(label_now), .{ .bytes = label_now }); + const entry = try DB.HashMap(.read_write).init(try dumps.putCursor(key(label_now))); + try entry.put(key("seq"), .{ .uint = seq }); + try entry.put(key("date"), .{ .uint = ctx.date }); + try entry.put(key("panes"), .{ .uint = ctx.r.panes }); + var session_at = (try from.getCursor(key("session"))) orelse return error.NoDump; + var size = try session_at.count(); + try copyBytes(DB, &session_at, try entry.putCursor(key("session")), session_tag); + const texts = try DB.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const hashes = try DB.ArrayList(.read_write).init(try entry.putCursor(key("blobs"))); + if (try from.getCursor(key("blobs"))) |list_at| { + var it = try (try Memory.ArrayList(.read_only).init(list_at)).iterator(); + while (try it.next()) |h| { + var hash: [32]u8 = undefined; + if ((try h.readBytes(&hash)).len != 32) return error.NotADump; + try hashes.append(.{ .bytes = &hash }); + var text_at = (try image_texts.getCursor(blobKey(&hash))) orelse return error.NotADump; + size += try text_at.count(); + if (try texts.getCursor(blobKey(&hash)) == null) try copyBytes(DB, &text_at, try texts.putCursor(blobKey(&hash)), null); + } + } + try entry.put(key("size"), .{ .uint = size }); + try root.put(key("last"), .{ .bytes = label_now }); + try trim(DB, ctx.gpa, dumps, texts); + } + }{ .r = r, .label = label, .buf = buf, .name = &name, .date = @intCast(@max(0, ts.sec)), .gpa = r.gpa }); + return name; + }, + } +} + +/// The bytes at `from` (an image's, read while it is open) written at `to`, +/// streamed. +fn copyBytes(comptime DB: type, from: *Memory.Cursor(.read_only), to: DB.Cursor(.read_write), tag: ?[2]u8) !void { + var at = to; + var in_buf: [64 << 10]u8 = undefined; + var reader = try from.reader(&in_buf); + var out_buf: [64 << 10]u8 = undefined; + var writer = try at.writer(&out_buf); + _ = try reader.interface.streamRemaining(&writer.interface); + writer.format_tag = tag; + try writer.finish(); +} + +/// The newest `kept` dumps kept, the older dropped, and every text no kept +/// dump names (one replaced or dropped) dropped with them. +fn trim(comptime DB: type, gpa: std.mem.Allocator, dumps: DB.HashMap(.read_write), texts: DB.HashMap(.read_write)) !void { + const Dumped = struct { seq: u64, hash: u256 }; + var all: std.ArrayList(Dumped) = .empty; + defer all.deinit(gpa); + var it = try dumps.iterator(); + while (try it.next()) |kv_at| { + const kv = try kv_at.readKeyValuePair(); + const entry = try DB.HashMap(.read_only).init(kv.value_cursor); + const seq = if (try entry.getCursor(key("seq"))) |c| try c.readUint() else 0; + try all.append(gpa, .{ .seq = seq, .hash = kv.hash }); + } + std.mem.sort(Dumped, all.items, {}, struct { + fn newer(_: void, a: Dumped, b: Dumped) bool { + return a.seq > b.seq; + } + }.newer); + if (all.items.len > kept) for (all.items[kept..]) |d| { + _ = try dumps.remove(d.hash); + }; + // The texts the kept ones name; every other one goes. + var named: std.AutoHashMapUnmanaged(u256, void) = .empty; + defer named.deinit(gpa); + for (all.items[0..@min(all.items.len, kept)]) |d| { + const entry = try DB.HashMap(.read_only).init((try dumps.getCursor(d.hash)) orelse continue); + const list_at = (try entry.getCursor(key("blobs"))) orelse continue; + var hashes = try (try DB.ArrayList(.read_only).init(list_at)).iterator(); + while (try hashes.next()) |h| { + var hash: [32]u8 = undefined; + if ((try h.readBytes(&hash)).len == 32) try named.put(gpa, blobKey(&hash), {}); + } + } + var gone: std.ArrayList(u256) = .empty; + defer gone.deinit(gpa); + var texts_it = try texts.iterator(); + while (try texts_it.next()) |kv_at| { + const kv = try kv_at.readKeyValuePair(); + if (!named.contains(kv.hash)) try gone.append(gpa, kv.hash); + } + for (gone.items) |h| _ = try texts.remove(h); +} + +/// The dumps the state file keeps, a line each, the newest first: its +/// label, its date (UTC), its size in bytes and its pane count. +pub fn listDumps(db: *Db, p: *Pardes, w: *std.Io.Writer) !void { + const o = db.opened(p) orelse return; + pardes.turn.yield(); + defer pardes.turn.back(); + const gpa = p.recentGpa(); + const Row = struct { seq: u64, date: u64, size: u64, panes: u64, label: []u8 }; + var rows: std.ArrayList(Row) = .empty; + defer { + for (rows.items) |r| gpa.free(r.label); + rows.deinit(gpa); + } + switch (o.where) { + inline else => |*d| { + const DB = @TypeOf(d.db); + const root = try DB.HashMap(.read_only).init(d.db.rootCursor().readOnly()); + const dumps_at = (try root.getCursor(key("dumps"))) orelse return; + var it = try (try DB.HashMap(.read_only).init(dumps_at)).iterator(); + while (try it.next()) |kv_at| { + const kv = try kv_at.readKeyValuePair(); + const entry = try DB.HashMap(.read_only).init(kv.value_cursor); + var row: Row = .{ .seq = 0, .date = 0, .size = 0, .panes = 0, .label = @constCast(try kv.key_cursor.readBytesAlloc(gpa, 4096)) }; + inline for (.{ "seq", "date", "size", "panes" }) |field| if (try entry.getCursor(key(field))) |c| { + @field(row, field) = c.readUint() catch 0; + }; + rows.append(gpa, row) catch |err| { + gpa.free(row.label); + return err; + }; + } + }, + } + std.mem.sort(Row, rows.items, {}, struct { + fn newer(_: void, a: Row, b: Row) bool { + return a.seq > b.seq; + } + }.newer); + for (rows.items) |r| { + const es: std.time.epoch.EpochSeconds = .{ .secs = r.date }; + const yd = es.getEpochDay().calculateYearDay(); + const md = yd.calculateMonthDay(); + const ds = es.getDaySeconds(); + try w.print("{s} {d:0>4}-{d:0>2}-{d:0>2}T{d:0>2}:{d:0>2}:{d:0>2}Z {d} {d}\n", .{ + r.label, yd.year, md.month.numeric(), @as(u8, md.day_index) + 1, + ds.getHoursIntoDay(), ds.getMinutesIntoHour(), ds.getSecondsIntoMinute(), r.size, + r.panes, + }); + } +} + +/// At a session's end: the state file compacted (into `<path>.new`, +/// renamed over it) once it is past twice what its dumps and Recent hold, +/// plus 64 MiB. Under the file's lock: no session commits to the old file +/// meanwhile, and one that opened it reopens the new one (`opened`). +pub fn compact(db: *Db, p: *Pardes) void { + if (db.kind != .disk or db.open == null) return; + const size = db.open.?.where.disk.file.length(ioOf()) catch return; + if (size < 64 << 20) return; + const o = db.locked(p, true) catch return; + defer unlock(o); + const d = &o.where.disk; + const io = ioOf(); + const live = liveSize(&d.db) catch return; + if (size <= 2 * live + (64 << 20)) return; + var buf: [4300]u8 = undefined; + const new = std.fmt.bufPrint(&buf, "{s}.new", .{d.path}) catch return; + const cwd = std.Io.Dir.cwd(); + const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return; + var new_buf: std.Io.Writer.Allocating = .init(p.recentGpa()); + defer new_buf.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(p.recentGpa()); + defer offsets.deinit(); + _ = d.db.compact(.buffered_file, .{ .file = file, .buffer = &new_buf, .hash_id = hash_id }, &offsets) catch { + file.close(io); + cwd.deleteFile(io, new) catch {}; + return; + }; + file.close(io); + cwd.rename(new, cwd, d.path, io) catch cwd.deleteFile(io, new) catch {}; +} + +/// What the state file's latest moment holds: Recent, each dump's +/// session, each text once. +fn liveSize(db: *Disk) !u64 { + const root = try Disk.HashMap(.read_only).init(db.rootCursor().readOnly()); + var live: u64 = 0; + if (try root.getCursor(key("recent"))) |c| live += try c.count(); + if (try root.getCursor(key("dumps"))) |at| { + var it = try (try Disk.HashMap(.read_only).init(at)).iterator(); + while (try it.next()) |kv_at| { + const entry = try Disk.HashMap(.read_only).init((try kv_at.readKeyValuePair()).value_cursor); + if (try entry.getCursor(key("session"))) |c| live += try c.count(); + } + } + if (try root.getCursor(key("blobs"))) |at| { + var it = try (try Disk.HashMap(.read_only).init(at)).iterator(); + while (try it.next()) |kv_at| live += try (try kv_at.readKeyValuePair()).value_cursor.count(); + } + return live; +} + // ---- tests ---- const testing = std.testing; @@ -379,11 +866,42 @@ test "state file: one that is no database of pardes's never stops pardes: it is try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len); a.recent.opened(a.recentGpa(), "/w/a.zig"); a.saveRecent(); - const kept = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20)); - defer testing.allocator.free(kept); - try testing.expect(std.mem.startsWith(u8, kept, "\x00\xffnot a database")); + const aside = try tmp.dir.readFileAlloc(testing.io, file_name ++ ".broken", testing.allocator, .limited(1 << 20)); + defer testing.allocator.free(aside); + try testing.expect(std.mem.startsWith(u8, aside, "\x00\xffnot a database")); const b = try testWindow(dir); defer b.deinit(); try testing.expectEqual(@as(usize, 1), b.recent.entries.items.len); try testing.expectEqualStrings("/w/a.zig", b.recent.entries.items[0].path); } + +test "state file: two dumps with one label get labels of their own" { + const p = try Pardes.init(testing.allocator, .{ .tty_only = true, .state = .memory }); + defer p.deinit(); + const bytes = try pardes.dump.dumpBytes(p); + defer testing.allocator.free(bytes); + var buf: [64]u8 = undefined; + try testing.expectEqualStrings("pardes-x", try p.db.putDump(p, bytes, "pardes-x", &buf)); + try testing.expectEqualStrings("pardes-x-2", try p.db.putDump(p, bytes, "pardes-x", &buf)); + try testing.expectEqualStrings("pardes-x-3", try p.db.putDump(p, bytes, "pardes-x", &buf)); + var label_buf: [64]u8 = undefined; + try testing.expectEqualStrings("pardes-x-3", p.db.labelled(p, null, &label_buf).?); +} + +test "state file: a Dump makes every directory missing on the way to its DumpDir, and the file there" { + if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest; + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const base = try testDir(&tmp, &dir_buf); + var setting: [4200]u8 = undefined; + const dir = try std.fmt.bufPrint(&setting, "{s}/fresh/.local/share/pardes", .{base}); + const p = try testWindow(dir); + defer p.deinit(); + try pardes.dump.dumpState(p); + pardes.host_io.writeDump(p, p.dump_out.?); + try testing.expect(std.mem.startsWith(u8, p.last_dump.?, "pardes-")); + try testing.expect(fileSize(dir) > 0); + var label_buf: [64]u8 = undefined; + try testing.expect(p.db.labelled(p, p.last_dump.?, &label_buf) != null); +} diff --git a/src/Handover.zig b/src/Handover.zig index d245a6af..2233d228 100644 --- a/src/Handover.zig +++ b/src/Handover.zig @@ -1,7 +1,8 @@ //! Panes between windows, as a browser's tabs go between its windows. A //! pane's grip, or a column's, released past the window's edge carries it -//! out: dumped alone (`Carried`, each pane as dump.savePane makes it), it -//! opens in a new window of its own, `pardes-gui -l` of that dump. The pane +//! out: dumped alone (`Carried`, each pane as dump.savePane makes it, in +//! the image of one dump, `handover-<seq>`: Db.zig), it opens in a new +//! window of its own, `pardes-gui -l` of that dump. The pane //! closes here only once //! the new window answers on its 9P socket, so its text is never in no //! window at all; a failed handover leaves it where it was, and a pane @@ -35,7 +36,8 @@ const libc = std.c; /// to it, or a whole column. pub const Kind = enum { pane, column }; -/// A handover's dump: `carried_magic`, then this. +/// What a handover's image holds, as its root `carried` (a build before +/// the image's, 0.24, wrote `carried_magic`, then this as a byte dump). const Carried = struct { kind: Kind, /// A carried column's tag. @@ -114,7 +116,7 @@ pub fn begin(h: *Handover, p: *Pardes, carried: Kind, at: usize) void { else "only text panes go to another window, for now"); } - const bytes = dumpCarried(p, carried, at, ids[0..n]) catch |err| return p.reportError(said, "handover", err); + const bytes = dumpCarried(p, carried, at, ids[0..n], h.seq +% 1) catch |err| return p.reportError(said, "handover", err); h.deinit(p.gpa); h.bytes = bytes; h.seq +%= 1; @@ -180,8 +182,8 @@ pub fn finish(h: *Handover, p: *Pardes, seq: u32, to: []const u8, failure: ?[]co } /// The dump of panes `ids` (gpa's), carried as `kind` from column `at`'s -/// (a pane's, `at` is its slot). -fn dumpCarried(p: *Pardes, kind: Kind, at: usize, ids: []const usize) ![]u8 { +/// (a pane's, `at` is its slot): an image of one dump, handover `seq`. +fn dumpCarried(p: *Pardes, kind: Kind, at: usize, ids: []const usize, seq: u32) ![]u8 { const arena = p.scratch.allocator(); const bytes = try arena.alloc([]const u8, ids.len); const weights = try arena.alloc(f32, ids.len); @@ -200,8 +202,44 @@ fn dumpCarried(p: *Pardes, kind: Kind, at: usize, ids: []const usize) ![]u8 { .weights = weights, .owners = owners, }; - const body = try walk.save(Carried, arena, &value); - return std.mem.concat(p.gpa, u8, &.{ carried_magic, body }); + var buf: std.Io.Writer.Allocating = .init(p.gpa); + defer buf.deinit(); + var db = try walk.Mem.init(std.Io.failing, .{ .buffer = &buf, .file_id = 0 }); + var w: walk.Put = .{ .gpa = p.gpa }; + defer w.deinit(); + try db.runContext(struct { + w: *walk.Put, + value: *const Carried, + pub fn run(ctx: @This(), cursor: *walk.Mem.Cursor(.read_write)) !void { + try ctx.w.member(try walk.Mem.HashMap(.read_write).init(cursor.*), "carried", Carried, ctx.value); + } + }{ .w = &w, .value = &value }); + var name: [32]u8 = undefined; + return pardes.Db.image(p.gpa, std.fmt.bufPrint(&name, "handover-{d}", .{seq}) catch "handover", buf.written(), w.blobs.items, ids.len); +} + +/// Whether `bytes` are a handover's dump (`dumpCarried`'s, or an older +/// build's), not a session's. +pub fn carries(bytes: []const u8) bool { + if (std.mem.startsWith(u8, bytes, carried_magic)) return true; + if (!pardes.Db.isImage(bytes)) return false; + const r = pardes.Db.Read.open(std.heap.page_allocator, bytes, null) catch return false; + defer r.close(); + return (walk.Get.member(r.root(), "carried") catch null) != null; +} + +/// The carried panes in `bytes`, made in `a`. +fn readCarried(a: std.mem.Allocator, bytes: []const u8) !Carried { + if (pardes.Db.isImage(bytes)) { + const r = pardes.Db.Read.open(a, bytes, null) catch return error.NotCarried; + defer r.close(); + const at = (walk.Get.member(r.root(), "carried") catch return error.NotCarried) orelse return error.NotCarried; + var l: walk.Get = .init(a); + defer l.deinit(); + return l.get(Carried, at) catch error.NotCarried; + } + if (!std.mem.startsWith(u8, bytes, carried_magic)) return error.NotCarried; + return walk.load(Carried, a, bytes[carried_magic.len..], null); } // ---- the receiving window ---- @@ -212,10 +250,9 @@ fn dumpCarried(p: *Pardes, kind: Kind, at: usize, ids: []const usize) ![]u8 { /// alone. Every other pane stays where it was. Answers the slot of the /// first pane taken; on an error nothing of it is left here. pub fn adopt(p: *Pardes, bytes: []const u8) !usize { - if (!std.mem.startsWith(u8, bytes, carried_magic)) return error.NotCarried; var arena: std.heap.ArenaAllocator = .init(p.gpa); defer arena.deinit(); - const st = try walk.load(Carried, arena.allocator(), bytes[carried_magic.len..], null); + const st = try readCarried(arena.allocator(), bytes); const carried = st.kind; if (st.panes.len == 0 or st.weights.len != st.panes.len or st.owners.len != st.panes.len) return error.NotCarried; // Where the pointer is, else the middle of the pane with the keyboard. @@ -327,7 +364,7 @@ pub const Job = struct { pub fn dumpPath(gpa: std.mem.Allocator, seq: u32) ![:0]u8 { var dir_buf: [1024]u8 = undefined; const dir: []const u8 = if (libc.getenv("XDG_RUNTIME_DIR")) |x| std.mem.span(x) else dump.defaultDirectory(&dir_buf) orelse return error.NoDirectory; - return std.fmt.allocPrintSentinel(gpa, "{s}/pardes-handover-{d}-{d}.dump", .{ dir, libc.getpid(), seq }, 0); + return std.fmt.allocPrintSentinel(gpa, "{s}/pardes-handover-{d}-{d}.xitdb", .{ dir, libc.getpid(), seq }, 0); } /// Runs the handover's carrier on `io` (io.concurrent), kept in `h.task` @@ -494,7 +531,7 @@ test "a grip let go past the window's edge dumps its pane alone, and closes it o try std.testing.expectEqual(before, p.rects[other]); var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); defer arena.deinit(); - const carried = try walk.load(Carried, arena.allocator(), p.handover.bytes.?[carried_magic.len..], null); + const carried = try readCarried(arena.allocator(), p.handover.bytes.?); try std.testing.expectEqual(Kind.pane, carried.kind); try std.testing.expectEqual(@as(usize, 1), carried.panes.len); const pane = try walk.load(pardes.Pane, arena.allocator(), carried.panes[0], null); diff --git a/src/builtins.zig b/src/builtins.zig index de69c545..2b7a9c1e 100644 --- a/src/builtins.zig +++ b/src/builtins.zig @@ -705,31 +705,48 @@ pub const Kill = struct { } }; -/// Write the session's panes and layout to a dump file, for Restore. +/// Write the session's panes and layout to the state file, for Restore; +/// `Dump export <path>` writes them to a file of their own instead. pub const Dump = struct { pub const scope: Scope = .session; + pub const takes_arg = true; pub fn run(c: Ctx) void { + var to: ?[]const u8 = null; + if (c.arg) |arg| { + const trimmed = std.mem.trim(u8, arg, " \t"); + if (!std.mem.startsWith(u8, trimmed, "export ")) return c.p.reportFailure(c.id, "Dump: takes export <path>, or nothing"); + const path = std.mem.trim(u8, trimmed["export".len..], " \t"); + if (path.len == 0) return c.p.reportFailure(c.id, "Dump: takes export <path>, or nothing"); + to = path; + } // Never a Dump said done that wrote nothing: its failure is said. - dump.dumpState(c.p) catch |err| c.p.reportError(c.id, "Dump", err); + dump.dumpTo(c.p, to) catch |err| c.p.reportError(c.id, "Dump", err); } }; -/// Replace every pane with those of a dump file, the last one Dump wrote when bare. +/// Replace every pane with those of a dump: the state file's of that label, +/// or a file's; bare, the last one this session made, else the newest. pub const Restore = struct { pub const scope: Scope = .session; pub const takes_arg = true; pub fn run(c: Ctx) void { - const typed = c.arg orelse (c.p.last_dump orelse return c.p.reportFailure(c.id, "Restore: no dump yet; name one")); - // The file it reads, whole (`~`, the DumpDir, the directory it runs - // in): what the log's `restore` record and every message name. + const typed: ?[]const u8 = c.arg orelse c.p.last_dump; + // A file it names, its whole path (`~`, the DumpDir, the directory + // it runs in): what the log's `restore` record and every message + // name. Else a label of the state file's. var full_buf: [4096]u8 = undefined; - const path = @import("fs.zig").restorePath(typed, c.p.settings.dump_dir.get(), &full_buf); - if (path.len > c.p.restore_buf.len) return c.p.reportFailure(c.id, "Restore: that name is too long"); - // A dump that cannot be read is said now, and fails a ctl write: the - // host reads it only after this step, where nobody hears. First, - // so a Restore that could never happen spends no warning. - { - const bytes = @import("fs.zig").readRestore(c.p.gpa, path, c.p.settings.dump_dir.get()) catch |err| { + var label_buf: [256]u8 = undefined; + const file = if (typed) |t| dump.restoreFile(t, c.p.settings.dump_dir.get(), &full_buf) else null; + const name = if (file) |path| path else pardes.tape.authority(pardes.Db.labelled, .{ &c.p.db, c.p, typed, @as([]u8, &label_buf) }) orelse { + var buf: [320]u8 = undefined; + return c.p.reportFailure(c.id, if (typed) |t| std.fmt.bufPrint(&buf, "Restore: no dump labelled {s}", .{t[0..@min(t.len, 256)]}) catch "Restore: no dump of that label" else "Restore: no dump yet; name one"); + }; + if (name.len > c.p.restore_buf.len) return c.p.reportFailure(c.id, "Restore: that name is too long"); + // A file that cannot be read is said now, and fails a ctl write: + // the host reads it only after this step, where nobody hears. + // First, so a Restore that could never happen spends no warning. + if (file) |path| { + const bytes = @import("fs.zig").readFile(c.p.gpa, path) catch |err| { var buf: [limits.host_path_cap + 64]u8 = undefined; var words: [128]u8 = undefined; const why = switch (err) { @@ -752,8 +769,8 @@ pub const Restore = struct { // acme's Load adds a dump's windows to the ones there; a Restore // replaces them all, so it asks what Exit asks first. if (warnModifiedIn(c, .Restore, .all)) return; - @memcpy(c.p.restore_buf[0..path.len], path); - c.p.restore_req = c.p.restore_buf[0..path.len]; + @memcpy(c.p.restore_buf[0..name.len], name); + c.p.restore_req = c.p.restore_buf[0..name.len]; } }; diff --git a/src/config.zig b/src/config.zig index abc901d6..f3b9cc4f 100644 --- a/src/config.zig +++ b/src/config.zig @@ -933,7 +933,7 @@ pub const Runtime = struct { .{ .word = "InactiveDim", .action = .inactive_dim, .doc = "Fade the unfocused panes' text by a percentage, 0 through 100, 0 off." }, .{ .word = "Pet", .action = .{ .choice = .pet }, .doc = "The window's companion in the workspace tag's blank space: off, cat or frog." }, .{ .word = "MessageLinger", .action = .message_linger, .doc = "How long, in milliseconds, a message stays after what dismisses it." }, - .{ .word = "DumpDir", .action = .dump_dir, .doc = "The directory Dump writes to and a relative Restore reads from; bare, the default." }, + .{ .word = "DumpDir", .action = .dump_dir, .doc = "The directory of the state file Dump writes to and Recent is kept in, where a relative Restore name is looked for too; bare, the default." }, .{ .word = "Theme", .action = .theme, .doc = "Switch to the named theme." }, .{ .word = "Shell", .action = .shell, .doc = "The shell new terminals run; bare, $SHELL if it is executable, else /bin/sh." }, .{ .word = "Font", .action = .font, .doc = "The body font, by name or path, and its size." }, diff --git a/src/dump.zig b/src/dump.zig index 3ded45ca..8847152d 100644 --- a/src/dump.zig +++ b/src/dump.zig @@ -20,64 +20,29 @@ const Options = pardes.Options; const Pardes = pardes.Pardes; const views = pardes.views; -/// Where the next dump goes. $PARDES_DUMP wins verbatim (the snapshot harness -/// pins it for deterministic goldens); else $XDG_DATA_HOME|~/.local/share -/// /pardes/pardes-<utc>.dump — timestamped so dumps never overwrite each other. -/// Creates the directory and any parent missing (a fresh HOME has no -/// ~/.local/share); a failure surfaces at open. -pub fn outPath(buf: *[1024:0]u8, dir_setting: []const u8) ?[:0]const u8 { - if (std.c.getenv("PARDES_DUMP")) |p| - return std.fmt.bufPrintSentinel(buf, "{s}", .{std.mem.span(p)}, 0) catch null; - var dir_buf: [900]u8 = undefined; - const dir = directory(&dir_buf, dir_setting) orelse return null; - _ = std.Io.Dir.cwd().createDirPathStatus(std.Io.Threaded.global_single_threaded.io(), dir, .fromMode(0o755)) catch {}; +/// A Dump's label in the state file (Db.zig): `pardes-<date>-<time>`, UTC +/// (another in the same second is `-2` and on there), in `buf`. +pub fn label(buf: []u8) []const u8 { var ts: std.c.timespec = undefined; _ = std.c.clock_gettime(.REALTIME, &ts); const es: std.time.epoch.EpochSeconds = .{ .secs = @intCast(@max(0, ts.sec)) }; const yd = es.getEpochDay().calculateYearDay(); const md = yd.calculateMonthDay(); const ds = es.getDaySeconds(); - var stamp: [32]u8 = undefined; - const base = std.fmt.bufPrint(&stamp, "pardes-{d:0>4}{d:0>2}{d:0>2}-{d:0>2}{d:0>2}{d:0>2}", .{ + return std.fmt.bufPrint(buf, "pardes-{d:0>4}{d:0>2}{d:0>2}-{d:0>2}{d:0>2}{d:0>2}", .{ yd.year, md.month.numeric(), @as(u8, md.day_index) + 1, ds.getHoursIntoDay(), ds.getMinutesIntoHour(), ds.getSecondsIntoMinute(), - }) catch return null; - return uniqueIn(buf, dir, base); + }) catch "pardes-dump"; } -/// `<dir>/<base>.dump`, or `<base>-2.dump` and on when two dumps land in one -/// second: a dump never overwrites another. -fn uniqueIn(buf: *[1024:0]u8, dir: []const u8, base: []const u8) ?[:0]const u8 { - var n: usize = 1; - while (n < 1000) : (n += 1) { - const path = (if (n == 1) - std.fmt.bufPrintSentinel(buf, "{s}/{s}.dump", .{ dir, base }, 0) - else - std.fmt.bufPrintSentinel(buf, "{s}/{s}-{d}.dump", .{ dir, base, n }, 0)) catch return null; - if (std.c.access(path.ptr, std.c.F_OK) != 0) return path; - } - return null; -} - -test "two dumps in one second get names of their own" { - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var dir_buf: [900]u8 = undefined; - const dir = dir_buf[0..try tmp.dir.realPath(std.testing.io, &dir_buf)]; - var buf: [1024:0]u8 = undefined; - const first = try std.testing.allocator.dupe(u8, uniqueIn(&buf, dir, "pardes-x").?); - defer std.testing.allocator.free(first); - try std.testing.expect(std.mem.endsWith(u8, first, "/pardes-x.dump")); - try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "pardes-x.dump", .data = "" }); - try std.testing.expect(std.mem.endsWith(u8, uniqueIn(&buf, dir, "pardes-x").?, "/pardes-x-2.dump")); -} /// The host's environment. fn hostEnv(name: [*:0]const u8) ?[*:0]u8 { return std.c.getenv(name); } -/// Where dumps go and relative Restore names are looked up: the `DumpDir` -/// setting when set (a leading `~/` is the home directory), else the default. +/// Where the state file is (Db.zig) and relative Restore names are looked +/// up: the `DumpDir` setting when set (a leading `~/` is the home +/// directory), else the default. pub fn directory(buf: []u8, dir_setting: []const u8) ?[]const u8 { if (dir_setting.len == 0) return defaultDirectory(buf); var joined: [4096]u8 = undefined; @@ -92,21 +57,6 @@ pub fn directory(buf: []u8, dir_setting: []const u8) ?[]const u8 { return std.fmt.bufPrint(buf, "{s}", .{resolved}) catch null; } -test "a dump makes every directory missing on the way to its DumpDir" { - if (std.c.getenv("PARDES_DUMP") != null) return error.SkipZigTest; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - var at: [4096]u8 = undefined; - const base = at[0..try tmp.dir.realPath(std.testing.io, &at)]; - var setting: [4200]u8 = undefined; - const dir = try std.fmt.bufPrint(&setting, "{s}/fresh/.local/share/pardes", .{base}); - var out: [1024:0]u8 = undefined; - const path = outPath(&out, dir).?; - try std.testing.expect(std.mem.startsWith(u8, path, dir)); - var made = try tmp.dir.openDir(std.testing.io, "fresh/.local/share/pardes", .{}); - made.close(std.testing.io); -} - test "a DumpDir reads back absolute, however it was written" { var buf: [4096]u8 = undefined; try std.testing.expectEqualStrings("/tmp/dumps", directory(&buf, "/tmp/dumps/").?); @@ -130,11 +80,15 @@ pub var terminal_gpa: ?std.mem.Allocator = null; // ---- a whole session to and from a dump (acme keeps this in rows.c) ---- // -// A dump (src/walk.zig) is everything the session is but what is the -// host's, the frame's or worked out again: its panes with their text and -// undo, terminals with their screens, history and kitty images (ghostty's -// snapshot), registers, places, tags, layout and settings. It crosses -// builds: its fields are matched by name. +// A dump is everything the session is but what is the host's, the frame's +// or worked out again: its panes with their text and undo, terminals with +// their screens, history and kitty images (ghostty's snapshot), registers, +// places, tags, layout and settings. It crosses builds: its fields are +// matched by name. It is an image (Db.zig): a database in memory the walk +// writes by name (walk.Put), which the state file keeps under a label, +// `Dump export` and a handover write as a file, and `-l` and Restore read. +// The byte dump before it (walk.Dump) is still a replica's join, a closed +// pane's, and for one release (0.24) what an old `.dump` file is read as. // // Loading is two halves. `load` makes the core the dump was and nothing // else (a replica joining a session takes that, phase 5); `revivePane` and @@ -166,11 +120,92 @@ pub fn dumpState(p: *Pardes) !void { p.emit(.write_dump); } -/// The session as a dump, gpa-owned. +/// A Dump: into the state file, or (`Dump export`) a file of its own at +/// `to` (`~` the home directory). +pub fn dumpTo(p: *Pardes, to: ?[]const u8) !void { + if (p.dump_export) |old| p.gpa.free(old); + p.dump_export = null; + if (to) |path| { + var home_buf: [4096]u8 = undefined; + p.dump_export = try p.gpa.dupe(u8, filesystem.expandHome(path, &home_buf)); + } + try dumpState(p); +} + +/// The file a Restore of `name` reads, its whole path in `out`: one named +/// with a `/` or `~/` (as it resolves, else as typed: the read says why), +/// or a bare word that names a file in the DumpDir or the directory pardes +/// runs in; null for one that names none: a label of the state file's. +pub fn restoreFile(name: []const u8, dir_setting: []const u8, out: *[4096]u8) ?[]const u8 { + const path = filesystem.restorePath(name, dir_setting, out); + if (std.mem.indexOfScalar(u8, name, '/') != null or std.mem.startsWith(u8, name, "~")) return path; + return if (std.fs.path.isAbsolute(path)) path else null; +} + +/// What a Restore of `name` reads, `gpa`'s: the file it names (an image, +/// or an old byte dump), else the state file's dump of that label. +pub fn restoreBytes(p: *Pardes, gpa: std.mem.Allocator, name: []const u8) ![]u8 { + var buf: [4096]u8 = undefined; + if (restoreFile(name, p.settings.dump_dir.get(), &buf)) |path| return filesystem.readFile(gpa, path); + const got = p.db.getDump(p, gpa, name) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.BadDump; + return got orelse error.FileNotFound; +} + +/// The session as a dump's image, gpa-owned (the host puts it in the state +/// file under a label of its own). pub fn dumpBytes(p: *Pardes) ![]u8 { + return image(p, "dump"); +} + +/// The session as a byte dump (walk.Dump), gpa-owned: what a replica's join +/// streams, its blobs in place. +pub fn byteDump(p: *Pardes) ![]u8 { return dumpWith(p, null); } +/// The session as the image of one dump labelled `name` (Db.zig), its +/// roots by name: the core, its 9P mounts, which of its panes' files were +/// on disk, its theme's name and the views not live (see `Session`). +pub fn image(p: *Pardes, name: []const u8) ![]u8 { + const gpa = p.gpa; + var buf: std.Io.Writer.Allocating = .init(gpa); + defer buf.deinit(); + var db = try walk.Mem.init(std.Io.failing, .{ .buffer = &buf, .file_id = 0 }); + var w: walk.Put = .{ .gpa = gpa }; + defer w.deinit(); + const mounts: []const filesystem.Mount = p.fs.mounts.items; + var on_disk: OnDisk = @splat(false); + var panes: u64 = 0; + for (p.panes, &on_disk) |slot, *on| if (slot) |pane| { + on.* = onDisk(pane); + panes += 1; + }; + const theme: []const u8 = colors.themes[@min(p.settings.theme, colors.themes.len - 1)].name; + const others = try views.dumped(p, gpa); + defer { + for (others) |o| gpa.free(o.panes); + gpa.free(others); + } + const kept: []const views.Dumped = others; + try db.runContext(struct { + w: *walk.Put, + p: *Pardes, + mounts: *const []const filesystem.Mount, + on_disk: *const OnDisk, + theme: *const []const u8, + others: *const []const views.Dumped, + pub fn run(ctx: @This(), cursor: *walk.Mem.Cursor(.read_write)) !void { + const root = try walk.Mem.HashMap(.read_write).init(cursor.*); + try ctx.w.member(root, "core", Pardes, ctx.p); + try ctx.w.member(root, "mounts", []const filesystem.Mount, ctx.mounts); + try ctx.w.member(root, "on_disk", OnDisk, ctx.on_disk); + try ctx.w.member(root, "theme", []const u8, ctx.theme); + try ctx.w.member(root, "views", []const views.Dumped, ctx.others); + } + }{ .w = &w, .p = p, .mounts = &mounts, .on_disk = &on_disk, .theme = &theme, .others = &kept }); + return pardes.Db.image(gpa, name, buf.written(), w.blobs.items, panes); +} + /// A dump for a screen-state replica's join (src/replica.zig): its blobs' /// bytes left out and noted in `deferred` (they are the session's, alive /// while pinned: memory.zig `pin`), its terminals' emulators left out (the @@ -213,6 +248,19 @@ fn dumpWith(p: *Pardes, deferred: ?*std.ArrayListUnmanaged(walk.Deferred)) ![]u8 /// Restore asks before it warns about unsaved text or replaces anything. /// Why not is left in `why`, ended by a 0. pub fn check(gpa: std.mem.Allocator, bytes: []const u8, why: []u8) !void { + if (pardes.Db.isImage(bytes)) { + const r = pardes.Db.Read.open(gpa, bytes, null) catch |err| { + const said = std.fmt.bufPrint(why[0 .. why.len - 1], "{s}", .{switch (err) { + error.NoDump => "it holds no dump", + error.OutOfMemory => "out of memory", + else => "it is no dump of pardes's", + }}) catch ""; + why[said.len] = 0; + return err; + }; + r.close(); + return; + } var l = SessionLoad.init(gpa, bytes) catch |err| { const said = std.fmt.bufPrint(why[0 .. why.len - 1], "{s}", .{switch (err) { error.BadDump => "it does not begin as one", @@ -235,6 +283,7 @@ pub fn load(gpa: std.mem.Allocator, opts: Options, bytes: []const u8, notes: ?*s /// `load`, with which panes' files were on disk when it was dumped. fn loadKnowing(gpa: std.mem.Allocator, opts: Options, bytes: []const u8, notes: ?*std.ArrayList(u8), on_disk: ?*OnDisk) !*Pardes { + if (pardes.Db.isImage(bytes)) return loadImage(gpa, opts, bytes, notes, on_disk); const p = try Pardes.create(gpa, opts); var l = SessionLoad.init(gpa, bytes) catch |err| { p.deinit(); @@ -304,6 +353,70 @@ fn loadKnowing(gpa: std.mem.Allocator, opts: Options, bytes: []const u8, notes: return p; } +/// `loadKnowing` of an image (Db.zig): its roots read by name (walk.Get), +/// as the byte dump's are read in order. +fn loadImage(gpa: std.mem.Allocator, opts: Options, bytes: []const u8, notes: ?*std.ArrayList(u8), on_disk: ?*OnDisk) !*Pardes { + const r = pardes.Db.Read.open(gpa, bytes, null) catch |err| return if (err == error.OutOfMemory) err else error.BadDump; + defer r.close(); + const root = r.root(); + const core_at = (try walk.Get.member(root, "core")) orelse return error.BadDump; + const p = try Pardes.create(gpa, opts); + var l: walk.Get = .init(gpa); + defer l.deinit(); + l.blobs = r.blobs() catch null; + // A load that fails leaves nothing: what it read is freed, and the + // core with it, its fields never torn down one by one. + errdefer { + l.rollback(); + p.fs.deinit(gpa); + gpa.destroy(p); + } + l.allocs = .{ .image = p.image_gpa, .pdf = p.pdf_gpa, .terminal = terminal_gpa }; + try l.into(Pardes, core_at, p); + const read = l.made.items.len; + const mounts: []const filesystem.Mount = if (try walk.Get.member(root, "mounts")) |c| try l.get([]const filesystem.Mount, c) else &.{}; + defer { + for (mounts) |m| { + gpa.free(m.name); + gpa.free(m.dial); + } + gpa.free(mounts); + // Freed here: no rollback of them. + l.made.shrinkRetainingCapacity(read); + } + const was: OnDisk = if (try walk.Get.member(root, "on_disk")) |c| try l.get(OnDisk, c) else @splat(false); + if (on_disk) |out| out.* = was; + const theme: []const u8 = if (try walk.Get.member(root, "theme")) |c| try l.get([]const u8, c) else ""; + defer gpa.free(theme); + p.settings.theme = for (colors.themes, 0..) |t, i| { + if (std.mem.eql(u8, t.name, theme)) break i; + } else if (p.settings.theme < colors.themes.len) p.settings.theme else 0; + for (mounts) |m| { + const existing = for (p.fs.mounts.items) |e| { + if (std.mem.eql(u8, e.name, m.name)) break e; + } else null; + if (existing) |e| { + if (!std.mem.eql(u8, e.dial, m.dial)) return error.MountConflict; + continue; + } + p.fs.mount(gpa, m.name, m.dial) catch |err| l.note("mount {s}: {t}", .{ m.name, err }); + } + if (l.notes.items.len > 0) { + dropMissing(p); + for (p.panes) |slot| if (slot) |pane| try ensureKind(p, pane); + if (notes) |n| try n.appendSlice(gpa, l.notes.items); + } + // The views not live, last: what it moves into the core is never + // rolled back from under it, nothing after it failing. + const at = l.made.items.len; + const others: []const views.Dumped = if (try walk.Get.member(root, "views")) |c| try l.get([]const views.Dumped, c) else &.{}; + try views.undumped(p, others); + for (others) |o| gpa.free(o.panes); + gpa.free(others); + l.made.shrinkRetainingCapacity(at); + return p; +} + /// A pane that is none of its kinds (a terminal kept slim, or one another /// build's dump could not give): a terminal, a new emulator for its shell. fn ensureKind(p: *Pardes, pane: *pardes.panes.Pane) !void { @@ -349,7 +462,7 @@ pub fn restore(p: *Pardes, bytes: []const u8, from: []const u8) !*Pardes { pub fn initFromDump(gpa: std.mem.Allocator, opts: Options, bytes: []const u8) !*Pardes { pardes.tape.startFromEnv(); const taped = pardes.tape.enter(null, .{ .init_from_dump = .{ .opts = opts, .bytes = bytes } }); - const p = (if (std.mem.startsWith(u8, bytes, Handover.carried_magic)) carriedWindow(gpa, opts, bytes) else loadAndRevive(gpa, opts, bytes, null, null)) catch |err| { + const p = (if (Handover.carries(bytes)) carriedWindow(gpa, opts, bytes) else loadAndRevive(gpa, opts, bytes, null, null)) catch |err| { pardes.tape.leave(null, taped); return err; }; @@ -375,16 +488,6 @@ pub fn resumeFromDump(gpa: std.mem.Allocator, opts: Options, bytes: []const u8, return loadAndRevive(gpa, opts, bytes, null, kept); } -/// Where an upgrade leaves its dump: the DumpDir (a relative Restore finds -/// it there by name), as `pardes-upgrade-<pid>.dump`; never $PARDES_DUMP, -/// which is a Dump's. -pub fn upgradePath(buf: *[1024:0]u8, dir_setting: []const u8) ?[:0]const u8 { - var dir_buf: [900]u8 = undefined; - const dir = directory(&dir_buf, dir_setting) orelse return null; - _ = std.Io.Dir.cwd().createDirPathStatus(std.Io.Threaded.global_single_threaded.io(), dir, .fromMode(0o755)) catch {}; - return std.fmt.bufPrintSentinel(buf, "{s}/pardes-upgrade-{d}.dump", .{ dir, std.c.getpid() }, 0) catch null; -} - fn loadAndRevive(gpa: std.mem.Allocator, opts: Options, bytes: []const u8, previous: ?*Pardes, kept: ?*const [MAX_PANES]bool) !*Pardes { var notes: std.ArrayList(u8) = .empty; defer notes.deinit(gpa); @@ -711,7 +814,7 @@ test "an upgraded session's dump keeps its views and its running terminal as the try std.testing.expect(r.panes[0].?.terminal.?.vt.screens.active_key == .primary); } -test "a session read by name, field by field as another build reads it, is the same session" { +test "a session's image, read by name as every load reads it, is the same session, and dumps to the same bytes again" { const gpa = std.testing.allocator; const p = try Pardes.init(gpa, .{ .tty_only = true }); defer p.deinit(); @@ -720,6 +823,26 @@ test "a session read by name, field by field as another build reads it, is the s try std.testing.expect(p.executeBuiltinLine(p.active, "Zerox")); const bytes = try dumpBytes(p); defer gpa.free(bytes); + var notes: std.ArrayList(u8) = .empty; + defer notes.deinit(gpa); + const q = try load(gpa, p.opts, bytes, ¬es); + defer q.deinit(); + try std.testing.expectEqualStrings("", notes.items); + try std.testing.expectEqual(walk.digest(Pardes, p), walk.digest(Pardes, q)); + const again = try dumpBytes(q); + defer gpa.free(again); + try std.testing.expectEqualSlices(u8, bytes, again); +} + +test "a session's byte dump read by name, field by field as another build reads it, is the same session" { + const gpa = std.testing.allocator; + const p = try Pardes.init(gpa, .{ .tty_only = true }); + defer p.deinit(); + const pane = try p.setTestFile("one\ntwo\n"); + _ = ctlfs.pane.spliceBody(p, pane, 0, 0, "unsaved ").?; + try std.testing.expect(p.executeBuiltinLine(p.active, "Zerox")); + const bytes = try byteDump(p); + defer gpa.free(bytes); const q = try Pardes.create(gpa, p.opts); defer q.deinit(); var l = try SessionLoad.init(gpa, bytes); diff --git a/src/fs-help.txt b/src/fs-help.txt index 35594e6d..d0c27c42 100644 --- a/src/fs-help.txt +++ b/src/fs-help.txt @@ -13,9 +13,10 @@ sessions the pardes daemon's, as pardes --ps: daemon <pid>; session <pid> tty focus the serial of the pane with the keyboard (empty on a column tag); write one to give it the keyboard kbdin write runes: keys typed at the pane with the keyboard, done before the write answers (\x1b Esc, \n Enter) mousein write `m col row buttons` (1 left, 2 middle, 4 right): the mouse as it now is; m 10 5 4, m 10 5 0 is a right click -ctl settings a line; or Newcol, Dump, Restore [f], Exit (refuses once over unsaved panes), Kill [w], size C R +ctl settings a line; or Newcol, Dump [export f], Restore [label|f], Exit (refuses once over unsaved panes), Kill [w], size C R commands every builtin: word, arg, root|pane|both (the ctl that takes it), -- what it does recent files opened lately, closed ones too: open|closed <path> +dumps the dumps the state file keeps, the newest first: <label> <date> <size> <panes> layout a line per column: serial index x width current|notcurrent empty|full pane-serials; then active <serial> tag the workspace tag, tagexec a word as a click in it; col/<serial>/ a column's tag, ctl, exec; rmdir: empty only pane/new open it to make a pane (<session dir>/+New) in the active column; the read answers its serial diff --git a/src/host_io.zig b/src/host_io.zig index 0c638f71..1a328ad5 100644 --- a/src/host_io.zig +++ b/src/host_io.zig @@ -106,7 +106,7 @@ pub fn stopWorkers() void { /// window): the others are let go, as a detach of each. pub fn restoreCore(gpa: std.mem.Allocator, core: *pardes.Pardes, path: []const u8, peers: *replica.Peers, inotify_fd: c_int, watches: *file_watch.Table) ?*pardes.Pardes { peers.dropAll(core); - const bytes = filesystem.readRestore(gpa, path, core.settings.dump_dir.get()) catch |err| { + const bytes = pardes.dump.restoreBytes(core, gpa, path) catch |err| { core.reportError(core.active, "Restore", err); return null; }; @@ -3509,12 +3509,31 @@ pub fn writeFile(core: *pardes.Pardes, watches: *file_watch.Table, pane: u8, pat pardes.exec.reapplyInitFile(core, path, bytes); } -/// A Dump's write, into the DumpDir. +/// A Dump's write: into the state file under a label of its own +/// (Db.putDump), or as a file of its own where `Dump export` names one or +/// $PARDES_DUMP does (verbatim: the snapshot harness pins it for its +/// goldens). pub fn writeDump(core: *pardes.Pardes, bytes: []const u8) void { - var pbuf: [1024:0]u8 = undefined; - const path = pardes.dump.outPath(&pbuf, core.settings.dump_dir.get()) orelse return core.dumpFailed(core.settings.dump_dir.get(), error.NoDumpDirectory); - filesystem.write(core, path, bytes) catch |err| return core.dumpFailed(path, err); - core.setLastDump(path); + const to: ?[]const u8 = core.dump_export orelse if (std.c.getenv("PARDES_DUMP")) |p| std.mem.span(p) else null; + if (to) |path| { + filesystem.write(core, path, bytes) catch |err| return core.dumpFailed(path, err); + return core.setLastDump(path); + } + var stamp: [64]u8 = undefined; + var buf: [128]u8 = undefined; + const label = core.db.putDump(core, bytes, pardes.dump.label(&stamp), &buf) catch |err| { + var path: [4200]u8 = undefined; + // In the words a host reports failures in (tape.Failure). + const said: pardes.tape.Failure = switch (err) { + error.NoStateFile => error.NoDumpDirectory, + error.OutOfMemory => error.OutOfMemory, + error.NoSpaceLeft => error.NoSpaceLeft, + error.AccessDenied => error.AccessDenied, + else => error.InputOutput, + }; + return core.dumpFailed(pardes.Db.statePath(&path, core.settings.dump_dir.get()) orelse core.settings.dump_dir.get(), said); + }; + core.setLastDump(label); } /// DumpThemes: the built-in themes written below the config directory. diff --git a/src/macos.zig b/src/macos.zig index 147930c0..505aeb7f 100644 --- a/src/macos.zig +++ b/src/macos.zig @@ -1144,7 +1144,7 @@ export fn pardes_tick() bool { fn restoreCore(st: *State) bool { if (st.core.quit) return false; const path = st.core.takeRestore() orelse return false; - const bytes = filesystem.readRestore(st.gpa, path, st.core.settings.dump_dir.get()) catch |err| { + const bytes = dump.restoreBytes(st.core, st.gpa, path) catch |err| { st.core.reportError(st.core.active, "Restore", err); return false; }; @@ -2214,11 +2214,7 @@ fn writeFile(ctx: ?*anyopaque, pane: u8, path: []const u8, bytes: []const u8) vo } fn writeDump(ctx: ?*anyopaque, bytes: []const u8) void { - const st = hostState(ctx); - var pbuf: [1024:0]u8 = undefined; - const path = pardes.dump.outPath(&pbuf, st.core.settings.dump_dir.get()) orelse return st.core.dumpFailed(st.core.settings.dump_dir.get(), error.NoDumpDirectory); - filesystem.write(st.core, path, bytes) catch |err| return st.core.dumpFailed(path, err); - st.core.setLastDump(path); + host_io.writeDump(hostState(ctx).core, bytes); } fn watchFile(ctx: ?*anyopaque, pane: u8, path: []const u8, on: bool, mode: pardes.WatchMode) void { diff --git a/src/ninep/tree.zig b/src/ninep/tree.zig index 690d1d1f..707ce40a 100644 --- a/src/ninep/tree.zig +++ b/src/ninep/tree.zig @@ -260,6 +260,10 @@ pub const TopFile = enum(u5) { kbdin, /// devmouse's mousein: `m col row buttons`, the mouse as it now is. mousein, + /// The dumps the state file keeps (src/Db.zig), a line each, the newest + /// first: `<label> <date> <size> <panes>`; `Restore <label>` brings one + /// back. + dumps, pub fn fileName(f: TopFile) []const u8 { return if (f == .root) "/" else @tagName(f); @@ -269,7 +273,7 @@ pub const TopFile = enum(u5) { return switch (f) { .root, .pane, .col => 0o755, .look, .exec, .log, .focus, .ctl, .tag, .tagexec, .pager => 0o666, - .README, .index, .status, .screen, .listeners, .new, .commands, .layout, .recent, .sessions => 0o444, + .README, .index, .status, .screen, .listeners, .new, .commands, .layout, .recent, .dumps, .sessions => 0o444, .kbdin, .mousein => 0o222, }; } @@ -670,7 +674,7 @@ fn topSize(p: *Pardes, f: TopFile) u64 { // frame, /pane/new until an open has a pane to name, and the index, // layout, recent list and listeners, which a stat would otherwise // have to build to count. - .root, .pane, .col, .screen, .new, .layout, .index, .recent, .listeners, .sessions, .kbdin, .mousein => 0, + .root, .pane, .col, .screen, .new, .layout, .index, .recent, .dumps, .listeners, .sessions, .kbdin, .mousein => 0, .tag => cols.headerText(p, null).?.len, .README => fs.help.len, .status => ctl.statusText(p).len, @@ -754,7 +758,7 @@ fn readdir(p: *Pardes, req: Req, target: Target) Reply { .root => { // The guide, then what describes the session, then the // files that act, then the directories. - inline for (.{ TopFile.README, TopFile.index, TopFile.status, TopFile.focus, TopFile.ctl, TopFile.commands, TopFile.recent, TopFile.look, TopFile.exec, TopFile.pager, TopFile.log, TopFile.screen, TopFile.listeners, TopFile.sessions, TopFile.layout, TopFile.tag, TopFile.tagexec, TopFile.kbdin, TopFile.mousein, TopFile.pane, TopFile.col }) |t| { + inline for (.{ TopFile.README, TopFile.index, TopFile.status, TopFile.focus, TopFile.ctl, TopFile.commands, TopFile.recent, TopFile.dumps, TopFile.look, TopFile.exec, TopFile.pager, TopFile.log, TopFile.screen, TopFile.listeners, TopFile.sessions, TopFile.layout, TopFile.tag, TopFile.tagexec, TopFile.kbdin, TopFile.mousein, TopFile.pane, TopFile.col }) |t| { if (skip > 0) skip -= 1 else stageDirent(out, p.gpa, @backingInt(t), t.dir(), t.fileName()); } if (skip > 0) skip -= 1 else stageDirent(out, p.gpa, fs.os_root, true, "os"); @@ -842,7 +846,7 @@ fn open(p: *Pardes, req: Req, target: Target) Reply { }, .screen => .{ .snapshot = null }, .log => .{ .log = .{} }, - .index, .layout, .recent, .commands, .status, .listeners, .sessions => .{ .frozen = &.{} }, + .index, .layout, .recent, .dumps, .commands, .status, .listeners, .sessions => .{ .frozen = &.{} }, // The root's ctl opened only to read: its settings, frozen. .ctl => if (linesOpen(target, req)) .lines else .{ .frozen = &.{} }, else => if (linesOpen(target, req)) .lines else return .{ .tag = req.tag }, @@ -1102,6 +1106,16 @@ fn readFile(p: *Pardes, req: Req, target: Target) Reply { pardes.panes.Output.recentText(p, &text.writer, false) catch break :recent Reply.fail(req.tag, E.NOMEM); break :recent window(p, req, text.written()); }, + .dumps => dumps: { + // The state file read now, unless a step is out mid-syscall + // (its database is the step's): then none. + var text: std.Io.Writer.Allocating = .init(p.scratch.allocator()); + if (pardes.turn.out == 0) p.db.listDumps(p, &text.writer) catch |err| switch (err) { + error.OutOfMemory, error.WriteFailed => break :dumps Reply.fail(req.tag, E.NOMEM), + else => break :dumps Reply.fail(req.tag, E.IO), + }; + break :dumps window(p, req, text.written()); + }, .layout => layout: { var text: std.Io.Writer.Allocating = .init(p.scratch.allocator()); cols.writeLayout(p, &text.writer) catch break :layout Reply.fail(req.tag, E.NOMEM); @@ -1522,13 +1536,13 @@ test "readdir lists the root and a pane directory without creating anything" { @memcpy(top_bytes[0..top_reply.bytes.len], top_reply.bytes); const top = th.dirents(top_bytes[0..top_reply.bytes.len], &buf); try testing.expect(top.len >= 10); - for ([_][]const u8{ "README", "index", "status", "focus", "ctl", "commands", "recent", "look", "exec", "pager", "log", "screen", "listeners", "sessions", "layout", "tag", "tagexec", "kbdin", "mousein", "pane", "col", "os" }, 0..) |name, i| { + for ([_][]const u8{ "README", "index", "status", "focus", "ctl", "commands", "recent", "dumps", "look", "exec", "pager", "log", "screen", "listeners", "sessions", "layout", "tag", "tagexec", "kbdin", "mousein", "pane", "col", "os" }, 0..) |name, i| { try testing.expectEqualStrings(name, top[i].name); - try testing.expectEqual(i >= 19, top[i].dir); + try testing.expectEqual(i >= 20, top[i].dir); } - try testing.expectEqual(@backingInt(TopFile.pane), top[19].node); - try testing.expectEqual(@backingInt(TopFile.col), top[20].node); - try testing.expectEqual(fs.os_root, top[21].node); + try testing.expectEqual(@backingInt(TopFile.pane), top[20].node); + try testing.expectEqual(@backingInt(TopFile.col), top[21].node); + try testing.expectEqual(fs.os_root, top[22].node); var idbuf: [16]u8 = undefined; try testing.expect(th.nameAt(top, try std.fmt.bufPrint(&idbuf, "{d}", .{serial})) == null); try testing.expect(th.nameAt(top, "self") == null); diff --git a/src/pardes.zig b/src/pardes.zig index b42bf643..571f7db4 100644 --- a/src/pardes.zig +++ b/src/pardes.zig @@ -19,7 +19,7 @@ pub const tagline = @import("tagline.zig"); pub const exec = @import("exec.zig"); const recent_files = @import("recent.zig"); const Store = @import("Store.zig"); -const Db = @import("Db.zig"); +pub const Db = @import("Db.zig"); pub const mouse = @import("mouse.zig"); pub const normal = @import("normal.zig"); pub const edit = @import("edit.zig"); @@ -3540,6 +3540,9 @@ pub const Pardes = struct { clip_pending: ?edit.ClipRequest = null, /// the last serialized dump (gpa-owned), read by the write_dump effect dump_out: ?[]u8 = null, + /// Where the last Dump goes as a file of its own (`Dump export`; gpa- + /// owned), else the state file. + dump_export: ?[]u8 = null, /// where the shell wrote the last dump (shell reports back after /// write_dump); shown in the topbar as `Restore <path>` last_dump: ?[]const u8 = null, @@ -3602,7 +3605,7 @@ pub const Pardes = struct { "parked", "undrawn", // How this process was started, and the last dump it made for // its host to write: a session loaded elsewhere has its own. - "opts", "dump_out", + "opts", "dump_out", "dump_export", }, .derived = .{ "shell_rows", "diff_output", "locations_cache", "retired_panes", "tty_filter_palette", "body_chrome" }, // The host's handles, the 9P server's fids and log, the state file @@ -3906,6 +3909,8 @@ pub const Pardes = struct { // Every file open now is closed with its dot, and the list kept. for (p.panes) |slot| if (slot) |pane| p.noteClosed(pane); p.db.writeRecent(p, true); + // The state file compacted once it is mostly what no moment holds. + p.db.compact(p); p.db.deinit(); p.store.deinit(p.recentGpa()); p.recent.deinit(p.recentGpa()); @@ -3931,6 +3936,7 @@ pub const Pardes = struct { p.macro.deinit(gpa); if (p.retired_edit) |r| gpa.free(r); if (p.dump_out) |d| gpa.free(d); + if (p.dump_export) |d| gpa.free(d); p.fallback.deinit(); if (p.last_dump) |d| gpa.free(d); if (p.custom_theme) |theme_value| gpa.free(theme_value.name); diff --git a/src/replica.zig b/src/replica.zig index 59b0dbb3..5bcc0a53 100644 --- a/src/replica.zig +++ b/src/replica.zig @@ -3599,7 +3599,7 @@ test "a join's dump, its blobs put back where they go, is the session's dump" { _ = try a.local(.{ .key = .{ .cp = pardes.Key.escape, .text = "" } }); const was = Stepping.begin(&a.t, null); defer was.end(); - const whole = try pardes.dump.dumpBytes(a.core); + const whole = try pardes.dump.byteDump(a.core); defer a.core.gpa.free(whole); var blobs: std.ArrayListUnmanaged(walk.Deferred) = .empty; defer blobs.deinit(a.core.gpa); @@ -3762,7 +3762,7 @@ test "a far join is sent each blob once: joined again with them on disk or in th for ("gg70xd80xy") |c| _ = try a.local(.{ .key = .{ .cp = c, .text = &.{c} } }); const was = Stepping.begin(&a.t, null); defer was.end(); - const whole = try pardes.dump.dumpBytes(a.core); + const whole = try pardes.dump.byteDump(a.core); defer a.core.gpa.free(whole); // What the session's list will say: its blobs, those of one text once. var listed: std.ArrayListUnmanaged(walk.Deferred) = .empty; diff --git a/src/upgrade.zig b/src/upgrade.zig index d8f8f3c3..212a8176 100644 --- a/src/upgrade.zig +++ b/src/upgrade.zig @@ -2,15 +2,17 @@ //! session, and the daemon, moved onto the pardes installed since they //! started, in place. //! -//! At a step boundary a session writes its dump (phase 2) and execs the -//! path it was started from. Its pty masters, terminals, stubs and daemon +//! At a step boundary a session commits its dump to the state file +//! (Db.zig, as `upgrade-<pid>`) and execs the path it was started from. Its pty masters, terminals, stubs and daemon //! link stay open across the exec, named in a record on the new command //! line: `--upgraded=<record>` ahead of the arguments it was started with. //! The pid stays, so its shells stay its children; the new process loads //! the dump and takes those fds back (each host's `resume`). A new build //! that cannot load the dump execs the old one again, from the copy this //! process left open (Linux), and with none it starts afresh with the -//! shells adopted, the dump left for a Restore by hand. +//! shells adopted, the dump left for a Restore by hand. A record of an older +//! build names a `.dump` file instead (0.24 reads one, and deletes it once +//! loaded). const std = @import("std"); const builtin = @import("builtin"); const libc = std.c; @@ -114,8 +116,11 @@ pub const Record = struct { nttys: usize = 0, /// The path exec'd, the one a later Upgrade execs again. exe: []const u8 = "", - /// The session's dump. + /// The state file the session's dump is in (an older build's: the + /// `.dump` file it is). dump: []const u8 = "", + /// The dump's label there; none, `dump` is a file of its own. + label: []const u8 = "", /// Why the new build could not load it: a start that went back to the /// old binary says so. refused: []const u8 = "", @@ -158,6 +163,7 @@ pub const Record = struct { for (rec.ptys[0..rec.nptys]) |p| try w.print("pty {d} {d} {d}\n", .{ p.pane, p.pid, p.fd }); for (rec.ttys[0..rec.nttys]) |t| try w.print("tty {d} {d} {d} {d} {x}\n", .{ t.view, t.fd, t.stub, @intFromBool(t.primary), std.mem.asBytes(&t.termios) }); if (rec.refused.len > 0) try w.print("refused {s}\n", .{rec.refused}); + if (rec.label.len > 0) try w.print("label {s}\n", .{rec.label}); try w.print("exe {s}\ndump {s}\n", .{ rec.exe, rec.dump }); } @@ -206,6 +212,8 @@ pub const Record = struct { rec.nttys += 1; } else if (std.mem.eql(u8, word, "refused")) { rec.refused = rest; + } else if (std.mem.eql(u8, word, "label")) { + rec.label = rest; } else if (std.mem.eql(u8, word, "exe")) { rec.exe = rest; } else if (std.mem.eql(u8, word, "dump")) { @@ -333,44 +341,44 @@ pub fn exec(gpa: std.mem.Allocator, path: ?[:0]const u8, old: c_int, rec: *const }; } -/// The dump of a session to `path` (0600, replaced): false when it could -/// not be written whole. -fn writeDump(path: [:0]const u8, bytes: []const u8) bool { - const fd = libc.open(path, .{ .ACCMODE = .WRONLY, .CREAT = true, .TRUNC = true, .CLOEXEC = true }, @as(libc.mode_t, 0o600)); - if (fd < 0) return false; - defer _ = libc.close(fd); - if (@import("host_io.zig").writeFd(fd, bytes)) return true; - _ = libc.unlink(path); - return false; -} +/// The state file's path an upgrade's record names (`dumpInto`'s). +var state_path: [4200]u8 = undefined; -/// An Upgrade's dump, the terminal's and the window's alike: written -/// where a Restore finds it (dump.upgradePath, into `at`) and named in -/// `rec`, the binary it replaces held open for a new build that cannot -/// load it. Null when it is, else why in words (`why`'s). +/// An Upgrade's dump, the terminal's and the window's alike: committed to +/// the state file as `upgrade-<pid>` (its label kept in `at`), where a +/// Restore finds it, and named in `rec`, the binary it replaces held open +/// for a new build that cannot load it. Null when it is, else why in words +/// (`why`'s). pub fn dumpInto(core: *pardes.Pardes, rec: *Record, at: *[1024:0]u8, why: []u8) ?[]const u8 { - const path = pardes.dump.upgradePath(at, core.settings.dump_dir.get()) orelse return "Upgrade: no directory for the session's dump; this session stays as it is"; - const bytes = pardes.dump.dumpBytes(core) catch |err| { + const path = pardes.Db.statePath(&state_path, core.settings.dump_dir.get()) orelse return "Upgrade: no directory for the session's dump; this session stays as it is"; + var name: [64]u8 = undefined; + const label = std.fmt.bufPrint(&name, "upgrade-{d}", .{std.c.getpid()}) catch unreachable; + const bytes = pardes.dump.image(core, label) catch |err| { var words: [128]u8 = undefined; return std.fmt.bufPrint(why, "Upgrade: the session would not dump: {s}; it stays as it is", .{pardes.Messages.errorWords(err, &words)}) catch "Upgrade: the session would not dump"; }; defer core.gpa.free(bytes); - if (!writeDump(path, bytes)) return std.fmt.bufPrint(why, "Upgrade: the session's dump could not be written to {s}; it stays as it is", .{path[0..@min(path.len, 600)]}) catch "Upgrade: its dump could not be written"; + const put = core.db.putDump(core, bytes, label, at[0 .. at.len - 1]) catch |err| { + var words: [128]u8 = undefined; + return std.fmt.bufPrint(why, "Upgrade: the session's dump could not be written to {s} ({s}); it stays as it is", .{ path[0..@min(path.len, 600)], pardes.Messages.errorWords(err, &words) }) catch "Upgrade: its dump could not be written"; + }; + at[put.len] = 0; rec.dump = path; + rec.label = at[0..put.len]; rec.old = openSelf(); return null; } /// The daemon told, 9P's last answers sent (`fs`), and this process -/// exec'd as `path` with `rec`, whose dump `dumpInto` wrote at `at`. -/// Returns only when the exec failed, that dump removed and the binary it -/// replaces let go: why, in words (`why`'s). +/// exec'd as `path` with `rec`, whose dump `dumpInto` labelled `at`. +/// Returns only when the exec failed, the binary it replaces let go (the +/// dump stays, a dump among the state file's): why, in words (`why`'s). pub fn execAs(gpa: std.mem.Allocator, path: [:0]const u8, rec: *const Record, at: [*:0]const u8, fs: ?*@import("9p_io.zig").Listener, why: []u8) []const u8 { + _ = at; @import("daemon.zig").upgraded(); if (fs) |f| f.flushOut(); const err_words = exec(gpa, path, -1, rec); // Still here: the exec failed. - _ = libc.unlink(at); if (rec.old >= 0) _ = libc.close(rec.old); return std.fmt.bufPrint(why, "Upgrade: {s} would not start: {s}; this session stays as it is", .{ path[0..@min(path.len, 600)], err_words }) catch "Upgrade: the new pardes would not start"; } @@ -391,13 +399,17 @@ pub fn load(gpa: std.mem.Allocator, core_gpa: std.mem.Allocator, opts: pardes.Op const loaded = loaded: { if (libc.getenv("PARDES_TEST_UPGRADE_REFUSE")) |how| if (rec.refused.len == 0 or std.mem.eql(u8, std.mem.span(how), "all")) break :loaded error.RefusedByTest; - const bytes = pardes.filesystem.readFile(gpa, rec.dump) catch |err| break :loaded err; + const bytes = (if (rec.label.len > 0) pardes.Db.readDumpAt(gpa, rec.dump, rec.label) else pardes.filesystem.readFile(gpa, rec.dump)) catch |err| break :loaded err; defer gpa.free(bytes); break :loaded pardes.dump.resumeFromDump(core_gpa, opts, bytes, &kept); }; var buf: [700]u8 = undefined; + // Restore's word for the dump left behind: its label, or its file. + const again_name = if (rec.label.len > 0) rec.label else rec.dump; if (loaded) |p| { - _ = libc.unlink(path); + // An older build's dump, a file of its own, goes once loaded; a + // label stays among the state file's dumps. + if (rec.label.len == 0) _ = libc.unlink(path); // The binary it replaced, let go: held, its file (deleted) would // keep its blocks for as long as this session runs. if (rec.old >= 0) _ = libc.close(rec.old); @@ -424,13 +436,13 @@ pub fn load(gpa: std.mem.Allocator, core_gpa: std.mem.Allocator, opts: pardes.Op var o = opts; o.load_path = null; const p = try pardes.Pardes.init(core_gpa, o); - p.setMessage(p.active, std.fmt.bufPrint(&buf, "Upgrade: no build could load the session ({s}); its shells are here, and Restore {s} brings the rest back", .{ why, rec.dump[0..@min(rec.dump.len, 400)] }) catch "Upgrade: no build could load the session; its dump is kept for a Restore"); + p.setMessage(p.active, std.fmt.bufPrint(&buf, "Upgrade: no build could load the session ({s}); its shells are here, and Restore {s} brings the rest back", .{ why, again_name[0..@min(again_name.len, 400)] }) catch "Upgrade: no build could load the session; its dump is kept for a Restore"); return p; } } test "an upgrade's record reads back as it was written, and refuses one cut short" { - var rec: Record = .{ .link = 5, .stub = 7, .old = 9, .window = true, .exe = "/a b/pardes", .dump = "/run/u/pardes-upgrade-1.dump" }; + var rec: Record = .{ .link = 5, .stub = 7, .old = 9, .window = true, .exe = "/a b/pardes", .dump = "/home/u/.local/share/pardes/pardes.xitdb", .label = "upgrade-1" }; rec.addPty(3, 1234, 11); rec.addPty(0, 99, 12); var tio: posix.termios = std.mem.zeroes(posix.termios); @@ -449,7 +461,12 @@ test "an upgrade's record reads back as it was written, and refuses one cut shor try std.testing.expect(back.ttys[0].primary and back.ttys[0].view == 2 and back.ttys[0].stub == 14); try std.testing.expectEqual(@as(u8, 0x5a), std.mem.asBytes(&back.ttys[0].termios)[0]); try std.testing.expectEqualStrings("/a b/pardes", back.exe); - try std.testing.expectEqualStrings("/run/u/pardes-upgrade-1.dump", back.dump); + try std.testing.expectEqualStrings("/home/u/.local/share/pardes/pardes.xitdb", back.dump); + try std.testing.expectEqualStrings("upgrade-1", back.label); + // An older build's: its dump a file, no label. + const older = try Record.decode("link 5\nexe /a\ndump /run/u/pardes-upgrade-1.dump\n"); + try std.testing.expectEqualStrings("/run/u/pardes-upgrade-1.dump", older.dump); + try std.testing.expectEqualStrings("", older.label); try std.testing.expectError(error.BadRecord, Record.decode("link 5\n")); try std.testing.expectError(error.BadRecord, Record.decode("tty 0 1 2 1 abc\ndump x\n")); } diff --git a/src/walk.zig b/src/walk.zig index 426408c8..8360a8b6 100644 --- a/src/walk.zig +++ b/src/walk.zig @@ -1,9 +1,11 @@ //! The reflective walk over the core's state: one comptime `@typeInfo` walk //! that hashes a session into a digest (the replay harness compares two), //! encodes values for the tape, and dumps a session for another build to -//! load by name (`Dump`, `Load`: Restore, -l, the store, a handover, and -//! the join of phase 5). The rollback copy is to walk the same way -//! (docs/next-daemon.md). +//! load by name: into a xitdb database's maps (`Put`, `Get`: a Dump, +//! Restore, -l, an Upgrade, a handover; Db.zig), or as bytes with a table +//! of its types (`Dump`, `Load`: a closed pane, the join of phase 5, and +//! for one release the `.dump` files before). The rollback copy is to walk +//! the same way (docs/next-daemon.md). //! //! Values, never addresses: a pointer is followed to what it points at, a //! slice is its elements, an ArrayList its items, a hash map its entries in @@ -33,6 +35,7 @@ //! digest(*const T, *Hasher) void` of its own hashes itself; one with //! `dumpWrite`, `dumpRead` (and `dumpFree`) dumps itself as bytes. const std = @import("std"); +const xitdb = @import("xitdb"); pub const Hasher = std.hash.Wyhash; const builtin = @import("builtin"); @@ -1331,27 +1334,8 @@ pub fn Load(comptime roots: []const type) type { try l.fields(T, s, out, true); } - /// `out` as a new `T` is before its fields are read: each field's - /// default, an allocator the dump's. fn fresh(l: *Self, comptime T: type, out: *T) DecodeError!void { - if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom) return; - if (comptime kindOf(T) == .skip) return; - const s = @typeInfo(T).@"struct"; - inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| { - if (attr.@"comptime") continue; - if (comptime attr.default_value_ptr) |dp| { - @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*; - } else if (comptime F == std.mem.Allocator) { - @field(out, field) = l.gpa; - } else if (comptime !walkedField(T, field)) { - @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to"); - } - } - } - - fn walkedField(comptime T: type, comptime field: []const u8) bool { - inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true; - return false; + freshOf(T, out, l.gpa); } /// Whether the source's `ty` and `T` are of one kind, else the @@ -1731,6 +1715,28 @@ pub fn Load(comptime roots: []const type) type { }; } +/// `out` as a new `T` is before a load reads its fields: each field's +/// default, an allocator `gpa`. +fn freshOf(comptime T: type, out: *T, gpa: std.mem.Allocator) void { + if (comptime @typeInfo(T) != .@"struct" or isList(T) or isAtomic(T) or isBitset(T) or kindOf(T) == .custom or kindOf(T) == .skip) return; + const s = @typeInfo(T).@"struct"; + inline for (s.field_names, s.field_types, s.field_attrs) |field, F, attr| { + if (attr.@"comptime") continue; + if (comptime attr.default_value_ptr) |dp| { + @field(out, field) = @as(*const F, @ptrCast(@alignCast(dp))).*; + } else if (comptime F == std.mem.Allocator) { + @field(out, field) = gpa; + } else if (comptime !walkedField(T, field)) { + @compileError("walk: " ++ @typeName(T) ++ "." ++ field ++ " has no default to load to"); + } + } +} + +fn walkedField(comptime T: type, comptime field: []const u8) bool { + inline for (comptime dumpFields(T)) |n| if (comptime std.mem.eql(u8, n, field)) return true; + return false; +} + /// A value `save` made, `gpa`'s; what was let go is said in `notes` when /// it is given. pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes: ?*std.ArrayList(u8)) DecodeError!T { @@ -1742,6 +1748,762 @@ pub fn load(comptime T: type, gpa: std.mem.Allocator, bytes: []const u8, notes: return v; } +// ---- a dump as a database: the same walk, by name, into xitdb's maps ---- +// +// A dump (Db.zig) is a xitdb database in memory, its bytes one value of the +// state file, written by the walk by name: a struct is a map whose keys are +// its fields' names (`putKey`: a reader goes through them by name, never by +// a hash of its own), a union a map of its one field, a slice, an array or +// a list of records a list, of plain numbers one bytes value (its format +// tag the numbers' kind and width), an enum the bytes of its name, a number +// a uint or an int (a wider one its bytes), an optional an absent key (an +// explicit none where the field's default is not null), a type with +// `dumpWrite` its bytes, and a `.blob`'s text of `blob_min` bytes and up +// its BLAKE3 hash (tag "b3"): the text is kept once beside the dump +// (`Blob`). A pointer or a slice met again is the first one's value again: +// what two fields share is shared again. +// +// Another build reads it by name: a field the dump has not takes its +// default, a key this build has not is passed over, a value of another kind +// takes its default and is said (`notes`); with no default the value it is +// part of is refused, and an element of an array of optional pointers (a +// session's panes) is then left out, nothing above it. + +pub const Mem = xitdb.Database(.memory, u256); + +/// A text a dump keeps beside it, once: its BLAKE3 hash and its bytes (the +/// session's, while the dump is made). +pub const Blob = struct { hash: [32]u8, bytes: []const u8 }; + +/// A field's key: its name's word, unique within its struct (a reader +/// matches names, so another build's words need not be these). +fn nameKey(comptime name: []const u8) u256 { + return comptime std.hash.Wyhash.hash(0, name); +} + +/// Whether a run of `E` is one bytes value: plain numbers. +fn numbers(comptime E: type) bool { + return switch (@typeInfo(E)) { + .int => |i| @sizeOf(E) * 8 == i.bits and @sizeOf(E) <= 8, + .bool => true, + .float => |f| f.bits == 32 or f.bits == 64, + else => false, + }; +} + +/// A run of numbers' format tag: their kind and byte width ("u4", "i8", "b1"). +fn numbersTag(comptime E: type) [2]u8 { + const kind: u8 = switch (@typeInfo(E)) { + .int => |i| if (i.signedness == .signed) 'i' else 'u', + .bool => 'b', + .float => 'f', + else => unreachable, + }; + return .{ kind, '0' + @sizeOf(E) }; +} + +const blob_tag = "b3".*; +const wide_tag = "w8".*; +const custom_tag = "cu".*; +const bitset_tag = "bs".*; + +fn typeWord(comptime T: type) u64 { + return comptime std.hash.Wyhash.hash(0, @typeName(T)); +} + +/// Writes values into a database in memory, by name (see above). +pub const Put = struct { + gpa: std.mem.Allocator, + /// Each pointer and slice written, by address, length and type, and + /// where its value is: met again, that value is written again. + memo: std.AutoHashMapUnmanaged(Key, xitdb.Slot) = .empty, + /// The blobs met, each once. + blobs: std.ArrayListUnmanaged(Blob) = .empty, + in_blob: bool = false, + + const Key = struct { addr: usize, len: usize, ty: u64 }; + const Cursor = Mem.Cursor(.read_write); + + pub fn deinit(w: *Put) void { + w.memo.deinit(w.gpa); + w.blobs.deinit(w.gpa); + } + + /// `v` written as `name`'s value in `map` (a dump's roots). + pub fn member(w: *Put, map: Mem.HashMap(.read_write), comptime name: []const u8, comptime T: type, v: *const T) anyerror!void { + try map.putKey(comptime nameKey(name), .{ .bytes = name }); + var at = try map.putCursor(comptime nameKey(name)); + _ = try w.value(T, v, &at); + } + + /// `v` written at `c`; where its value is. + pub fn value(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot { + if (comptime isAtomic(T)) return w.value(@FieldType(T, "raw"), &v.raw, c); + if (comptime isList(T)) return w.many(@typeInfo(@FieldType(T, "items")).pointer.child, v.items, c, true); + switch (comptime kindOf(T)) { + .skip, .void => try c.write(.{ .slot = null }), + .bool => try c.write(.{ .uint = @intFromBool(v.*) }), + .int => if (@typeInfo(T).int.bits <= 64) { + try c.write(if (@typeInfo(T).int.signedness == .signed) .{ .int = v.* } else .{ .uint = v.* }); + } else { + const wide: i128 = @intCast(v.*); + try c.write(.{ .bytes_object = .{ .value = std.mem.asBytes(&wide), .format_tag = wide_tag } }); + }, + .float => try c.write(.{ .float = v.* }), + .@"enum" => if (std.enums.tagName(T, v.*)) |name| { + try c.write(.{ .bytes = name }); + } else try c.write(.{ .int = @intCast(@intFromEnum(v.*)) }), + .optional => { + const C = @typeInfo(T).optional.child; + if (v.*) |*payload| { + // An allocator, say, is no state: there or not, which a + // load answers with its own. + if (comptime kindOf(Canon(C)) == .skip) try c.write(.{ .uint = 1 }) else return w.value(C, payload, c); + } else try c.write(.{ .slot = null }); + }, + .one => { + const C = @typeInfo(T).pointer.child; + const key: Key = .{ .addr = @intFromPtr(v.*), .len = @max(1, @sizeOf(C)), .ty = typeWord(T) }; + if (w.memo.get(key)) |slot| { + try c.write(.{ .slot = slot }); + return slot; + } + const slot = try w.value(C, v.*, c); + try w.memo.put(w.gpa, key, slot); + return slot; + }, + .slice => return w.many(@typeInfo(T).pointer.child, v.*, c, true), + .array => return w.many(@typeInfo(T).array.child, v, c, false), + .prefix => unreachable, + .@"struct" => return w.fields(T, v, c), + .@"union" => { + const map = try Mem.HashMap(.read_write).init(c.*); + switch (v.*) { + inline else => |*payload, tag| { + const name = @tagName(tag); + try map.putKey(comptime nameKey(name), .{ .bytes = name }); + var at = try map.putCursor(comptime nameKey(name)); + _ = try w.value(@TypeOf(payload.*), payload, &at); + }, + } + return map.slot(); + }, + // No type a dump walks has them. + .err, .error_union => unreachable, + .custom => { + var buf: [4096]u8 = undefined; + var out = try c.writer(&buf); + try T.dumpWrite(v, w.gpa, &out.interface); + out.format_tag = custom_tag; + try out.finish(); + }, + .bitset => { + const words = (v.bit_length + @bitSizeOf(usize) - 1) / @bitSizeOf(usize); + var buf: [64]u8 = undefined; + var out = try c.writer(&buf); + try out.interface.writeAll(std.mem.asBytes(&@as(u64, v.bit_length))); + if (words > 0) try out.interface.writeAll(std.mem.sliceAsBytes(v.masks[0..words])); + out.format_tag = bitset_tag; + try out.finish(); + }, + } + return c.slot(); + } + + /// A run of `E`: numbers as one bytes value (a blob's text by its + /// hash), records as a list; a slice met again (`shared`) its first + /// value. + fn many(w: *Put, comptime E: type, items: []const E, c: *Cursor, comptime shared: bool) anyerror!xitdb.Slot { + const key: Key = .{ .addr = @intFromPtr(items.ptr), .len = items.len, .ty = typeWord([]const E) }; + if (shared and items.len > 0) if (w.memo.get(key)) |slot| { + try c.write(.{ .slot = slot }); + return slot; + }; + const slot = if (comptime numbers(E)) slot: { + const bytes = std.mem.sliceAsBytes(items); + if (E == u8 and w.in_blob and bytes.len >= blob_min) { + var sum: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(bytes, &sum, .{}); + const known = for (w.blobs.items) |b| { + if (std.mem.eql(u8, &b.hash, &sum)) break true; + } else false; + if (!known) try w.blobs.append(w.gpa, .{ .hash = sum, .bytes = bytes }); + try c.write(.{ .bytes_object = .{ .value = &sum, .format_tag = blob_tag } }); + } else try c.write(.{ .bytes_object = .{ .value = bytes, .format_tag = numbersTag(E) } }); + break :slot try c.slot(); + } else slot: { + const list = try Mem.ArrayList(.read_write).init(c.*); + for (items) |*e| { + var at = try list.appendCursor(); + _ = try w.value(E, e, &at); + } + break :slot try list.slot(); + }; + if (shared and items.len > 0) try w.memo.put(w.gpa, key, slot); + return slot; + } + + fn fields(w: *Put, comptime T: type, v: *const T, c: *Cursor) anyerror!xitdb.Slot { + const map = try Mem.HashMap(.read_write).init(c.*); + const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed"; + inline for (comptime dumpFields(T)) |name| { + const F = @FieldType(T, name); + if (comptime kindOf(Canon(F)) != .void) put: { + // Null where the field's default is: no key at all. + if (comptime @typeInfo(F) == .optional and defaultIsNull(T, name)) if (@field(v, name) == null) break :put; + const was = w.in_blob; + w.in_blob = was or comptime blob(T, name); + defer w.in_blob = was; + try map.putKey(comptime nameKey(name), .{ .bytes = name }); + var at = try map.putCursor(comptime nameKey(name)); + if (comptime cutArray(T, name)) { + const all = &@field(v, name); + _ = try w.many(@typeInfo(F).array.child, all[0..@min(all.len, lengthIn(v, comptime lengthOf(T, name).?))], &at, false); + } else if (comptime packed_struct) { + const copy: F = @field(v, name); + _ = try w.value(F, ©, &at); + } else _ = try w.value(F, &@field(v, name), &at); + } + } + return map.slot(); + } +}; + +fn defaultIsNull(comptime T: type, comptime name: []const u8) bool { + const F = @FieldType(T, name); + const attr = @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, name).?]; + const dp = attr.default_value_ptr orelse return false; + return @as(*const F, @ptrCast(@alignCast(dp))).* == null; +} + +/// Reads values a `Put` wrote, by name, as another build may have written +/// them (see above). +pub const Get = struct { + /// Where what is read is made (a `replica.alloc` field's own allocator + /// while it is read). + gpa: std.mem.Allocator, + /// The reader's own lists'. + own: std.mem.Allocator, + /// The texts its blobs' hashes name (Db.zig's `blobs`), by hash. + blobs: ?Mem.HashMap(.read_only) = null, + /// Each pointer and slice made so far: what a value met again is, and + /// what `rollback` frees. + made: std.ArrayList(Made) = .empty, + /// Where each value made is: met again, the same one. + memo: std.AutoHashMapUnmanaged(u128, usize) = .empty, + /// What a type's `dumpRead` made, freed by its `dumpFree` too. + customs: std.ArrayList(struct { ptr: *anyopaque, gpa: std.mem.Allocator, free: *const fn (*anyopaque, std.mem.Allocator) void }) = .empty, + /// What was let go, for the person whose session it is. + notes: std.ArrayList(u8) = .empty, + /// The field being read, for a note. + field: []const u8 = "", + allocs: Allocators = .{}, + + const Cursor = Mem.Cursor(.read_only); + pub const Error = error{ BadDump, Incompatible, OutOfMemory }; + + pub fn init(gpa: std.mem.Allocator) Get { + return .{ .gpa = gpa, .own = gpa }; + } + + pub fn deinit(l: *Get) void { + l.made.deinit(l.own); + l.memo.deinit(l.own); + l.customs.deinit(l.own); + l.notes.deinit(l.own); + } + + /// Everything read so far freed: a load that fails leaves nothing. + /// What it was read into must not be used after (nor freed). + pub fn rollback(l: *Get) void { + for (l.customs.items) |c| c.free(c.ptr, c.gpa); + for (l.made.items) |m| if (m.bytes.len > 0) m.gpa.rawFree(m.bytes, m.alignment, @returnAddress()); + l.customs.clearRetainingCapacity(); + l.made.clearRetainingCapacity(); + l.memo.clearRetainingCapacity(); + } + + pub fn note(l: *Get, comptime fmt: []const u8, args: anytype) void { + if (l.notes.items.len > 4096) return; + l.notes.print(l.own, fmt ++ "\n", args) catch {}; + } + + /// The value at `c`, made: what it points at is `gpa`'s. + pub fn get(l: *Get, comptime T: type, c: Cursor) Error!T { + var out: T = undefined; + try l.fresh(T, &out); + try l.value(T, c, &out); + return out; + } + + /// The struct at `c` read into `out` as it stands: a field the dump + /// holds is replaced (what it held is not freed), every other one is + /// left as it is, and one another build cannot give is left too, and + /// noted. + pub fn into(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void { + try l.fields(T, c, out, true); + } + + /// Where the value of `name` is in the map at `c` (a dump's roots), by + /// its name; null when it is not there. + pub fn member(c: Cursor, name: []const u8) Error!?Cursor { + const map = Mem.HashMap(.read_only).init(c) catch return error.BadDump; + var it = map.iterator() catch return error.BadDump; + while (it.next() catch return error.BadDump) |kv_at| { + const kv = kv_at.readKeyValuePair() catch return error.BadDump; + var buf: [64]u8 = undefined; + const got = kv.key_cursor.readBytes(&buf) catch continue; + if (std.mem.eql(u8, got, name)) return kv.value_cursor; + } + return null; + } + + fn fresh(l: *Get, comptime T: type, out: *T) Error!void { + freshOf(T, out, l.gpa); + } + + fn slotWord(slot: xitdb.Slot) u128 { + return @as(u72, @bitCast(slot)); + } + + fn slotOf(c: Cursor) Error!xitdb.Slot { + return c.slot() catch error.BadDump; + } + + /// Whether the value at `slot` has a place of its own (an inline number + /// or a short text is its slot alone: no identity to share). + fn placed(slot: xitdb.Slot) bool { + return switch (slot.tag) { + .none, .uint, .int, .float, .short_bytes => false, + else => true, + }; + } + + /// An allocation made for the value at `slot`, kept: what a value met + /// again is, and what `rollback` frees. + fn keep(l: *Get, bytes: []u8, alignment: std.mem.Alignment, slot: xitdb.Slot) Error!usize { + try l.made.append(l.own, .{ .bytes = bytes, .alignment = alignment, .gpa = l.gpa }); + const at = l.made.items.len - 1; + // Not kept after all: the caller frees it. + if (placed(slot)) l.memo.put(l.own, slotWord(slot), at) catch |err| { + _ = l.made.pop(); + return err; + }; + return at; + } + + /// The value made before for the same place, when there was one. + fn again(l: *Get, slot: xitdb.Slot) Error!?[]u8 { + if (!placed(slot)) return null; + const at = l.memo.get(slotWord(slot)) orelse return null; + // Made, then let go of by the reader's caller (a dump's mounts). + if (at >= l.made.items.len) return null; + const m = l.made.items[at]; + if (m.dead) return error.Incompatible; + return m.bytes; + } + + /// `T` read from `c` into `out`, which `fresh` has made ready. + fn value(l: *Get, comptime T: type, c: Cursor, out: *T) Error!void { + if (comptime isAtomic(T)) { + const R = @FieldType(T, "raw"); + var raw: R = undefined; + try l.fresh(R, &raw); + try l.value(R, c, &raw); + out.* = .{ .raw = raw }; + return; + } + const slot = try slotOf(c); + if (comptime isList(T)) { + const got = try l.slice(@typeInfo(@FieldType(T, "items")).pointer.child, c, slot); + out.* = .empty; + out.items = got; + out.capacity = got.len; + return; + } + switch (comptime kindOf(T)) { + .skip, .void => {}, + .bool => out.* = switch (slot.tag) { + .uint => slot.value != 0, + else => return error.Incompatible, + }, + .int => out.* = try l.int(T, c, slot), + .float => out.* = switch (slot.tag) { + .float => @floatCast(@as(f64, @bitCast(slot.value))), + else => return error.Incompatible, + }, + .@"enum" => { + const e = @typeInfo(T).@"enum"; + switch (slot.tag) { + .bytes, .short_bytes => { + var buf: [256]u8 = undefined; + const name = c.readBytes(&buf) catch return error.Incompatible; + inline for (e.field_names) |field| if (std.mem.eql(u8, name, field)) { + out.* = @field(T, field); + return; + }; + l.note("{s}: no {s} here", .{ l.field, name }); + return error.Incompatible; + }, + .int, .uint => if (e.mode == .nonexhaustive) { + const n: i64 = @bitCast(slot.value); + out.* = @enumFromInt(std.math.cast(e.tag_type, n) orelse return error.Incompatible); + } else return error.Incompatible, + else => return error.Incompatible, + } + }, + .optional => { + const C = @typeInfo(T).optional.child; + if (slot.tag == .none) { + out.* = null; + return; + } + if (comptime kindOf(Canon(C)) == .skip) { + out.* = if (comptime C == std.mem.Allocator) l.gpa else null; + return; + } + // Through a local, as the byte loader does: a pointer's + // null is its address. + if (comptime @typeInfo(C) == .pointer) { + var payload: C = undefined; + try l.value(C, c, &payload); + out.* = payload; + return; + } + out.* = @as(C, undefined); + try l.fresh(C, &out.*.?); + l.value(C, c, &out.*.?) catch |err| { + out.* = null; + return err; + }; + }, + .one => { + const C = @typeInfo(T).pointer.child; + if (try l.again(slot)) |bytes| { + out.* = @ptrCast(@alignCast(bytes.ptr)); + return; + } + const one = try l.gpa.create(C); + const at = l.keep(std.mem.asBytes(one), .of(C), slot) catch |err| { + l.gpa.destroy(one); + return err; + }; + try l.fresh(C, one); + l.value(C, c, one) catch |err| { + // Left half made: no later value may be it. + l.made.items[at].dead = true; + return err; + }; + out.* = one; + }, + .slice => { + if (comptime @typeInfo(T).pointer.sentinel_ptr != null) @compileError("walk: a sentinel slice: " ++ @typeName(T)); + out.* = try l.slice(@typeInfo(T).pointer.child, c, slot); + }, + .array => try l.elements(@typeInfo(T).array.child, c, slot, out), + .prefix => unreachable, + .@"struct" => try l.fields(T, c, out, false), + .@"union" => { + const u = @typeInfo(T).@"union"; + const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible; + var it = map.iterator() catch return error.BadDump; + const kv_at = (it.next() catch return error.BadDump) orelse return error.Incompatible; + const kv = kv_at.readKeyValuePair() catch return error.BadDump; + var buf: [256]u8 = undefined; + const name = kv.key_cursor.readBytes(&buf) catch return error.Incompatible; + inline for (u.field_names, u.field_types) |field, F| if (std.mem.eql(u8, name, field)) { + out.* = @unionInit(T, field, undefined); + try l.fresh(F, &@field(out.*, field)); + return l.value(F, kv.value_cursor, &@field(out.*, field)); + }; + l.note("{s}: no {s} here", .{ l.field, name }); + return error.Incompatible; + }, + .err, .error_union => unreachable, + .custom => { + const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible; + defer l.own.free(got.value); + T.dumpRead(out, l.gpa, got.value) catch |err| { + if (err == error.OutOfMemory) return error.OutOfMemory; + l.note("{s}: {t}", .{ l.field, err }); + return error.Incompatible; + }; + if (comptime @hasDecl(T, "dumpFree")) try l.customs.append(l.own, .{ .ptr = out, .gpa = l.gpa, .free = @ptrCast(&T.dumpFree) }); + }, + .bitset => { + const got = c.readBytesObjectAlloc(l.own, null) catch return error.Incompatible; + defer l.own.free(got.value); + if (got.value.len < 8) return error.BadDump; + const bits = countOf(std.mem.bytesToValue(u64, got.value[0..8])) catch return error.BadDump; + const words = (bits + @bitSizeOf(usize) - 1) / @bitSizeOf(usize); + if (got.value.len != 8 + words * @sizeOf(usize)) return error.BadDump; + out.* = try .initEmpty(l.gpa, bits); + if (words > 0) @memcpy(std.mem.sliceAsBytes(out.masks[0..words]), got.value[8..]); + }, + } + } + + fn int(l: *Get, comptime I: type, c: Cursor, slot: xitdb.Slot) Error!I { + _ = l; + return switch (slot.tag) { + .uint => std.math.cast(I, slot.value) orelse error.Incompatible, + .int => std.math.cast(I, @as(i64, @bitCast(slot.value))) orelse error.Incompatible, + .bytes, .short_bytes => wide: { + var buf: [16]u8 = undefined; + const got = c.readBytesObject(&buf) catch return error.Incompatible; + if (got.value.len != 16 or !std.mem.eql(u8, &(got.format_tag orelse return error.Incompatible), &wide_tag)) return error.Incompatible; + break :wide std.math.cast(I, std.mem.bytesToValue(i128, got.value[0..16])) orelse error.Incompatible; + }, + else => error.Incompatible, + }; + } + + /// Numbers' `bytes` (format `tag`) as `E`s into `out`, as many as fit: + /// another width of the same kind (another build's) converted one by + /// one. How many the bytes hold. + fn convert(comptime E: type, bytes: []const u8, tag: [2]u8, out: []E) Error!usize { + const width: usize = if (tag[1] >= '1' and tag[1] <= '8') tag[1] - '0' else return error.BadDump; + if (bytes.len % width != 0) return error.BadDump; + const n = bytes.len / width; + const fits = @min(n, out.len); + if (std.mem.eql(u8, &tag, &numbersTag(E))) { + @memcpy(std.mem.sliceAsBytes(out[0..fits]), bytes[0 .. fits * width]); + return n; + } + if (tag[0] != numbersTag(E)[0]) return error.Incompatible; + for (out[0..fits], 0..) |*e, i| { + const raw = bytes[i * width ..][0..width]; + e.* = switch (@typeInfo(E)) { + .int => |info| if (info.signedness == .signed) cast: { + var wide: i64 = 0; + @memcpy(std.mem.asBytes(&wide)[0..width], raw); + if (width < 8) { + const shift: u6 = @intCast(64 - width * 8); + wide = (wide << shift) >> shift; + } + break :cast std.math.cast(E, wide) orelse return error.Incompatible; + } else cast: { + var wide: u64 = 0; + @memcpy(std.mem.asBytes(&wide)[0..width], raw); + break :cast std.math.cast(E, wide) orelse return error.Incompatible; + }, + .float => if (width == 8) @floatCast(std.mem.bytesToValue(f64, raw[0..8])) else if (width == 4) @floatCast(std.mem.bytesToValue(f32, raw[0..4])) else return error.Incompatible, + .bool => raw[0] != 0, + else => unreachable, + }; + } + return n; + } + + /// The numbers' bytes at `c` and their format tag, `alloc`'s. + fn numbersAt(c: Cursor, alloc: std.mem.Allocator) Error!struct { bytes: []u8, tag: [2]u8 } { + const got = c.readBytesObjectAlloc(alloc, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.Incompatible; + const bytes: []u8 = @constCast(got.value); + const tag = got.format_tag orelse { + alloc.free(bytes); + return error.Incompatible; + }; + return .{ .bytes = bytes, .tag = tag }; + } + + /// A slice of `E` at `c`: numbers' bytes, a blob's text by its hash, or + /// a list; one met again is the same. + fn slice(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot) Error![]E { + if (try l.again(slot)) |bytes| return @as([*]E, @ptrCast(@alignCast(bytes.ptr)))[0 .. bytes.len / @max(1, @sizeOf(E))]; + if (comptime numbers(E)) { + switch (slot.tag) { + .bytes, .short_bytes => {}, + else => return error.Incompatible, + } + if (E == u8) if (try l.blob(c, slot)) |text| return text; + // A text, as it is written: its bytes are the slice. + if (E == u8) { + const got = try numbersAt(c, l.gpa); + if (!std.mem.eql(u8, &got.tag, &numbersTag(u8))) { + l.gpa.free(got.bytes); + return error.Incompatible; + } + if (got.bytes.len == 0) { + l.gpa.free(got.bytes); + return &.{}; + } + _ = l.keep(got.bytes, .@"1", slot) catch |err| { + l.gpa.free(got.bytes); + return err; + }; + return got.bytes; + } + const got = try numbersAt(c, l.own); + defer l.own.free(got.bytes); + const width: usize = if (got.tag[1] >= '1' and got.tag[1] <= '8') got.tag[1] - '0' else return error.BadDump; + const n = got.bytes.len / width; + if (n == 0) return &.{}; + const many = try l.gpa.alloc(E, n); + const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| { + l.gpa.free(many); + return err; + }; + _ = convert(E, got.bytes, got.tag, many) catch |err| { + l.made.items[at].dead = true; + return err; + }; + return many; + } + const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible; + const n = countOf(list.count() catch return error.BadDump) catch return error.BadDump; + if (n == 0) return &.{}; + const many = try l.gpa.alloc(E, n); + const at = l.keep(std.mem.sliceAsBytes(many), .of(E), slot) catch |err| { + l.gpa.free(many); + return err; + }; + l.listInto(E, list, many) catch |err| { + l.made.items[at].dead = true; + return err; + }; + return many; + } + + /// A blob's text, when `c` holds a blob's hash: made in `gpa`'s. + fn blob(l: *Get, c: Cursor, slot: xitdb.Slot) Error!?[]u8 { + var buf: [32]u8 = undefined; + const got = c.readBytesObject(&buf) catch return null; + const tag = got.format_tag orelse return null; + if (!std.mem.eql(u8, &tag, &blob_tag) or got.value.len != 32) return null; + const texts = l.blobs orelse return error.BadDump; + const at = (texts.getCursor(std.mem.readInt(u256, got.value[0..32], .big)) catch return error.BadDump) orelse return error.BadDump; + const text = at.readBytesAlloc(l.gpa, null) catch |err| return if (err == error.OutOfMemory) error.OutOfMemory else error.BadDump; + const bytes: []u8 = @constCast(text); + _ = l.keep(bytes, .@"1", slot) catch |err| { + l.gpa.free(bytes); + return err; + }; + return bytes; + } + + /// An array of `E` at `c`: as many as it holds, the rest as they are. + fn elements(l: *Get, comptime E: type, c: Cursor, slot: xitdb.Slot, out: []E) Error!void { + if (comptime numbers(E)) { + switch (slot.tag) { + .bytes, .short_bytes => {}, + else => return error.Incompatible, + } + const got = try numbersAt(c, l.own); + defer l.own.free(got.bytes); + _ = try convert(E, got.bytes, got.tag, out); + return; + } + const list = Mem.ArrayList(.read_only).init(c) catch return error.Incompatible; + try l.listInto(E, list, out); + } + + /// The list's elements into `out`, as many as fit; an element of an + /// array of optional pointers (a session's panes) that cannot be read is + /// left out, not the array. + fn listInto(l: *Get, comptime E: type, list: Mem.ArrayList(.read_only), out: []E) Error!void { + const droppable = comptime @typeInfo(E) == .optional and @typeInfo(@typeInfo(E).optional.child) == .pointer; + var failed = false; + var it = list.iterator() catch return error.BadDump; + var i: usize = 0; + while (it.next() catch return error.BadDump) |at| : (i += 1) { + if (i >= out.len) break; + try l.fresh(E, &out[i]); + l.value(E, at, &out[i]) catch |err| { + if (err != error.Incompatible) return err; + if (droppable) { + out[i] = null; + l.note("{s}[{d}] left out", .{ l.field, i }); + } else failed = true; + }; + } + if (failed) return error.Incompatible; + } + + fn fields(l: *Get, comptime T: type, c: Cursor, out: *T, existing: bool) Error!void { + const names = comptime dumpFields(T); + const outer = l.field; + defer l.field = outer; + const map = Mem.HashMap(.read_only).init(c) catch return error.Incompatible; + var found: std.StaticBitSet(@max(1, names.len)) = .empty; + var failed = false; + var it = map.iterator() catch return error.BadDump; + while (it.next() catch return error.BadDump) |kv_at| { + const kv = kv_at.readKeyValuePair() catch return error.BadDump; + var buf: [256]u8 = undefined; + // A key that is no field's name of this build: passed over. + const name = kv.key_cursor.readBytes(&buf) catch continue; + const k: ?usize = inline for (names, 0..) |field, k| { + if (std.mem.eql(u8, name, field)) break k; + } else null; + const at = k orelse continue; + found.set(at); + inline for (names, 0..) |field, i| if (i == at) { + l.field = field; + l.oneField(T, field, kv.value_cursor, out, existing) catch |err| { + if (err != error.Incompatible) return err; + const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?]; + if (comptime attr.default_value_ptr) |dp| { + @field(out, field) = @as(*const @FieldType(T, field), @ptrCast(@alignCast(dp))).*; + l.note("{s}.{s}: its kind changed; its default taken", .{ @typeName(T), field }); + } else if (existing) { + l.note("{s}.{s}: its kind changed; left as it was", .{ @typeName(T), field }); + } else failed = true; + }; + }; + } + // A field the writer did not have (or held null at a null default): + // its default (`fresh`), or the value cannot be made. + inline for (names, 0..) |field, i| if (!found.isSet(i)) { + const F = @FieldType(T, field); + const attr = comptime @typeInfo(T).@"struct".field_attrs[std.meta.fieldIndex(T, field).?]; + if (comptime attr.default_value_ptr == null and kindOf(Canon(F)) != .void) { + if (!existing) { + l.note("{s}.{s}: not in the dump", .{ @typeName(T), field }); + failed = true; + } + } else if (comptime @typeInfo(F) == .optional and defaultIsNull(T, field)) if (existing) { + // A null left out: null here too. + @field(out, field) = null; + }; + }; + if (failed) return error.Incompatible; + } + + fn oneField(l: *Get, comptime T: type, comptime field: []const u8, c: Cursor, out: *T, existing: bool) Error!void { + const F = @FieldType(T, field); + const packed_struct = comptime @typeInfo(T).@"struct".layout == .@"packed"; + const gpa = l.gpa; + defer l.gpa = gpa; + if (comptime allocOf(T, field)) |which| if (@field(l.allocs, which)) |a| { + l.gpa = a; + }; + // A field this build compiles out (a PDF slot where there is no + // MuPDF): passed over, said when it held something. + if (comptime kindOf(Canon(F)) == .void) { + if ((try slotOf(c)).tag != .none) l.note("{s}: not in this build", .{l.field}); + return; + } + if (comptime cutArray(T, field)) { + const slot = try slotOf(c); + return l.elements(@typeInfo(F).array.child, c, slot, &@field(out, field)); + } + // A struct held in place in one read as it stands (the root's): + // what it does not hold stays as it is too. + if (comptime kindOf(Canon(F)) == .@"struct" and Canon(F) == F and @typeInfo(F).@"struct".layout != .@"packed") if (existing) { + return l.fields(F, c, &@field(out, field), true); + }; + if (packed_struct) { + var copy: F = undefined; + try l.fresh(F, ©); + try l.value(F, c, ©); + @field(out, field) = copy; + return; + } + try l.fresh(F, &@field(out, field)); + try l.value(F, c, &@field(out, field)); + } +}; + test "an optional slice comes back, and the fields after it" { const S = struct { before: u32 = 0, command: ?[]u8 = null, after: u32 = 0 }; var text = "sleep 30".*; |
