//! The store: the state every pardes window shares, one bounded file under //! $XDG_STATE_HOME/pardes (Options.store_path), an append-only log of //! framed records, `@ \n\n`: the files opened //! lately (Recent, recent.zig) and the panes each session closed, each a //! one-pane Dump, which Panes brings back where it was. docs/next-store.md //! argues the format and the sharing; in short: //! //! - A writer takes `flock` on `store.lock` (never blocking a frame: busy, //! its records wait for the next step), reads what others appended since //! it last read, and appends its own with one write at the end. //! - A reader takes no lock: on use, one stat of the path; longer, it reads //! the tail; another inode (a compaction), all of it again. //! - A record running past the end is being written: read next time. One //! that is garbage is passed over to the next valid one, and the next //! writer compacts. A missing, foreign or garbage file is an empty store: //! pardes always starts. //! - Compaction (under the lock): the live records, the closed panes of //! sessions still running, into `store.new`, renamed over the store. The //! file never passes `max_bytes`. //! //! With no path (no home, the web, a test that gives none) nothing is //! kept past the session, and a closed pane's dump is held here instead. const Store = @This(); const std = @import("std"); const pardes = @import("pardes.zig"); const dump = @import("dump.zig"); const layout = @import("layout.zig"); const exec = @import("exec.zig"); const recent = @import("recent.zig"); const Pardes = pardes.Pardes; const Pane = pardes.Pane; /// The file is never longer: an append that would pass it compacts first, /// and a compaction keeps the newest live records within half of it. pub const max_bytes: u64 = 100 << 20; /// A closed pane's record is never longer: a terminal's that would be is /// kept without its history, any other pane's is not kept. pub const max_record: usize = 8 << 20; /// A session keeps this many closed panes, the newest. pub const max_closed = 64; /// The first record's body, its version; then a generation, new at each /// compaction, so an inode number used again is no other file read on. const magic = "pardes-store 1"; /// A header line `@k \n` is never longer. const max_head = 32; /// A closed pane's first line, its place and name, is never longer. const max_meta = 4096 + 128; /// A pane closed in this session, as Panes lists it and brings it back. pub const Closed = struct { serial: u32, /// The column it was in, by serial. column: u32, /// The pane its rows went to, by serial, and whether it was below it: /// brought back, it takes them from that pane alone. gainer: u32, below: bool, /// The pane it was attached to, by serial (0 none). owner: u32, rows: u16, /// What its Panes row shows: its file, command or directory (gpa). name: []u8, /// Its record's offset in the file read as `inode`. at: u64 = 0, /// Its record's body while it is in no file (not written yet, or no /// file to write it to); empty once written. body: []u8 = &.{}, /// Where its record is in `pending`, until that is written. pending_at: ?usize = null, }; /// The store file; null, nothing is kept past the session. path: ?[]const u8 = null, /// Its bound: `max_bytes`, less in a test. cap: u64 = max_bytes, /// This session's, on its closed panes' records. session: u64 = 0, pid: i32 = 0, /// The file as read so far: its inode, generation and how far. inode: std.Io.File.INode = 0, gen: u64 = 0, end: u64 = 0, /// Its length when first read whole: a file past twice that (and 1 MiB) /// is compacted. base: u64 = 0, /// A record that is not one was passed over: the next writer compacts. garbage: bool = false, /// Read once at start (or there was none): only then is anything written, /// so a session never writes over a store it did not read. loaded: bool = false, /// Framed records waiting for the lock. pending: std.ArrayList(u8) = .empty, /// This session's closed panes, oldest first. closed: std.ArrayList(Closed) = .empty, fn clockNs() i128 { var ts: std.c.timespec = .{ .sec = 0, .nsec = 0 }; _ = std.c.clock_gettime(.REALTIME, &ts); return @as(i128, ts.sec) * std.time.ns_per_s + ts.nsec; } fn ioOf() std.Io { return std.Io.Threaded.global_single_threaded.io(); } /// One host file call, `f(args)`, made out of the turn (pardes.turn): a /// store under a mount of this session's own tree is answered by this /// session. Only the call: while a step is out nothing may change the core, /// so the store and Recent change between these, never inside one. fn host(comptime f: anytype, args: anytype) @typeInfo(@TypeOf(f)).@"fn".return_type.? { pardes.turn.yield(); defer pardes.turn.back(); return @call(.auto, f, args); } const cwd = std.Io.Dir.cwd; const File = std.Io.File; const Dir = std.Io.Dir; /// A new session's store at `path`: its id from the pid and the clock. pub fn init(path: ?[]const u8) Store { var s: Store = .{ .path = path }; if (comptime pardes.hosted) { s.pid = std.c.getpid(); const now = clockNs(); s.session = std.hash.Wyhash.hash(@intCast(s.pid), std.mem.asBytes(&now)); } else s.session = 1; return s; } pub fn deinit(s: *Store, gpa: std.mem.Allocator) void { for (s.closed.items) |c| freeClosed(gpa, c); s.closed.deinit(gpa); s.pending.deinit(gpa); } fn freeClosed(gpa: std.mem.Allocator, c: Closed) void { gpa.free(c.name); gpa.free(c.body); } /// A copy for the session a Restore makes: the same session (the same /// process), its closed panes still its own. What is pending is the old /// session's to write (its deinit does); the copies hold their bodies /// until they are read back from the file. pub fn clone(s: *const Store, gpa: std.mem.Allocator) Store { var copy = s.*; copy.pending = .empty; copy.closed = .empty; for (s.closed.items) |c| { const name = gpa.dupe(u8, c.name) catch break; const body = gpa.dupe(u8, c.body) catch { gpa.free(name); break; }; var kept = c; kept.name = name; kept.body = body; kept.pending_at = null; copy.closed.append(gpa, kept) catch { freeClosed(gpa, kept); break; }; } return copy; } // ---- reading ---- /// A window of the file, read ahead: records are parsed from it, and a /// body that is skipped is never read. const Reader = struct { file: std.Io.File, io: std.Io, size: u64, buf: []u8, at: u64 = 0, len: usize = 0, /// Bytes `[from, from + n)`, `n` at most the window; null past the end. fn bytes(r: *Reader, from: u64, n: usize) ?[]const u8 { if (n > r.buf.len or from + n > r.size) return null; if (from < r.at or from + n > r.at + r.len) { r.len = host(File.readPositionalAll, .{ r.file, r.io, r.buf, from }) catch return null; r.at = from; if (n > r.len) return null; } const o: usize = @intCast(from - r.at); return r.buf[o..][0..n]; } }; const Rec = struct { kind: u8, at: u64, body: u64, len: usize, crc: u32, fn end(rec: Rec) u64 { return rec.body + rec.len + 1; } }; const Next = union(enum) { rec: Rec, partial, bad }; /// The record at `at`: whole, still being written (`partial`: its length /// runs past the end), or not a record. fn next(r: *Reader, at: u64) Next { if (at >= r.size) return .partial; const head = r.bytes(at, @intCast(@min(max_head, r.size - at))) orelse return .partial; const nl = std.mem.indexOfScalar(u8, head, '\n') orelse return if (head.len < max_head) .partial else .bad; const line = head[0..nl]; if (line.len < 5 or line[0] != '@' or line[2] != ' ') return .bad; var fields = std.mem.splitScalar(u8, line[3..], ' '); const len = std.fmt.parseInt(usize, fields.next() orelse return .bad, 10) catch return .bad; const crc_text = fields.next() orelse return .bad; if (crc_text.len != 8 or fields.next() != null or len > max_record + max_meta) return .bad; const crc = std.fmt.parseInt(u32, crc_text, 16) catch return .bad; const rec: Rec = .{ .kind = line[1], .at = at, .body = at + nl + 1, .len = len, .crc = crc }; if (rec.end() > r.size) return .partial; const tail = r.bytes(rec.end() - 1, 1) orelse return .partial; if (tail[0] != '\n') return .bad; return .{ .rec = rec }; } /// Whether `rec`'s body is the one its CRC says. fn verify(r: *Reader, rec: Rec) bool { var crc = std.hash.Crc32.init(); var at = rec.body; var left = rec.len; while (left > 0) { const n = @min(left, r.buf.len); crc.update(r.bytes(at, n) orelse return false); at += n; left -= n; } return crc.final() == rec.crc; } /// After garbage: the next `\n@` that starts a whole, valid record (or one /// still being written), else the end. fn resync(r: *Reader, from: u64) u64 { var at = from; while (at + 1 < r.size) : (at += 1) { const b = r.bytes(at, 2) orelse return r.size; if (b[0] != '\n' or b[1] != '@') continue; switch (next(r, at + 1)) { .rec => |rec| if (verify(r, rec)) return at + 1, .partial => return at + 1, .bad => {}, } } return r.size; } /// A closed pane's first line: ` /// \t`. const Meta = struct { session: u64, pid: i32, serial: u32, column: u32, gainer: u32, below: bool, owner: u32, rows: u16, name: []const u8, /// Where its dump starts in the body. dump_at: usize, }; fn parseMeta(body: []const u8) ?Meta { const nl = std.mem.indexOfScalar(u8, body, '\n') orelse return null; const tab = std.mem.indexOfScalar(u8, body[0..nl], '\t') orelse return null; var f = std.mem.splitScalar(u8, body[0..tab], ' '); var m: Meta = undefined; m.session = std.fmt.parseInt(u64, f.next() orelse return null, 16) catch return null; m.pid = std.fmt.parseInt(i32, f.next() orelse return null, 10) catch return null; m.serial = std.fmt.parseInt(u32, f.next() orelse return null, 10) catch return null; m.column = std.fmt.parseInt(u32, f.next() orelse return null, 10) catch return null; m.gainer = std.fmt.parseInt(u32, f.next() orelse return null, 10) catch return null; m.below = (std.fmt.parseInt(u1, f.next() orelse return null, 10) catch return null) == 1; m.owner = std.fmt.parseInt(u32, f.next() orelse return null, 10) catch return null; m.rows = std.fmt.parseInt(u16, f.next() orelse return null, 10) catch return null; if (f.next() != null) return null; m.name = body[tab + 1 .. nl]; m.dump_at = nl + 1; return m; } /// The store read at start; the old `recent` file beside it read into the /// list once, when there is no store yet, so nobody's list is lost. pub fn load(s: *Store, p: *Pardes) void { const path = s.path orelse return; if (comptime !pardes.hosted) return; defer s.loaded = true; const io = ioOf(); const file = host(Dir.openFile, .{ cwd(), io, path, Dir.OpenFileOptions{} }) catch |err| { if (err != error.FileNotFound) return; const dir = std.fs.path.dirname(path) orelse return; var buf: [4096 + 16]u8 = undefined; const old = std.fmt.bufPrint(&buf, "{s}/recent", .{dir}) catch return; const bytes = host(Dir.readFileAlloc, .{ cwd(), io, old, p.gpa, std.Io.Limit.limited(1 << 20) }) catch return; defer p.gpa.free(bytes); p.recent.read(p.recentGpa(), bytes); return; }; defer host(File.close, .{ file, io }); const st = host(File.stat, .{ file, io }) catch return; s.readFile(p, file, st); } /// On use: what other windows wrote since this one last read, read. One /// stat when nothing changed. pub fn refresh(s: *Store, p: *Pardes) void { const path = s.path orelse return; if (comptime !pardes.hosted) return; if (!s.loaded) return; const io = ioOf(); const st = host(Dir.statFile, .{ cwd(), io, path, Dir.StatFileOptions{} }) catch return; if (st.inode == s.inode and st.size == s.end) return; const file = host(Dir.openFile, .{ cwd(), io, path, Dir.OpenFileOptions{} }) catch return; defer host(File.close, .{ file, io }); const fst = host(File.stat, .{ file, io }) catch return; s.readFile(p, file, fst); } /// `file` read on from where this session left it, or all of it again /// when it is another file than the one read so far. fn readFile(s: *Store, p: *Pardes, file: File, st: File.Stat) void { const gpa = p.recentGpa(); const buf = gpa.alloc(u8, 64 << 10) catch return; defer gpa.free(buf); var r: Reader = .{ .file = file, .io = ioOf(), .size = st.size, .buf = buf }; var whole = st.inode != s.inode or st.size < s.end; // The same inode number may be a new file (one compaction freed it, // the next took it): its header's generation says. if (!whole and s.end > 0) { switch (next(&r, 0)) { .rec => |rec| whole = rec.kind != 'h' or headerGen(&r, rec) != s.gen, else => whole = true, } } if (whole) { s.inode = st.inode; s.end = 0; s.gen = 0; s.base = st.size; s.garbage = false; // Its closed panes are read again from it, at their offsets there; // the ones in no file yet stay. var w: usize = 0; for (s.closed.items) |c| { if (c.body.len == 0) { freeClosed(gpa, c); continue; } s.closed.items[w] = c; w += 1; } s.closed.shrinkRetainingCapacity(w); } if (st.size == s.end) return; s.end = s.scan(p, &r, s.end); } fn headerGen(r: *Reader, rec: Rec) ?u64 { if (rec.len > 64) return null; const b = r.bytes(rec.body, rec.len) orelse return null; if (!std.mem.startsWith(u8, b, magic ++ " ")) return null; return std.fmt.parseInt(u64, b[magic.len + 1 ..], 16) catch null; } /// The records from `from` applied; where reading stopped (before a /// record still being written, or the end). fn scan(s: *Store, p: *Pardes, r: *Reader, from: u64) u64 { var at = from; while (at < r.size) { switch (next(r, at)) { .partial => return at, .bad => { s.garbage = true; // A file that does not start as a store is none: empty. if (at == 0) return r.size; at = resync(r, at + 1); }, .rec => |rec| { if (at == 0) { s.gen = (if (rec.kind == 'h') headerGen(r, rec) else null) orelse { s.garbage = true; return r.size; }; } else if (!s.apply(p, r, rec)) s.garbage = true; at = rec.end(); }, } } return at; } /// One record read: noted in Recent, or a closed pane of this session's /// listed or unlisted. False when it is not what its header says. fn apply(s: *Store, p: *Pardes, r: *Reader, rec: Rec) bool { const gpa = p.recentGpa(); switch (rec.kind) { 'r' => { if (rec.len > 8192) return false; const b = r.bytes(rec.body, rec.len) orelse return false; if (std.hash.Crc32.hash(b) != rec.crc) return false; const e = recent.List.parseLine(b) orelse return false; p.recent.note(gpa, e.path, e.nums[0], e.nums[1], e.nums[2], e.nums[3]); }, 'c' => { // Only its first line is read: a closed pane is this session's // or none of its business, and its dump is read when it comes // back (`reopen`, which checks the CRC). const b = r.bytes(rec.body, @min(rec.len, max_meta)) orelse return false; const m = parseMeta(b) orelse return false; if (m.session != s.session) return true; s.list(gpa, m, rec.at, &.{}); }, 'x' => { if (rec.len > 64) return false; const b = r.bytes(rec.body, rec.len) orelse return false; if (std.hash.Crc32.hash(b) != rec.crc) return false; const sp = std.mem.indexOfScalar(u8, b, ' ') orelse return false; const session = std.fmt.parseInt(u64, b[0..sp], 16) catch return false; const serial = std.fmt.parseInt(u32, b[sp + 1 ..], 10) catch return false; if (session == s.session) s.unlist(gpa, serial); }, // A header in the middle is no record. 'h' => return false, // A kind a newer pardes writes: passed over. else => {}, } return true; } /// A closed pane listed: from the file (`body` empty, its record at `at`) /// or not in one yet. In the order closed, the oldest going past /// `max_closed`. fn list(s: *Store, gpa: std.mem.Allocator, m: Meta, at: u64, body: []u8) void { for (s.closed.items) |c| if (c.serial == m.serial) { gpa.free(body); return; }; const name = gpa.dupe(u8, m.name) catch { gpa.free(body); return; }; const c: Closed = .{ .serial = m.serial, .column = m.column, .gainer = m.gainer, .below = m.below, .owner = m.owner, .rows = m.rows, .name = name, .at = at, .body = body }; if (s.closed.items.len >= max_closed) freeClosed(gpa, s.closed.orderedRemove(0)); // One read from the file goes before those in none yet: they are newer. var i = s.closed.items.len; if (body.len == 0) while (i > 0 and s.closed.items[i - 1].body.len > 0) : (i -= 1) {}; s.closed.insert(gpa, i, c) catch freeClosed(gpa, c); } fn unlist(s: *Store, gpa: std.mem.Allocator, serial: u32) void { for (s.closed.items, 0..) |c, i| if (c.serial == serial) { freeClosed(gpa, s.closed.orderedRemove(i)); return; }; } /// The closed pane with this serial, if this session has it. pub fn closedBySerial(s: *const Store, serial: u32) ?*Closed { for (s.closed.items) |*c| if (c.serial == serial) return c; return null; } // ---- writing ---- fn frame(gpa: std.mem.Allocator, out: *std.ArrayList(u8), kind: u8, parts: []const []const u8) !void { var len: usize = 0; var crc = std.hash.Crc32.init(); for (parts) |part| { len += part.len; crc.update(part); } try out.print(gpa, "@{c} {d} {x:0>8}\n", .{ kind, len, crc.final() }); for (parts) |part| try out.appendSlice(gpa, part); try out.append(gpa, '\n'); } /// Pane `id`, about to close, kept as a record: what it was (a one-pane /// Dump) and where (its column, its rows, what it was attached to). Null /// for a pane that is no pane of the user's: a query pane, a preview, one /// made and refused in the same step (never announced), or one whose dump /// fails or is too big. `close` finishes it once its rows have gone. pub fn keep(s: *Store, p: *Pardes, id: usize) ?Kept { _ = s; const pane = p.panes[id] orelse return null; if (pane.fs.unannounced or pane.query != null) return null; if (pane.file) |f| if (f.output) |o| if (o.from == .preview) return null; const gpa = p.recentGpa(); const arena = p.scratch.allocator(); var dp = dump.dumpPane(p, arena, pane) catch return null; var zon = oneDump(p, arena, dp) catch return null; // A terminal's history is what makes it big: kept without it, it comes // back a fresh shell in its directory. if (zon.len > max_record) if (dp.terminal) |*t| { t.stream = ""; t.stream_b64 = ""; dp.body = ""; zon = oneDump(p, arena, dp) catch return null; }; if (zon.len > max_record) return null; const what: []const u8 = if (pane.file) |f| f.path else if (pane.pdfPath()) |pdf| pdf else if (pane.image) |iv| iv.path else if (pane.command) |c| c else pane.cwdSlice(); var name = gpa.alloc(u8, @min(what.len, 4096)) catch return null; // One line, a tab its end: a newline or a tab in it is a space. for (what[0..name.len], 0..) |c, i| name[i] = if (c == '\n' or c == '\t') ' ' else c; const owned = gpa.dupe(u8, zon) catch { gpa.free(name); return null; }; const at = layout.findPane(p, id); const owner = layout.ownerOf(p, id); return .{ .serial = pane.serial, .column = if (at) |f| layout.columnSerial(p, f.col) else 0, .owner = if (owner) |o| p.panes[o].?.serial else 0, .idx = if (at) |f| f.idx else 0, .rows = p.rects[id].h, .name = name, .dump = owned, }; } pub const Kept = struct { serial: u32, column: u32, owner: u32, /// Its place in its column, to say which side of the gainer it was. idx: usize, rows: u16, name: []u8, dump: []u8, pub fn free(k: Kept, gpa: std.mem.Allocator) void { gpa.free(k.name); gpa.free(k.dump); } }; /// A Dump of the one pane `dp`, as a Restore would read it. fn oneDump(p: *Pardes, arena: std.mem.Allocator, dp: dump.Pane) ![]const u8 { const ids = [_]usize{0}; const columns = [_]dump.Column{.{ .panes = &ids }}; const one = [_]dump.Pane{dp}; const state: dump.State = .{ .screen = .{ .cols = p.screen_w, .rows = p.screen_h }, .columns = &columns, .panes = &one }; try dump.validate(state); var out: std.Io.Writer.Allocating = .init(arena); try std.zon.stringify.serialize(state, .{}, &out.writer); return out.written(); } /// The pane `k` kept is gone, its rows given to `gainer` (a slot, after /// they moved; null none): listed, and its record framed for the store. pub fn close(s: *Store, p: *Pardes, k: Kept, gainer: ?usize) void { const gpa = p.recentGpa(); var below = false; var gainer_serial: u32 = 0; if (gainer) |g| if (p.panes[g]) |gp| { gainer_serial = gp.serial; // It went from just below the gainer's place, or it was the first // and the rows went down. if (layout.findPane(p, g)) |f| below = f.idx < k.idx; }; var meta_buf: [max_meta]u8 = undefined; const meta = std.fmt.bufPrint(&meta_buf, "{x:0>16} {d} {d} {d} {d} {d} {d} {d}\t{s}\n", .{ s.session, s.pid, k.serial, k.column, gainer_serial, @intFromBool(below), k.owner, k.rows, k.name, }) catch { k.free(gpa); return; }; const m = parseMeta(meta).?; const body = std.mem.concat(gpa, u8, &.{ meta, k.dump }) catch { k.free(gpa); return; }; gpa.free(k.dump); const pending_at = s.pending.items.len; const framed = if (s.path != null) blk: { frame(gpa, &s.pending, 'c', &.{body}) catch break :blk false; break :blk true; } else false; s.list(gpa, m, 0, body); gpa.free(k.name); if (framed) if (s.closedBySerial(m.serial)) |c| { c.pending_at = pending_at; }; } /// Written out when anything changed, at the end of a step and at the end /// (`wait`: the lock is waited for there, never in a step). pub fn flush(s: *Store, p: *Pardes, wait: bool) void { const gpa = p.recentGpa(); if (!p.recent.dirty and s.pending.items.len == 0) return; // No file: nothing to write it to, ever. if (comptime !pardes.hosted) return s.forget(p); const path: ?[]const u8 = s.path orelse return s.forget(p); if (!s.loaded) return; const io = ioOf(); var name_buf: [4096 + 16]u8 = undefined; const lock_path = std.fmt.bufPrint(&name_buf, "{s}.lock", .{path.?}) catch return; const lock = host(Dir.createFile, .{ cwd(), io, lock_path, Dir.CreateFileOptions{ .truncate = false } }) catch return; // Closing it lets go of the lock. defer host(File.close, .{ lock, io }); if (wait) { host(File.lock, .{ lock, io, File.Lock.exclusive }) catch return; } else if (!(host(File.tryLock, .{ lock, io, File.Lock.exclusive }) catch return)) return; const file = host(Dir.createFile, .{ cwd(), io, path.?, Dir.CreateFileOptions{ .read = true, .truncate = false } }) catch return; defer host(File.close, .{ file, io }); const st = host(File.stat, .{ file, io }) catch return; // Others' records first, so this window's go after them here too. s.readFile(p, file, st); // Under the lock, a record still "being written" is one whose writer // died: garbage. if (s.end < st.size) s.garbage = true; p.recent.changedFirst(); var i = p.recent.entries.items.len; var line_buf: [4096 + 64]u8 = undefined; while (i > 0) : (i -= 1) { const e = &p.recent.entries.items[i - 1]; if (!e.changed) continue; e.changed = false; var w = std.Io.Writer.fixed(&line_buf); recent.List.writeEntry(e.*, &w) catch continue; const line = std.mem.trimEnd(u8, w.buffered(), "\n"); if (line.len > 0) frame(gpa, &s.pending, 'r', &.{line}) catch {}; } p.recent.dirty = false; if (s.garbage or st.size == 0 or st.size + s.pending.items.len > s.cap or st.size > 2 * s.base + (1 << 20)) { s.compact(p, file, st, path.?); return; } host(File.writePositionalAll, .{ file, io, s.pending.items, st.size }) catch { // Out of disk: kept for no one, as the old list was. s.dropPending(gpa); return; }; for (s.closed.items) |*c| if (c.pending_at) |off| { c.at = st.size + off; c.pending_at = null; gpa.free(c.body); c.body = &.{}; }; s.end = st.size + s.pending.items.len; s.pending.clearRetainingCapacity(); } fn forget(s: *Store, p: *Pardes) void { for (p.recent.entries.items) |*e| e.changed = false; p.recent.dirty = false; s.pending.clearRetainingCapacity(); for (s.closed.items) |*c| c.pending_at = null; } fn dropPending(s: *Store, gpa: std.mem.Allocator) void { _ = gpa; s.pending.clearRetainingCapacity(); for (s.closed.items) |*c| c.pending_at = null; } fn alive(pid: i32) bool { if (pid <= 0) return false; const rc = std.c.kill(pid, @enumFromInt(0)); return rc == 0 or std.c.errno(rc) == .PERM; } /// Under the lock: the live records into `.new`, this window's /// pending ones after them, renamed over the store; then read again whole. fn compact(s: *Store, p: *Pardes, file: File, st: File.Stat, path: []const u8) void { const gpa = p.recentGpa(); const io = ioOf(); defer s.dropPending(gpa); const buf = gpa.alloc(u8, 64 << 10) catch return; defer gpa.free(buf); var r: Reader = .{ .file = file, .io = io, .size = st.size, .buf = buf }; var fold: recent.List = .{}; defer fold.deinit(gpa); const Live = struct { at: u64, end: u64, session: u64, pid: i32, serial: u32, keep: bool = false }; var lives: std.ArrayList(Live) = .empty; defer lives.deinit(gpa); const Gone = struct { session: u64, serial: u32 }; var gone: std.ArrayList(Gone) = .empty; defer gone.deinit(gpa); var at: u64 = 0; while (at < r.size) { switch (next(&r, at)) { .partial => break, .bad => at = resync(&r, at + 1), .rec => |rec| { at = rec.end(); if (rec.kind == 'h' or !verify(&r, rec)) continue; const b = r.bytes(rec.body, @min(rec.len, max_meta)) orelse continue; switch (rec.kind) { 'r' => if (recent.List.parseLine(b)) |e| fold.note(gpa, e.path, e.nums[0], e.nums[1], e.nums[2], e.nums[3]), 'c' => if (parseMeta(b)) |m| lives.append(gpa, .{ .at = rec.at, .end = rec.end(), .session = m.session, .pid = m.pid, .serial = m.serial }) catch {}, 'x' => { const sp = std.mem.indexOfScalar(u8, b, ' ') orelse continue; gone.append(gpa, .{ .session = std.fmt.parseInt(u64, b[0..sp], 16) catch continue, .serial = std.fmt.parseInt(u32, b[sp + 1 ..], 10) catch continue, }) catch {}; }, else => {}, } }, } } // The closed panes kept: of running sessions, not brought back, the // newest `max_closed` a session, the newest within half the cap. var budget: u64 = s.cap / 2 -| s.pending.items.len; var k = lives.items.len; while (k > 0) : (k -= 1) { const l = &lives.items[k - 1]; const reopened = for (gone.items) |g| { if (g.session == l.session and g.serial == l.serial) break true; } else false; if (reopened or (l.session != s.session and !alive(l.pid))) continue; var newer: usize = 0; for (lives.items[k..]) |o| if (o.keep and o.session == l.session) { newer += 1; }; if (newer >= max_closed or l.end - l.at > budget) continue; budget -= l.end - l.at; l.keep = true; } var name_buf: [4096 + 16]u8 = undefined; const new_path = std.fmt.bufPrint(&name_buf, "{s}.new", .{path}) catch return; const out = host(Dir.createFile, .{ cwd(), io, new_path, Dir.CreateFileOptions{} }) catch return; var ok = false; defer if (!ok) { host(File.close, .{ out, io }); host(Dir.deleteFile, .{ cwd(), io, new_path }) catch {}; }; var head: std.ArrayList(u8) = .empty; defer head.deinit(gpa); var gen_buf: [64]u8 = undefined; const now = clockNs(); const gen = std.hash.Wyhash.hash(s.session, std.mem.asBytes(&now)); frame(gpa, &head, 'h', &.{std.fmt.bufPrint(&gen_buf, magic ++ " {x}", .{gen}) catch return}) catch return; var line_buf: [4096 + 64]u8 = undefined; var i = fold.entries.items.len; while (i > 0) : (i -= 1) { var w = std.Io.Writer.fixed(&line_buf); recent.List.writeEntry(fold.entries.items[i - 1], &w) catch continue; const line = std.mem.trimEnd(u8, w.buffered(), "\n"); if (line.len > 0) frame(gpa, &head, 'r', &.{line}) catch return; } host(File.writePositionalAll, .{ out, io, head.items, 0 }) catch return; var off: u64 = head.items.len; for (lives.items) |l| if (l.keep) { var from = l.at; while (from < l.end) { const n: usize = @intCast(@min(buf.len, l.end - from)); const b = r.bytes(from, n) orelse return; host(File.writePositionalAll, .{ out, io, b, off }) catch return; off += n; from += n; } }; host(File.writePositionalAll, .{ out, io, s.pending.items, off }) catch return; host(File.sync, .{ out, io }) catch return; host(File.close, .{ out, io }); ok = true; host(Dir.rename, .{ cwd(), new_path, cwd(), path, io }) catch { host(Dir.deleteFile, .{ cwd(), io, new_path }) catch {}; return; }; // Every closed pane of this session is in the new file now: read again // from it, at its offsets there. for (s.closed.items) |c| freeClosed(gpa, c); s.closed.clearRetainingCapacity(); s.inode = 0; const again = host(Dir.openFile, .{ cwd(), io, path, Dir.OpenFileOptions{} }) catch return; defer host(File.close, .{ again, io }); const nst = host(File.stat, .{ again, io }) catch return; s.readFile(p, again, nst); } // ---- bringing a closed pane back ---- /// Closed pane `serial` of this session made again, where it was: beside /// the pane its rows went to, taking them from that pane alone (attached /// again to its owner when that is still open), else placed as a new pane /// from `from`. False when it is not this session's, or its record is gone /// or bad; the keyboard on it otherwise. pub fn reopen(s: *Store, p: *Pardes, serial: u32, from: usize) bool { s.refresh(p); const gpa = p.recentGpa(); const place = (s.closedBySerial(serial) orelse return false).*; const body = s.bodyOf(p, place) orelse return false; // Read from the file: this call's to free (`unlist` frees one held). defer if (place.body.len == 0) gpa.free(body); const m = parseMeta(body) orelse return false; var parsed = dump.readZon(p.gpa, body[m.dump_at..], "closed pane") catch return false; defer parsed.deinit(); const free = p.freeSlot() orelse { p.reportError(from, "Panes", error.NoPaneSlots); return false; }; const src = parsed.value.panes[0]; const pane = dump.restorePane(p, free, src) catch |err| { p.reportError(from, "Panes", err); return false; }; pane.collapsed = false; const owner = if (place.owner != 0) p.paneBySerial(place.owner) else null; const gainer = if (place.gainer != 0) p.paneBySerial(place.gainer) else null; const placed = if (owner) |o| layout.attachFrom(p, o, free, place.rows, o) else if (gainer) |g| layout.giveBack(p, free, g, place.below, place.rows) else false; if (!placed) exec.placeNew(p, from, from, free, .doc); if (layout.findPane(p, free) == null) { p.removePane(free, null) catch {}; return false; } // No room anywhere: refused, it closes at the step's end, still listed. if (p.unplaced.isSet(free)) return false; // Brought back: no longer listed, here or when read again. var rec_buf: [64]u8 = undefined; const rec = std.fmt.bufPrint(&rec_buf, "{x:0>16} {d}", .{ s.session, serial }) catch unreachable; if (s.path != null) frame(gpa, &s.pending, 'x', &.{rec}) catch {}; s.unlist(gpa, serial); p.active = free; if (layout.findPane(p, free)) |f| p.setActiveColumn(f.col); p.sync(); return true; } /// A closed pane's record body: held here, or read from the file and /// checked against its CRC (gpa; the caller frees it when it is not /// `c.body`). fn bodyOf(s: *Store, p: *Pardes, c: Closed) ?[]u8 { if (c.body.len > 0) return c.body; const path = s.path orelse return null; if (comptime !pardes.hosted) return null; const gpa = p.recentGpa(); const io = ioOf(); const file = host(Dir.openFile, .{ cwd(), io, path, Dir.OpenFileOptions{} }) catch return null; defer host(File.close, .{ file, io }); const st = host(File.stat, .{ file, io }) catch return null; if (st.inode != s.inode) return null; var head_buf: [max_head]u8 = undefined; var r: Reader = .{ .file = file, .io = io, .size = st.size, .buf = &head_buf }; const rec = switch (next(&r, c.at)) { .rec => |rec| rec, else => return null, }; if (rec.kind != 'c') return null; const body = gpa.alloc(u8, rec.len) catch return null; const n = host(File.readPositionalAll, .{ file, io, body, rec.body }) catch 0; if (n != rec.len or std.hash.Crc32.hash(body) != rec.crc) { gpa.free(body); return null; } return body; } /// `@p` naming a pane this session closed: its serial. pub fn closedAddress(s: *const Store, word: []const u8) ?u32 { const addr = @import("config.zig").pane_addr; if (!std.mem.startsWith(u8, word, addr)) return null; const digits = word[addr.len..]; const end = for (digits, 0..) |ch, i| { if (!std.ascii.isDigit(ch)) break i; } else digits.len; // `@p7` alone, or `@p7:` with nothing after it: the pane, no place in it. if (end == 0 or (end < digits.len and !(digits[end] == ':' and end + 1 == digits.len))) return null; const serial = std.fmt.parseInt(u32, digits[0..end], 10) catch return null; return if (s.closedBySerial(serial) != null) serial else null; } // ---- tests ---- const testing = std.testing; /// The store file of a test, in its own directory (never the user's). fn testPath(tmp: *std.testing.TmpDir) ![]u8 { var base_buf: [std.fs.max_path_bytes]u8 = undefined; const dir = base_buf[0..try tmp.dir.realPath(testing.io, &base_buf)]; return std.fs.path.join(testing.allocator, &.{ dir, "store" }); } fn testWindow(path: []const u8) !*Pardes { return Pardes.init(testing.allocator, .{ .tty_only = true, .cols = 120, .rows = 48, .store_path = path }); } fn fileSize(path: []const u8) u64 { const st = std.Io.Dir.cwd().statFile(testing.io, path, .{}) catch return 0; return st.size; } test "store: two windows share Recent, each reading the other's writes on use; a busy lock delays, never loses" { if (comptime !pardes.hosted) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); const path = try testPath(&tmp); defer testing.allocator.free(path); const a = try testWindow(path); defer a.deinit(); const b = try testWindow(path); defer b.deinit(); try testing.expect(a.store.session != b.store.session); a.recent.opened(a.recentGpa(), "/w/a.zig"); a.saveRecent(); b.recent.opened(b.recentGpa(), "/w/b.zig"); b.saveRecent(); a.recent.closed(a.recentGpa(), "/w/a.zig", 7, 3); a.saveRecent(); // Each has both, in the order of the log: a closed last, then b. a.store.refresh(a); b.store.refresh(b); for ([_]*Pardes{ a, b }) |w| { try testing.expectEqual(@as(usize, 2), w.recent.entries.items.len); try testing.expectEqualStrings("/w/a.zig", w.recent.entries.items[0].path); try testing.expectEqualStrings("/w/b.zig", w.recent.entries.items[1].path); try testing.expectEqual(@as(u32, 7), w.recent.get("/w/a.zig").?.line); } // Another window holding the lock: b's change waits in memory... const lock_path = try std.fmt.allocPrint(testing.allocator, "{s}.lock", .{path}); defer testing.allocator.free(lock_path); const lock = try std.Io.Dir.cwd().createFile(testing.io, lock_path, .{ .truncate = false }); try lock.lock(testing.io, .exclusive); b.recent.opened(b.recentGpa(), "/w/c.zig"); const size = fileSize(path); b.saveRecent(); try testing.expectEqual(size, fileSize(path)); try testing.expect(b.recent.dirty); // ...and is written at the next step after it is let go. lock.close(testing.io); b.saveRecent(); try testing.expect(fileSize(path) > size); a.store.refresh(a); try testing.expectEqualStrings("/w/c.zig", a.recent.entries.items[0].path); // A window started now folds the same list. const c = try testWindow(path); defer c.deinit(); try testing.expectEqual(@as(usize, 3), c.recent.entries.items.len); for (a.recent.entries.items, c.recent.entries.items) |x, y| try testing.expectEqualStrings(x.path, y.path); } test "store: the cap holds, compaction keeps the newest, Recent keeps its 200" { if (comptime !pardes.hosted) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); const path = try testPath(&tmp); defer testing.allocator.free(path); const a = try testWindow(path); defer a.deinit(); a.store.cap = 48 << 10; var name: [32]u8 = undefined; var inodes: usize = 0; var last_inode = a.store.inode; for (0..3000) |i| { const file = std.fmt.bufPrint(&name, "/w/file{d}.zig", .{i}) catch unreachable; a.recent.opened(a.recentGpa(), file); a.recent.closed(a.recentGpa(), file, 1, 1); a.saveRecent(); try testing.expect(fileSize(path) <= a.store.cap); if (a.store.inode != last_inode) inodes += 1; last_inode = a.store.inode; } try testing.expect(inodes > 2); try testing.expectEqual(@as(usize, recent.cap), a.recent.entries.items.len); try testing.expectEqualStrings("/w/file2999.zig", a.recent.entries.items[0].path); const b = try testWindow(path); defer b.deinit(); try testing.expectEqual(@as(usize, recent.cap), b.recent.entries.items.len); try testing.expectEqualStrings("/w/file2999.zig", b.recent.entries.items[0].path); try testing.expectEqualStrings("/w/file2800.zig", b.recent.entries.items[recent.cap - 1].path); // Closed panes' text counts too: the newest stay, within the cap. a.store.cap = 256 << 10; var text: [20 << 10]u8 = @splat('t'); var serial: u32 = 0; for (0..30) |i| { a.newScratchBelow(a.active); const id = a.active; const f = &a.panes[id].?.file.?; text[0] = @intCast('a' + i % 26); @import("File.zig").setContent(a, f, try a.gpa.dupe(u8, &text)); serial = a.panes[id].?.serial; // Made outside a step: announced as a step's end would. pardes.ctlfs.events.announce(a); try a.removePane(id, null); a.saveRecent(); try testing.expect(fileSize(path) <= a.store.cap); } try testing.expect(a.store.closed.items.len < 30); try testing.expectEqual(serial, a.store.closed.items[a.store.closed.items.len - 1].serial); try testing.expect(a.store.reopen(a, serial, a.active)); try testing.expectEqual(@as(u8, 'a' + 29 % 26), a.panes[a.active].?.file.?.content[0]); } test "store: a garbage or torn file never stops pardes, records past garbage are read, the next writer mends it" { if (comptime !pardes.hosted) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); const path = try testPath(&tmp); defer testing.allocator.free(path); try std.Io.Dir.cwd().writeFile(testing.io, .{ .sub_path = path, .data = "\x00\xffnot a store at all\n@r 9 zz\n" }); const a = try testWindow(path); defer a.deinit(); try testing.expectEqual(@as(usize, 0), a.recent.entries.items.len); try testing.expect(a.store.garbage); a.recent.opened(a.recentGpa(), "/w/a.zig"); a.saveRecent(); try testing.expect(!a.store.garbage); // A torn record at the end (its writer died), then garbage, then a // whole record: the whole one is read, the rest passed over. var tail: std.ArrayList(u8) = .empty; defer tail.deinit(testing.allocator); try tail.appendSlice(testing.allocator, "@r 40 0000"); try tail.appendSlice(testing.allocator, "\x00junk\n@x not a header\n"); try frame(testing.allocator, &tail, 'r', &.{"3 1 0 0 /w/after.zig"}); { const f = try std.Io.Dir.cwd().openFile(testing.io, path, .{ .mode = .read_write }); defer f.close(testing.io); try f.writePositionalAll(testing.io, tail.items, (try f.stat(testing.io)).size); } const b = try testWindow(path); defer b.deinit(); try testing.expect(b.recent.get("/w/a.zig") != null); try testing.expectEqual(@as(u32, 3), b.recent.get("/w/after.zig").?.line); try testing.expect(b.store.garbage); // A record half written (the length runs past the end) is waited for, // not garbage: a reader reads it once it is whole. var half: std.ArrayList(u8) = .empty; defer half.deinit(testing.allocator); try frame(testing.allocator, &half, 'r', &.{"5 1 0 0 /w/late.zig"}); { const f = try std.Io.Dir.cwd().openFile(testing.io, path, .{ .mode = .read_write }); defer f.close(testing.io); const end = (try f.stat(testing.io)).size; try f.writePositionalAll(testing.io, half.items[0..10], end); a.store.refresh(a); try testing.expect(a.recent.get("/w/late.zig") == null); try f.writePositionalAll(testing.io, half.items[10..], end + 10); } a.store.refresh(a); try testing.expectEqual(@as(u32, 5), a.recent.get("/w/late.zig").?.line); // The next write compacts it: a reader then finds no garbage. b.recent.opened(b.recentGpa(), "/w/b.zig"); b.saveRecent(); const c = try testWindow(path); defer c.deinit(); try testing.expect(!c.store.garbage); for ([_][]const u8{ "/w/a.zig", "/w/after.zig", "/w/late.zig", "/w/b.zig" }) |want| try testing.expect(c.recent.get(want) != null); } test "store: a closed pane comes back where it was, its rows from the pane that took them, nothing else moving; Panes lists it, marked" { if (comptime !pardes.hosted) return; var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); const path = try testPath(&tmp); defer testing.allocator.free(path); const p = try testWindow(path); defer p.deinit(); _ = try p.setTestFile("alpha\nbeta\n"); p.newScratchBelow(0); const sid = p.active; @import("File.zig").setContent(p, &p.panes[sid].?.file.?, try p.gpa.dupe(u8, "kept text\n")); const other = p.col_panes[0][1]; p.newScratchBelow(0); try testing.expect(layout.splitColumn(p, p.active, p.active, false)); const far = p.active; _ = other; p.active = 0; // Made outside a step: announced as a step's end would. pardes.ctlfs.events.announce(p); layout.compute(p); const before = p.rects; const serial = p.panes[sid].?.serial; try p.removePane(sid, null); layout.compute(p); try testing.expectEqualDeep(before[far], p.rects[far]); // Written to the store: brought back from the file, not memory. p.saveRecent(); try testing.expectEqual(@as(usize, 0), p.store.closedBySerial(serial).?.body.len); try testing.expect(p.store.reopen(p, serial, 0)); const back = p.active; layout.compute(p); try testing.expectEqualStrings("kept text\n", p.panes[back].?.file.?.content); try testing.expect(p.panes[back].?.serial != serial); // Its rect as it was, the one above as it was, the other column's // untouched: its rows came from the pane that took them alone. try testing.expectEqualDeep(before[sid], p.rects[back]); try testing.expectEqualDeep(before[0], p.rects[0]); try testing.expectEqualDeep(before[far], p.rects[far]); try testing.expect(p.store.closedBySerial(serial) == null); // Closed again: Panes lists it after the open ones, `(closed)`. const again = p.panes[back].?.serial; pardes.ctlfs.events.announce(p); try p.removePane(back, null); p.saveRecent(); p.active = 0; p.update(.{ .key = .{ .cp = ' ', .text = " " } }); for ("bb") |ch| p.update(.{ .key = .{ .cp = ch, .text = &.{ch} } }); const qid = for (p.panes, 0..) |slot, i| { if (slot) |qp| if (qp.query != null) break i; } else return error.NoQueryPane; const q = p.panes[qid].?.query.?; const row = for (q.items.items, 0..) |it, i| { if (it.ref == again) break i; } else return error.NotListed; const line = @import("modal.zig").lineSlice(p.panes[qid].?.file.?.content, @intCast(row + 1)); try testing.expect(std.mem.endsWith(u8, line, "(closed)")); // Enter on it: back, and listed no more (nor in the store when read). panes_mod.Query.land(p, qid, row); const found = for (p.panes) |slot| { if (slot) |pp| if (pp.file) |f| if (std.mem.eql(u8, f.content, "kept text\n")) break true; } else false; try testing.expect(found); try testing.expect(p.store.closedBySerial(again) == null); p.saveRecent(); p.store.inode = 0; p.store.refresh(p); try testing.expect(p.store.closedBySerial(again) == null); // Another session lists none of this one's. const w = try testWindow(path); defer w.deinit(); try testing.expectEqual(@as(usize, 0), w.store.closed.items.len); } const panes_mod = @import("panes.zig"); test "zz store timings on a 90 MiB store (STORE_TIMES=)" { const dir_z = std.c.getenv("STORE_TIMES") orelse return error.SkipZigTest; const gpa = testing.allocator; const path = try std.fs.path.join(gpa, &.{ std.mem.span(dir_z), "store" }); defer gpa.free(path); // A store as another running session leaves it: 200 files, then 90 // closed panes of 1 MiB each (this pid: alive, so a compaction keeps // what fits). { var bytes: std.ArrayList(u8) = .empty; defer bytes.deinit(gpa); try frame(gpa, &bytes, 'h', &.{magic ++ " 1"}); var line: [64]u8 = undefined; for (0..200) |i| try frame(gpa, &bytes, 'r', &.{std.fmt.bufPrint(&line, "1 1 0 0 /w/file{d}.zig", .{i}) catch unreachable}); const out = try std.Io.Dir.cwd().createFile(testing.io, path, .{}); defer out.close(testing.io); try out.writePositionalAll(testing.io, bytes.items, 0); var off: u64 = bytes.items.len; const big = try gpa.alloc(u8, 1 << 20); defer gpa.free(big); @memset(big, 'z'); for (0..90) |i| { bytes.clearRetainingCapacity(); var meta: [128]u8 = undefined; const m = std.fmt.bufPrint(&meta, "{x:0>16} {d} {d} 1 0 0 0 10\t/w/closed{d}\n", .{ @as(u64, 0xfeed), std.c.getpid(), i + 1, i }) catch unreachable; try frame(gpa, &bytes, 'c', &.{ m, big }); try out.writePositionalAll(testing.io, bytes.items, off); off += bytes.items.len; } } const t0 = nowNs(); const p = try testWindow(path); defer p.deinit(); const load_ns = nowNs() - t0; var t = nowNs(); for (0..1000) |_| p.store.refresh(p); const refresh_ns = (nowNs() - t) / 1000; t = nowNs(); var name: [32]u8 = undefined; for (0..100) |i| { const f = std.fmt.bufPrint(&name, "/w/new{d}.zig", .{i}) catch unreachable; p.recent.opened(p.recentGpa(), f); p.recent.closed(p.recentGpa(), f, 1, 1); p.saveRecent(); } const flush_ns = (nowNs() - t) / 100; const before = fileSize(path); p.store.garbage = true; p.recent.opened(p.recentGpa(), "/w/last.zig"); t = nowNs(); p.saveRecent(); const compact_ns = nowNs() - t; std.debug.print("store timings: load {d} us, refresh (unchanged) {d} ns, flush one record {d} us, compaction {d} -> {d} bytes {d} ms\n", .{ load_ns / 1000, refresh_ns, flush_ns / 1000, before, fileSize(path), compact_ns / 1_000_000, }); } fn nowNs() u64 { return @intCast(clockNs()); }