From 1eae252b80ddab0a06574de700b6b2957d3b64fc Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Thu, 5 Nov 2026 15:20:35 -0300 Subject: blob cache: a far client's blobs are kept in `cache.xitdb` (src/Db.zig's Cache, in $XDG_CACHE_HOME/pardes, else ~/.cache/pardes), a file of its own and never the state file (expendable, and up to a gigabyte): `blobs/` the texts, `seqs/` and `seen` (a sorted map, a number to its hash, the least recently used first) their order, `size` and `next`; a join's blobs are put in one transaction by the client's storing thread, with those it found there made the most recently used (a hit takes a new number), and at the client's end what it made since is put and the cache trimmed: past the cap (PARDES_BLOB_CACHE_MB, 1 GiB) the oldest go in one transaction and the file is compacted into `cache.xitdb.new` and renamed over it, where a directory of files was scanned, sorted and deleted from at each join; each use opens the file for itself (a database a thread, as xitdb asks), a text read back is hashed again (one kept wrong is a miss, and goes), and a file that is no database is started afresh; replica.zig's Cache (its directory, the hex names, put by rename, trim by scan) goes for Db.Cache; tests: replica.zig's 'a cached blob is hashed again as it is read: one cut short, corrupted, planted or longer is a miss, and goes' and 'the blob cache stays under its cap, the least recently used going first' are rewritten for the file (texts kept wrong under a hash through put, the order made by hits, and the file seen compacted), 'a blob cache that is no database is started afresh' is new, the far join test keeps its biggest blob wrong through the cache rather than a file, and daemon.py's far clients read the cache file's size, cut it short by hand where they corrupted files, and check the cap after a client's end where they planted junk files Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge --- src/Db.zig | 255 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/replica.zig | 272 ++++++++++++++++++++------------------------------------ test/daemon.py | 50 +++++------ 3 files changed, 373 insertions(+), 204 deletions(-) diff --git a/src/Db.zig b/src/Db.zig index a91a5054..846c8d96 100644 --- a/src/Db.zig +++ b/src/Db.zig @@ -762,6 +762,261 @@ fn liveSize(db: *Disk) !u64 { return live; } +// ---- a far client's blob cache ---- + +/// A far replica client's blob cache (replica.zig `Have`): `cache.xitdb` in +/// $XDG_CACHE_HOME/pardes (else ~/.cache/pardes), the texts a join was +/// sent, by their BLAKE3 hash, so a client started again or a second one on +/// this machine is not sent them again. Expendable, and up to a gigabyte, so +/// a file of its own, never the state file's: +/// +/// blobs/ the text +/// seqs/ its number in `seen` +/// seen a sorted map, a number (big endian) to its blob's hash: +/// the least recently used first (a hit takes a new number) +/// size, next the texts' bytes, the next number +/// +/// Past the cap (PARDES_BLOB_CACHE_MB, else 1 GiB) the oldest go, and the +/// file is compacted into `cache.xitdb.new` and renamed over it: one +/// transaction and one compaction, at a far client's end (`trim`), never on +/// a frame. A text read back is hashed again: one torn or changed is a miss +/// and goes, never wrong content; a file that is no database is started +/// afresh (it holds nothing a join cannot send again). Each use opens the +/// file for itself: a client's reader, its storing thread and its end each +/// have a database of their own, as xitdb asks. +pub const Cache = struct { + path_buf: [4096]u8 = undefined, + path_len: usize = 0, + /// Bytes it keeps at most. + cap: u64, + + pub const name = "cache.xitdb"; + + /// The cache's place, its directory made: null with no home for it. + pub fn open(io: std.Io) ?Cache { + var c: Cache = .{ .cap = (if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024) << 20 }; + const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else ""; + const where = (if (base.len > 0 and base[0] == '/') + std.fmt.bufPrint(&c.path_buf, "{s}/pardes/" ++ name, .{base}) + else if (std.c.getenv("HOME")) |home| + std.fmt.bufPrint(&c.path_buf, "{s}/.cache/pardes/" ++ name, .{std.mem.span(home)}) + else + return null) catch return null; + std.Io.Dir.cwd().createDirPath(io, std.fs.path.dirname(where).?) catch return null; + c.path_len = where.len; + return c; + } + + /// The cache at `where` (a test's). + pub fn of(where: []const u8, cap: u64) Cache { + var c: Cache = .{ .cap = cap, .path_len = where.len }; + @memcpy(c.path_buf[0..where.len], where); + return c; + } + + pub fn path(c: *const Cache) []const u8 { + return c.path_buf[0..c.path_len]; + } + + /// The cache open for one use. + const Use = struct { + file: std.Io.File, + buffer: std.Io.Writer.Allocating, + db: Disk, + }; + + fn use(c: *const Cache, io: std.Io, u: *Use) !void { + const cwd = std.Io.Dir.cwd(); + u.buffer = .init(std.heap.c_allocator); + errdefer u.buffer.deinit(); + for (0..2) |_| { + u.file = try cwd.createFile(io, c.path(), .{ .read = true, .truncate = false }); + u.db = Disk.init(io, .{ .file = u.file, .buffer = &u.buffer, .fsync = false, .hash_id = hash_id }) catch |err| { + u.file.close(io); + // No database: started afresh, as nothing in it is precious. + cwd.deleteFile(io, c.path()) catch return err; + continue; + }; + return; + } + return error.InvalidDatabase; + } + + fn done(u: *Use, io: std.Io) void { + u.file.close(io); + u.buffer.deinit(); + } + + fn mapAt(root: Disk.HashMap(.read_only), field: []const u8) !?Disk.HashMap(.read_only) { + const c = (try root.getCursor(key(field))) orelse return null; + return try Disk.HashMap(.read_only).init(c); + } + + /// The text of `hash` read into `out` (its length) and hashed again. + /// One of another length or content is a miss, and goes. + pub fn get(c: *const Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const ok = ok: { + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch break :ok false; + const blobs = (mapAt(root, "blobs") catch break :ok false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch break :ok false) orelse return false; + if ((at.count() catch break :ok false) != out.len) break :ok false; + _ = at.readBytes(out) catch break :ok false; + var d: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(out, &d, .{}); + break :ok std.mem.eql(u8, &d, hash); + }; + if (!ok) drop(&u, io, hash); + return ok; + } + + /// Whether a text of `len` bytes is there under `hash` (not read). + pub fn has(c: *const Cache, io: std.Io, hash: *const [32]u8, len: usize) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch return false; + const blobs = (mapAt(root, "blobs") catch return false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch return false) orelse return false; + return (at.count() catch return false) == len; + } + + /// The text of `hash`, gone (one read back wrong). + fn drop(u: *Use, io: std.Io, hash: *const [32]u8) void { + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + hash: *const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const blobs = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const at = (try blobs.getCursor(blobKey(ctx.hash))) orelse return; + const len = try at.count(); + _ = try blobs.remove(blobKey(ctx.hash)); + const size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + try root.put(key("size"), .{ .uint = size -| len }); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + if (try seqs.getCursor(blobKey(ctx.hash))) |s| { + const was = try s.readUint(); + _ = try seqs.remove(blobKey(ctx.hash)); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + _ = try seen.remove(&seqKey(was)); + } + } + }{ .hash = hash }) catch {}; + } + + fn seqKey(seq: u64) [8]u8 { + var k: [8]u8 = undefined; + std.mem.writeInt(u64, &k, seq, .big); + return k; + } + + /// `blobs` kept (each text not there yet, under its hash) and `hits`, + /// texts read from here, made the most recently used: one transaction, + /// the file's lock waited for (a storing thread's, a client's end's). + pub fn put(c: *const Cache, io: std.Io, blobs: []const pardes.walk.Blob, hits: []const [32]u8) void { + if (blobs.len == 0 and hits.len == 0) return; + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + blobs: []const pardes.walk.Blob, + hits: []const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + var next = if (try root.getCursor(key("next"))) |n| try n.readUint() else 0; + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + for (ctx.blobs) |*b| { + if (try texts.getCursor(blobKey(&b.hash)) != null) continue; + try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes }); + try seqs.put(blobKey(&b.hash), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = &b.hash }); + next += 1; + size += b.bytes.len; + } + for (ctx.hits) |*h| { + const at = (try seqs.getCursor(blobKey(h))) orelse continue; + _ = try seen.remove(&seqKey(try at.readUint())); + try seqs.put(blobKey(h), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = h }); + next += 1; + } + try root.put(key("next"), .{ .uint = next }); + try root.put(key("size"), .{ .uint = size }); + } + }{ .blobs = blobs, .hits = hits }) catch {}; + } + + /// Under the cap: the least recently used texts gone, in one + /// transaction, and the file compacted into `.new` and renamed + /// over it once any went (what no moment holds is the file's growth). + pub fn trim(c: *const Cache, io: std.Io) void { + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + var went = false; + u.db.runContext(struct { + cap: u64, + went: *bool, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + if (size <= ctx.cap) return; + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + // The oldest numbers, read before any goes. + var gone: std.ArrayList(struct { seq: [8]u8, hash: [32]u8 }) = .empty; + defer gone.deinit(std.heap.c_allocator); + var left = size; + var it = try seen.iterator(); + while (left > ctx.cap) { + const kv_at = (try it.next()) orelse break; + const kv = try kv_at.readKeyValuePair(); + var g: @TypeOf(gone.items[0]) = undefined; + if ((try kv.key_cursor.readBytes(&g.seq)).len != 8 or (try kv.value_cursor.readBytes(&g.hash)).len != 32) continue; + if (try texts.getCursor(blobKey(&g.hash))) |at| left -|= try at.count(); + try gone.append(std.heap.c_allocator, g); + } + for (gone.items) |*g| { + if (try texts.getCursor(blobKey(&g.hash))) |at| size -|= try at.count(); + _ = try texts.remove(blobKey(&g.hash)); + _ = try seqs.remove(blobKey(&g.hash)); + _ = try seen.remove(&g.seq); + } + try root.put(key("size"), .{ .uint = size }); + ctx.went.* = gone.items.len > 0; + } + }{ .cap = c.cap, .went = &went }) catch return; + if (!went) return; + var new_buf: [4200]u8 = undefined; + const new = std.fmt.bufPrint(&new_buf, "{s}.new", .{c.path()}) catch return; + const cwd = std.Io.Dir.cwd(); + const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return; + var buffer: std.Io.Writer.Allocating = .init(std.heap.c_allocator); + defer buffer.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(std.heap.c_allocator); + defer offsets.deinit(); + _ = u.db.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = false, .hash_id = hash_id }, &offsets) catch { + file.close(io); + cwd.deleteFile(io, new) catch {}; + return; + }; + file.close(io); + cwd.rename(new, cwd, c.path(), io) catch cwd.deleteFile(io, new) catch {}; + } +}; + // ---- tests ---- const testing = std.testing; diff --git a/src/replica.zig b/src/replica.zig index 5bcc0a53..b98d792b 100644 --- a/src/replica.zig +++ b/src/replica.zig @@ -2421,6 +2421,8 @@ fn joinOn(gpa: Allocator, fd: c_int, session: []const u8, why: []u8, have: ?*Hav pub const Have = struct { io: std.Io, cache: ?Cache = null, + /// The blobs a join found on disk: the most recently used once stored. + hits: std.ArrayListUnmanaged([32]u8) = .empty, /// The core it leaves (`leaving`), each blob hashed once when one of its /// length is looked for. ponytail: a scan a lookup, fine to some /// thousands of blobs; a map by length would take over past that. @@ -2438,7 +2440,7 @@ pub const Have = struct { h.left(); h.old.deinit(std.heap.c_allocator); h.old_hash.deinit(std.heap.c_allocator); - if (h.cache) |c| c.dir.close(h.io); + h.hits.deinit(std.heap.c_allocator); } fn wait(h: *Have) void { @@ -2475,189 +2477,83 @@ pub const Have = struct { @memcpy(out, b); return .memory; } - if (h.cache) |c| if (c.get(h.io, hash, out)) return .disk; + if (h.cache) |*c| if (c.get(h.io, hash, out)) { + h.hits.append(std.heap.c_allocator, hash.*) catch {}; + return .disk; + }; return null; } /// The blobs a join was sent, written to the cache on a thread of their - /// own, which frees the dump (`gpa`'s) and the list after: the first - /// frame waits for none of it. + /// own (with those it found there, made the most recently used), which + /// frees the dump (`gpa`'s) and the lists after: the first frame waits + /// for none of it. pub fn store(h: *Have, gpa: Allocator, j: *Joined) void { const dump = j.dump; const got = j.got; j.dump = &.{}; j.got = &.{}; h.wait(); + var hits = h.hits; + h.hits = .empty; const Store = struct { - fn run(c: Cache, io: std.Io, a: Allocator, bytes: []u8, list: []Got) void { + fn run(c: Cache, io: std.Io, a: Allocator, bytes: []u8, list: []Got, found: std.ArrayListUnmanaged([32]u8)) void { + var seen = found; defer { a.free(bytes); std.heap.c_allocator.free(list); + seen.deinit(std.heap.c_allocator); } - var incoming: u64 = 0; - for (list) |g| incoming += g.len; - c.trim(io, @min(incoming, c.cap)); + var blobs: std.ArrayListUnmanaged(walk.Blob) = .empty; + defer blobs.deinit(std.heap.c_allocator); var put: u64 = 0; for (list) |g| { if (put + g.len > c.cap) continue; put += g.len; - c.put(io, &g.hash, bytes[g.pos..][0..g.len]); + blobs.append(std.heap.c_allocator, .{ .hash = g.hash, .bytes = bytes[g.pos..][0..g.len] }) catch return; } + c.put(io, blobs.items, seen.items); } }; const c = h.cache orelse { gpa.free(dump); std.heap.c_allocator.free(got); + hits.deinit(std.heap.c_allocator); return; }; - h.storing = std.Thread.spawn(.{}, Store.run, .{ c, h.io, gpa, dump, got }) catch { - Store.run(c, h.io, gpa, dump, got); + h.storing = std.Thread.spawn(.{}, Store.run, .{ c, h.io, gpa, dump, got, hits }) catch { + Store.run(c, h.io, gpa, dump, got, hits); return; }; } /// `core`'s blobs the cache lacks, written there: what this client made - /// since it joined (an edit's text, its undo), for its next start. + /// since it joined (an edit's text, its undo), for its next start; then + /// the cache trimmed under its cap, at this client's end. pub fn keep(h: *Have, core: *const Pardes) void { h.wait(); const c = h.cache orelse return; var all: std.ArrayListUnmanaged([]const u8) = .empty; defer all.deinit(std.heap.c_allocator); walk.blobs(Pardes, core, &all); - var missing: std.ArrayListUnmanaged(struct { bytes: []const u8, hash: [32]u8 }) = .empty; + var missing: std.ArrayListUnmanaged(walk.Blob) = .empty; defer missing.deinit(std.heap.c_allocator); - var incoming: u64 = 0; + var put: u64 = 0; for (all.items) |b| { var d: [32]u8 = undefined; std.crypto.hash.Blake3.hash(b, &d, .{}); - if (c.has(h.io, &d, b.len)) continue; + if (put + b.len > c.cap or c.has(h.io, &d, b.len)) continue; + put += b.len; missing.append(std.heap.c_allocator, .{ .bytes = b, .hash = d }) catch return; - incoming += b.len; - } - if (missing.items.len == 0) return; - c.trim(h.io, @min(incoming, c.cap)); - var put: u64 = 0; - for (missing.items) |m| { - if (put + m.bytes.len > c.cap) continue; - put += m.bytes.len; - c.put(h.io, &m.hash, m.bytes); } + c.put(h.io, missing.items, h.hits.items); + h.hits.clearRetainingCapacity(); + c.trim(h.io); } }; -/// A far client's blobs on disk, a file each named by its BLAKE3 hash in -/// hex, in $XDG_CACHE_HOME/pardes/blobs (else ~/.cache/pardes/blobs): what -/// a client started again, or a second one on this machine, is not sent -/// again. The cache directory and not the runtime one: the blobs are worth -/// keeping across a logout or a reboot and safe to lose, and they can be -/// hundreds of megabytes, which the runtime directory (a tmpfs, often -/// capped at a tenth of memory and emptied at logout) is not for. A file -/// read back is hashed again: one cut short, corrupted or planted is a -/// miss and goes, never wrong content. -pub const Cache = struct { - dir: std.Io.Dir, - /// Bytes it keeps at most: PARDES_BLOB_CACHE_MB, else 1 GiB. - cap: u64, - - pub fn open(io: std.Io) ?Cache { - var buf: [4096]u8 = undefined; - const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else ""; - const path = if (base.len > 0 and base[0] == '/') - std.fmt.bufPrint(&buf, "{s}/pardes/blobs", .{base}) catch return null - else if (std.c.getenv("HOME")) |home| - std.fmt.bufPrint(&buf, "{s}/.cache/pardes/blobs", .{std.mem.span(home)}) catch return null - else - return null; - const cwd = std.Io.Dir.cwd(); - cwd.createDirPath(io, std.fs.path.dirname(path).?) catch return null; - cwd.createDir(io, path, .fromMode(0o700)) catch |err| if (err != error.PathAlreadyExists) return null; - const mb = if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024; - return .{ .dir = cwd.openDir(io, path, .{ .iterate = true }) catch return null, .cap = mb << 20 }; - } - - fn name(hash: *const [32]u8) [64]u8 { - return std.fmt.bytesToHex(hash.*, .lower); - } - - /// The blob of `hash` read into `out` (its length) and hashed again. - /// A file of another length or content is a miss, and goes; a hit's - /// time is made now, so the least recently used go first (`trim`). - pub fn get(c: Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool { - const file = name(hash); - var f = c.dir.openFile(io, &file, .{}) catch return false; - const ok = ok: { - defer f.close(io); - const st = f.stat(io) catch break :ok false; - if (st.size != out.len) break :ok false; - var reader = f.reader(io, &.{}); - const n = reader.interface.readSliceShort(out) catch break :ok false; - if (n != out.len) break :ok false; - var d: [32]u8 = undefined; - std.crypto.hash.Blake3.hash(out, &d, .{}); - break :ok std.mem.eql(u8, &d, hash); - }; - if (!ok) { - c.dir.deleteFile(io, &file) catch {}; - return false; - } - c.dir.setTimestampsNow(io, &file, .{}) catch {}; - return true; - } - - /// Whether a file of `len` bytes is there under `hash` (not read). - pub fn has(c: Cache, io: std.Io, hash: *const [32]u8, len: usize) bool { - const file = name(hash); - const st = c.dir.statFile(io, &file, .{}) catch return false; - return st.size == len; - } - - var tmp_serial: std.atomic.Value(u32) = .init(0); - - /// `bytes` kept under `hash`: written aside and renamed into place, so - /// that a reader never meets half a file (and would hash it wrong). - pub fn put(c: Cache, io: std.Io, hash: *const [32]u8, bytes: []const u8) void { - const file = name(hash); - if (c.has(io, hash, bytes.len)) return; - var buf: [96]u8 = undefined; - const tmp = std.fmt.bufPrint(&buf, ".tmp-{d}-{d}-{s}", .{ std.c.getpid(), tmp_serial.fetchAdd(1, .monotonic), file[0..16] }) catch return; - c.dir.writeFile(io, .{ .sub_path = tmp, .data = bytes }) catch { - c.dir.deleteFile(io, tmp) catch {}; - return; - }; - c.dir.rename(tmp, c.dir, &file, io) catch c.dir.deleteFile(io, tmp) catch {}; - } - - /// Room for `incoming` more bytes under the cap, the least recently - /// used files deleted first (a hit makes its file's time now). - /// ponytail: the directory scanned and sorted at each join's store; - /// fine to tens of thousands of blobs, where an index would take over. - pub fn trim(c: Cache, io: std.Io, incoming: u64) void { - const File = struct { name: [80]u8, len: u8, size: u64, mtime: i96 }; - var files: std.ArrayListUnmanaged(File) = .empty; - defer files.deinit(std.heap.c_allocator); - var total: u64 = 0; - var it = c.dir.iterate(); - while (it.next(io) catch null) |entry| { - if (entry.kind != .file or entry.name.len > 80) continue; - const st = c.dir.statFile(io, entry.name, .{}) catch continue; - var f: File = .{ .name = undefined, .len = @intCast(entry.name.len), .size = st.size, .mtime = st.mtime.nanoseconds }; - @memcpy(f.name[0..f.len], entry.name); - files.append(std.heap.c_allocator, f) catch return; - total += st.size; - } - if (total + incoming <= c.cap) return; - std.mem.sort(File, files.items, {}, struct { - fn older(_: void, a: File, b: File) bool { - return a.mtime < b.mtime; - } - }.older); - for (files.items) |f| { - if (total + incoming <= c.cap) break; - c.dir.deleteFile(io, f.name[0..f.len]) catch continue; - total -= f.size; - } - } -}; +/// A far client's blobs on disk (Db.zig): cache.xitdb, by BLAKE3 hash. +pub const Cache = pardes.Db.Cache; fn said(why: []u8, comptime fmt: []const u8, args: anytype) ?Joined { _ = std.fmt.bufPrint(why, fmt ++ "\x00", args) catch {}; @@ -3619,75 +3515,93 @@ test "a join's dump, its blobs put back where they go, is the session's dump" { try std.testing.expectEqualSlices(u8, whole, joined.items); } -test "a cached blob is hashed again as it is read: one cut short, corrupted, planted or longer is a miss, and goes" { +/// The blob cache of a test, in its own directory (never the user's). +fn testCache(tmp: *std.testing.TmpDir, buf: *[4200]u8, cap: u64) !Cache { + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir = dir_buf[0..try tmp.dir.realPath(std.testing.io, &dir_buf)]; + return Cache.of(try std.fmt.bufPrint(buf, "{s}/" ++ Cache.name, .{dir}), cap); +} + +test "a cached blob is hashed again as it is read: one kept cut short, corrupted, planted or longer is a miss, and goes" { const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - const c: Cache = .{ .dir = tmp.dir, .cap = 1 << 20 }; + var path: [4200]u8 = undefined; + const c = try testCache(&tmp, &path, 1 << 20); var text: [6000]u8 = undefined; for (&text, 0..) |*b, i| b.* = @truncate(i *% 31 +% i / 7); var hash: [32]u8 = undefined; std.crypto.hash.Blake3.hash(&text, &hash, .{}); - const file = Cache.name(&hash); var out: [6000]u8 = undefined; try std.testing.expect(!c.get(io, &hash, &out)); - c.put(io, &hash, &text); - try std.testing.expect(c.has(io, &hash, text.len)); - try std.testing.expect(c.get(io, &hash, &out)); - try std.testing.expectEqualSlices(u8, &text, &out); var corrupted = text; corrupted[4000] ^= 0x20; const planted: [6000]u8 = @splat('x'); var longer: [6001]u8 = undefined; @memcpy(longer[0..6000], &text); longer[6000] = '\n'; + // Each kept under the text's hash, as a torn write or another hand + // would leave it: read, it is a miss, and gone. for ([_][]const u8{ text[0..3000], &corrupted, &planted, &longer }) |bad| { - try tmp.dir.writeFile(io, .{ .sub_path = &file, .data = bad }); + c.put(io, &.{.{ .hash = hash, .bytes = bad }}, &.{}); + try std.testing.expect(c.has(io, &hash, bad.len)); @memset(&out, 0); try std.testing.expect(!c.get(io, &hash, &out)); - try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(io, &file, .{})); + try std.testing.expect(!c.has(io, &hash, bad.len)); } - // ...and put back, it is read again. - c.put(io, &hash, &text); + // ...and put right, it is read again. + c.put(io, &.{.{ .hash = hash, .bytes = &text }}, &.{}); try std.testing.expect(c.get(io, &hash, &out)); try std.testing.expectEqualSlices(u8, &text, &out); } -test "the blob cache stays under its cap, the least recently used going first" { +test "the blob cache stays under its cap, the least recently used going first, the file compacted" { const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - const c: Cache = .{ .dir = tmp.dir, .cap = 3 * 5000 }; - var blobs: [3][5000]u8 = undefined; - var hashes: [3][32]u8 = undefined; + var path: [4200]u8 = undefined; + const c = try testCache(&tmp, &path, 3 * 5000); + var blobs: [4][5000]u8 = undefined; + var hashes: [4][32]u8 = undefined; for (&blobs, &hashes, 0..) |*b, *h, k| { @memset(b, @intCast('a' + k)); std.crypto.hash.Blake3.hash(b, h, .{}); - c.put(io, h, b); - // Each older than the next, a thousand seconds apart. - const name = Cache.name(h); - try tmp.dir.setTimestamps(io, &name, .{ .modify_timestamp = .{ .new = .{ .nanoseconds = @as(i96, @intCast(k + 1)) * 1000 * std.time.ns_per_s } } }); } + for (0..3) |k| c.put(io, &.{.{ .hash = hashes[k], .bytes = &blobs[k] }}, &.{}); // The oldest read: it is the newest now. var out: [5000]u8 = undefined; try std.testing.expect(c.get(io, &hashes[0], &out)); - c.trim(io, 5000); + c.put(io, &.{}, &.{hashes[0]}); + // Under the cap: nothing goes. + c.trim(io); + for (hashes[0..3]) |*h| try std.testing.expect(c.has(io, h, 5000)); + // One more is past it: the least recently used goes. + c.put(io, &.{.{ .hash = hashes[3], .bytes = &blobs[3] }}, &.{}); + const before = (try std.Io.Dir.cwd().statFile(io, c.path(), .{})).size; + c.trim(io); try std.testing.expect(c.has(io, &hashes[0], 5000)); try std.testing.expect(!c.has(io, &hashes[1], 5000)); - try std.testing.expect(c.has(io, &hashes[2], 5000)); - // A file not a blob's counts, and goes by its time as well. - const junk: [9000]u8 = @splat(0); - try tmp.dir.writeFile(io, .{ .sub_path = ".tmp-1-1-0123456789abcdef", .data = &junk }); - try tmp.dir.setTimestamps(io, ".tmp-1-1-0123456789abcdef", .{ .modify_timestamp = .{ .new = .{ .nanoseconds = 500 * std.time.ns_per_s } } }); - c.trim(io, 5000); - try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(io, ".tmp-1-1-0123456789abcdef", .{})); - try std.testing.expect(c.has(io, &hashes[0], 5000) and c.has(io, &hashes[2], 5000)); - // Under the cap with room to spare: nothing goes. - c.trim(io, 0); - try std.testing.expect(c.has(io, &hashes[0], 5000) and c.has(io, &hashes[2], 5000)); - // Room for the whole cap: everything goes, the older first. - c.trim(io, c.cap); - try std.testing.expect(!c.has(io, &hashes[0], 5000) and !c.has(io, &hashes[2], 5000)); + try std.testing.expect(c.has(io, &hashes[2], 5000) and c.has(io, &hashes[3], 5000)); + try std.testing.expect((try std.Io.Dir.cwd().statFile(io, c.path(), .{})).size < before); + // Room for nothing: everything goes. + Cache.of(c.path(), 0).trim(io); + for (&hashes) |*h| try std.testing.expect(!c.has(io, h, 5000)); +} + +test "a blob cache that is no database is started afresh" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path: [4200]u8 = undefined; + const c = try testCache(&tmp, &path, 1 << 20); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = c.path(), .data = "xit but not a database, cut short" }); + const text: [5000]u8 = @splat('t'); + var hash: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(&text, &hash, .{}); + var out: [5000]u8 = undefined; + try std.testing.expect(!c.get(io, &hash, &out)); + c.put(io, &.{.{ .hash = hash, .bytes = &text }}, &.{}); + try std.testing.expect(c.get(io, &hash, &out)); } /// A far join over a socket pair, the session's writer and reader threads @@ -3785,9 +3699,10 @@ test "a far join is sent each blob once: joined again with them on disk or in th } try std.testing.expect(distinct.count() >= 3); - var tmp = std.testing.tmpDir(.{ .iterate = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - var have: Have = .{ .io = io, .cache = .{ .dir = tmp.dir, .cap = 64 << 20 } }; + var path: [4200]u8 = undefined; + var have: Have = .{ .io = io, .cache = try testCache(&tmp, &path, 64 << 20) }; defer { have.cache = null; have.deinit(); @@ -3813,12 +3728,13 @@ test "a far join is sent each blob once: joined again with them on disk or in th try std.testing.expectEqual(distinct.count(), j.from_disk); try std.testing.expectEqual(@as(u64, 0), j.sent_bytes); endJoin(&j, &have); - // The biggest's file corrupted: it alone is sent again. - const name = Cache.name(&biggest); + // The biggest kept wrong (read back once as another length, it goes; + // then junk under its hash): it alone is sent again. const junk = try gpa.alloc(u8, biggest_len); defer gpa.free(junk); @memset(junk, 'z'); - try tmp.dir.writeFile(io, .{ .sub_path = &name, .data = junk }); + try std.testing.expect(!have.cache.?.get(io, &biggest, junk[0 .. biggest_len - 1])); + have.cache.?.put(io, &.{.{ .hash = biggest, .bytes = junk }}, &.{}); j = try farJoinOnce(&ps, a.core, &have); try std.testing.expectEqualSlices(u8, whole, j.dump); try std.testing.expectEqual(distinct.count() - 1, j.from_disk); diff --git a/test/daemon.py b/test/daemon.py index 0c4c2f3c..074c860c 100644 --- a/test/daemon.py +++ b/test/daemon.py @@ -820,10 +820,15 @@ def joins(path): return out +def cache_file(cache): + """The blob cache (a far client's, XDG_CACHE_HOME/pardes/cache.xitdb).""" + return cache / 'pardes' / 'cache.xitdb' + + def cached(cache): - """The blob cache's files (a far client's, XDG_CACHE_HOME/pardes/blobs).""" - d = cache / 'pardes' / 'blobs' - return {p.name: p.stat().st_size for p in d.iterdir() if p.is_file()} if d.exists() else {} + """The blob cache's size in bytes, 0 with none.""" + f = cache_file(cache) + return f.stat().st_size if f.exists() else 0 def far(binary): @@ -832,9 +837,10 @@ def far(binary): blob and keeps them; a second on the same machine, and one put off the session's state that joins again, are sent none they hold, from disk and from the core they leave; one started after the first ended gets - what it made from disk; one after the cache was corrupted and cut short - by hand, and one with the cache full, are sent what they lack. Every - client's digest is the session's once quiet.""" + what it made from disk; one after the cache was cut short by hand (a + file that is no database: started afresh), and one with the cache full, + are sent what they lack. Every client's digest is the session's once + quiet.""" scratch = Path(__file__).resolve().parent.parent.parent / '.scratch' / 'daemon' / 'dmn' scratch.mkdir(parents=True, exist_ok=True) root = Path(tempfile.mkdtemp(prefix='rf-', dir=scratch)) @@ -895,7 +901,8 @@ def far(binary): # A: the first, its cache empty: every blob sent, and kept. ta, a = client('a') typed('a', ta, b'from-A') - wait_for('client A\'s blobs on disk', lambda: len(cached(cache)) >= 2) + # The notes and the big file, a megabyte and more between them. + wait_for('client A\'s blobs on disk', lambda: cached(cache) > 1 << 20) time.sleep(.5) # B: a second client on this machine, and put off the session's # state once quiet: its first join from disk, its rejoin from the @@ -921,37 +928,28 @@ def far(binary): detach('c', tc, c) c_join = joins(root / 'c.stats')[0] assert c_join['from_disk'] >= c_join['blobs'] - 1 and c_join['sent_kb'] < 64, c_join - # D: the cache corrupted and cut short by hand: a miss each, sent again. - files = sorted(cached(cache).items(), key=lambda f: -f[1]) - for k, (name, size) in enumerate(files): - p = cache / 'pardes' / 'blobs' / name - if k % 2: - p.write_bytes(p.read_bytes()[:size // 2]) - else: - data = bytearray(p.read_bytes()) - data[size // 2] ^= 0x55 - p.write_bytes(bytes(data)) + # D: the cache cut short by hand: no database, started afresh, every + # blob sent again. + whole = cache_file(cache).read_bytes() + cache_file(cache).write_bytes(whole[:len(whole) // 2]) td, d = client('d') typed('d', td, b'from-D') detach('d', td, d) d_join = joins(root / 'd.stats')[0] assert d_join['from_disk'] == 0 and d_join['sent_kb'] > 100, d_join - # E: the cache full (a 1 MiB cap, held by older files): they go first, - # and the cache stays under its cap. - for k in range(2): - p = cache / 'pardes' / 'blobs' / f'junk-{k}' - p.write_bytes(b'j' * (512 << 10)) - os.utime(p, (1000 + k, 1000 + k)) + # E: the cache full (D's, past a 1 MiB cap): the oldest go at its end, + # the file compacted, under its cap but for xitdb's own few KB. + before_e = cached(cache) + assert before_e > 1 << 20, before_e te, e = client('e', {'PARDES_BLOB_CACHE_MB': '1'}) typed('e', te, b'from-E') detach('e', te, e) - left = cached(cache) - assert not [n for n in left if n.startswith('junk')] and sum(left.values()) <= 1 << 20, left + assert cached(cache) <= (1 << 20) + (64 << 10), cached(cache) tf, f = client('f', {'PARDES_BLOB_CACHE_MB': '1'}) typed('f', tf, b'from-F') detach('f', tf, f) f_join = joins(root / 'f.stats')[0] - assert f_join['from_disk'] > 0 and sum(cached(cache).values()) <= 1 << 20, (f_join, cached(cache)) + assert f_join['from_disk'] > 0 and cached(cache) <= (1 << 20) + (64 << 10), (f_join, cached(cache)) far_joins(root / 'pardes-replica.sock', 'files', binary, 200) # Through ssh (a stand-in forwarding as `ssh -L` does): a client # refused and one detached each end their ssh and its socket (one -- cgit v1.3