diff options
| author | Gabriel Schneider <[email protected]> | 2026-11-05 15:20:35 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-11-05 19:16:37 -0300 |
| commit | 1eae252b80ddab0a06574de700b6b2957d3b64fc (patch) | |
| tree | c37ca247d17ade01d55129a448d38f7cc5b99aaa /src/Db.zig | |
| parent | 15a1b94eaa73747afaa299262a2d6e4fd6af42fd (diff) | |
| download | pardes-1eae252b80ddab0a06574de700b6b2957d3b64fc.tar.gz pardes-1eae252b80ddab0a06574de700b6b2957d3b64fc.zip | |
blob cache: a far client's blobs are kept in `cache.xitdb` (src/Db.zig's Cache, in $XDG_CACHE_HOME/pardes, else ~/.cache/pardes), a file of its own and never the state file (expendable, and up to a gigabyte): `blobs/<hash>` the texts, `seqs/<hash>` and `seen` (a sorted map, a number to its hash, the least recently used first) their order, `size` and `next`; a join's blobs are put in one transaction by the client's storing thread, with those it found there made the most recently used (a hit takes a new number), and at the client's end what it made since is put and the cache trimmed: past the cap (PARDES_BLOB_CACHE_MB, 1 GiB) the oldest go in one transaction and the file is compacted into `cache.xitdb.new` and renamed over it, where a directory of files was scanned, sorted and deleted from at each join; each use opens the file for itself (a database a thread, as xitdb asks), a text read back is hashed again (one kept wrong is a miss, and goes), and a file that is no database is started afresh; replica.zig's Cache (its directory, the hex names, put by rename, trim by scan) goes for Db.Cache; tests: replica.zig's 'a cached blob is hashed again as it is read: one cut short, corrupted, planted or longer is a miss, and goes' and 'the blob cache stays under its cap, the least recently used going first' are rewritten for the file (texts kept wrong under a hash through put, the order made by hits, and the file seen compacted), 'a blob cache that is no database is started afresh' is new, the far join test keeps its biggest blob wrong through the cache rather than a file, and daemon.py's far clients read the cache file's size, cut it short by hand where they corrupted files, and check the cap after a client's end where they planted junk files
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge
Diffstat (limited to 'src/Db.zig')
| -rw-r--r-- | src/Db.zig | 255 |
1 files changed, 255 insertions, 0 deletions
@@ -762,6 +762,261 @@ fn liveSize(db: *Disk) !u64 { return live; } +// ---- a far client's blob cache ---- + +/// A far replica client's blob cache (replica.zig `Have`): `cache.xitdb` in +/// $XDG_CACHE_HOME/pardes (else ~/.cache/pardes), the texts a join was +/// sent, by their BLAKE3 hash, so a client started again or a second one on +/// this machine is not sent them again. Expendable, and up to a gigabyte, so +/// a file of its own, never the state file's: +/// +/// blobs/<hash> the text +/// seqs/<hash> its number in `seen` +/// seen a sorted map, a number (big endian) to its blob's hash: +/// the least recently used first (a hit takes a new number) +/// size, next the texts' bytes, the next number +/// +/// Past the cap (PARDES_BLOB_CACHE_MB, else 1 GiB) the oldest go, and the +/// file is compacted into `cache.xitdb.new` and renamed over it: one +/// transaction and one compaction, at a far client's end (`trim`), never on +/// a frame. A text read back is hashed again: one torn or changed is a miss +/// and goes, never wrong content; a file that is no database is started +/// afresh (it holds nothing a join cannot send again). Each use opens the +/// file for itself: a client's reader, its storing thread and its end each +/// have a database of their own, as xitdb asks. +pub const Cache = struct { + path_buf: [4096]u8 = undefined, + path_len: usize = 0, + /// Bytes it keeps at most. + cap: u64, + + pub const name = "cache.xitdb"; + + /// The cache's place, its directory made: null with no home for it. + pub fn open(io: std.Io) ?Cache { + var c: Cache = .{ .cap = (if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024) << 20 }; + const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else ""; + const where = (if (base.len > 0 and base[0] == '/') + std.fmt.bufPrint(&c.path_buf, "{s}/pardes/" ++ name, .{base}) + else if (std.c.getenv("HOME")) |home| + std.fmt.bufPrint(&c.path_buf, "{s}/.cache/pardes/" ++ name, .{std.mem.span(home)}) + else + return null) catch return null; + std.Io.Dir.cwd().createDirPath(io, std.fs.path.dirname(where).?) catch return null; + c.path_len = where.len; + return c; + } + + /// The cache at `where` (a test's). + pub fn of(where: []const u8, cap: u64) Cache { + var c: Cache = .{ .cap = cap, .path_len = where.len }; + @memcpy(c.path_buf[0..where.len], where); + return c; + } + + pub fn path(c: *const Cache) []const u8 { + return c.path_buf[0..c.path_len]; + } + + /// The cache open for one use. + const Use = struct { + file: std.Io.File, + buffer: std.Io.Writer.Allocating, + db: Disk, + }; + + fn use(c: *const Cache, io: std.Io, u: *Use) !void { + const cwd = std.Io.Dir.cwd(); + u.buffer = .init(std.heap.c_allocator); + errdefer u.buffer.deinit(); + for (0..2) |_| { + u.file = try cwd.createFile(io, c.path(), .{ .read = true, .truncate = false }); + u.db = Disk.init(io, .{ .file = u.file, .buffer = &u.buffer, .fsync = false, .hash_id = hash_id }) catch |err| { + u.file.close(io); + // No database: started afresh, as nothing in it is precious. + cwd.deleteFile(io, c.path()) catch return err; + continue; + }; + return; + } + return error.InvalidDatabase; + } + + fn done(u: *Use, io: std.Io) void { + u.file.close(io); + u.buffer.deinit(); + } + + fn mapAt(root: Disk.HashMap(.read_only), field: []const u8) !?Disk.HashMap(.read_only) { + const c = (try root.getCursor(key(field))) orelse return null; + return try Disk.HashMap(.read_only).init(c); + } + + /// The text of `hash` read into `out` (its length) and hashed again. + /// One of another length or content is a miss, and goes. + pub fn get(c: *const Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const ok = ok: { + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch break :ok false; + const blobs = (mapAt(root, "blobs") catch break :ok false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch break :ok false) orelse return false; + if ((at.count() catch break :ok false) != out.len) break :ok false; + _ = at.readBytes(out) catch break :ok false; + var d: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(out, &d, .{}); + break :ok std.mem.eql(u8, &d, hash); + }; + if (!ok) drop(&u, io, hash); + return ok; + } + + /// Whether a text of `len` bytes is there under `hash` (not read). + pub fn has(c: *const Cache, io: std.Io, hash: *const [32]u8, len: usize) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch return false; + const blobs = (mapAt(root, "blobs") catch return false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch return false) orelse return false; + return (at.count() catch return false) == len; + } + + /// The text of `hash`, gone (one read back wrong). + fn drop(u: *Use, io: std.Io, hash: *const [32]u8) void { + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + hash: *const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const blobs = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const at = (try blobs.getCursor(blobKey(ctx.hash))) orelse return; + const len = try at.count(); + _ = try blobs.remove(blobKey(ctx.hash)); + const size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + try root.put(key("size"), .{ .uint = size -| len }); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + if (try seqs.getCursor(blobKey(ctx.hash))) |s| { + const was = try s.readUint(); + _ = try seqs.remove(blobKey(ctx.hash)); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + _ = try seen.remove(&seqKey(was)); + } + } + }{ .hash = hash }) catch {}; + } + + fn seqKey(seq: u64) [8]u8 { + var k: [8]u8 = undefined; + std.mem.writeInt(u64, &k, seq, .big); + return k; + } + + /// `blobs` kept (each text not there yet, under its hash) and `hits`, + /// texts read from here, made the most recently used: one transaction, + /// the file's lock waited for (a storing thread's, a client's end's). + pub fn put(c: *const Cache, io: std.Io, blobs: []const pardes.walk.Blob, hits: []const [32]u8) void { + if (blobs.len == 0 and hits.len == 0) return; + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + blobs: []const pardes.walk.Blob, + hits: []const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + var next = if (try root.getCursor(key("next"))) |n| try n.readUint() else 0; + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + for (ctx.blobs) |*b| { + if (try texts.getCursor(blobKey(&b.hash)) != null) continue; + try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes }); + try seqs.put(blobKey(&b.hash), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = &b.hash }); + next += 1; + size += b.bytes.len; + } + for (ctx.hits) |*h| { + const at = (try seqs.getCursor(blobKey(h))) orelse continue; + _ = try seen.remove(&seqKey(try at.readUint())); + try seqs.put(blobKey(h), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = h }); + next += 1; + } + try root.put(key("next"), .{ .uint = next }); + try root.put(key("size"), .{ .uint = size }); + } + }{ .blobs = blobs, .hits = hits }) catch {}; + } + + /// Under the cap: the least recently used texts gone, in one + /// transaction, and the file compacted into `<path>.new` and renamed + /// over it once any went (what no moment holds is the file's growth). + pub fn trim(c: *const Cache, io: std.Io) void { + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + var went = false; + u.db.runContext(struct { + cap: u64, + went: *bool, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + if (size <= ctx.cap) return; + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + // The oldest numbers, read before any goes. + var gone: std.ArrayList(struct { seq: [8]u8, hash: [32]u8 }) = .empty; + defer gone.deinit(std.heap.c_allocator); + var left = size; + var it = try seen.iterator(); + while (left > ctx.cap) { + const kv_at = (try it.next()) orelse break; + const kv = try kv_at.readKeyValuePair(); + var g: @TypeOf(gone.items[0]) = undefined; + if ((try kv.key_cursor.readBytes(&g.seq)).len != 8 or (try kv.value_cursor.readBytes(&g.hash)).len != 32) continue; + if (try texts.getCursor(blobKey(&g.hash))) |at| left -|= try at.count(); + try gone.append(std.heap.c_allocator, g); + } + for (gone.items) |*g| { + if (try texts.getCursor(blobKey(&g.hash))) |at| size -|= try at.count(); + _ = try texts.remove(blobKey(&g.hash)); + _ = try seqs.remove(blobKey(&g.hash)); + _ = try seen.remove(&g.seq); + } + try root.put(key("size"), .{ .uint = size }); + ctx.went.* = gone.items.len > 0; + } + }{ .cap = c.cap, .went = &went }) catch return; + if (!went) return; + var new_buf: [4200]u8 = undefined; + const new = std.fmt.bufPrint(&new_buf, "{s}.new", .{c.path()}) catch return; + const cwd = std.Io.Dir.cwd(); + const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return; + var buffer: std.Io.Writer.Allocating = .init(std.heap.c_allocator); + defer buffer.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(std.heap.c_allocator); + defer offsets.deinit(); + _ = u.db.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = false, .hash_id = hash_id }, &offsets) catch { + file.close(io); + cwd.deleteFile(io, new) catch {}; + return; + }; + file.close(io); + cwd.rename(new, cwd, c.path(), io) catch cwd.deleteFile(io, new) catch {}; + } +}; + // ---- tests ---- const testing = std.testing; |
