diff options
Diffstat (limited to 'src/Db.zig')
| -rw-r--r-- | src/Db.zig | 255 |
1 files changed, 255 insertions, 0 deletions
@@ -762,6 +762,261 @@ fn liveSize(db: *Disk) !u64 { return live; } +// ---- a far client's blob cache ---- + +/// A far replica client's blob cache (replica.zig `Have`): `cache.xitdb` in +/// $XDG_CACHE_HOME/pardes (else ~/.cache/pardes), the texts a join was +/// sent, by their BLAKE3 hash, so a client started again or a second one on +/// this machine is not sent them again. Expendable, and up to a gigabyte, so +/// a file of its own, never the state file's: +/// +/// blobs/<hash> the text +/// seqs/<hash> its number in `seen` +/// seen a sorted map, a number (big endian) to its blob's hash: +/// the least recently used first (a hit takes a new number) +/// size, next the texts' bytes, the next number +/// +/// Past the cap (PARDES_BLOB_CACHE_MB, else 1 GiB) the oldest go, and the +/// file is compacted into `cache.xitdb.new` and renamed over it: one +/// transaction and one compaction, at a far client's end (`trim`), never on +/// a frame. A text read back is hashed again: one torn or changed is a miss +/// and goes, never wrong content; a file that is no database is started +/// afresh (it holds nothing a join cannot send again). Each use opens the +/// file for itself: a client's reader, its storing thread and its end each +/// have a database of their own, as xitdb asks. +pub const Cache = struct { + path_buf: [4096]u8 = undefined, + path_len: usize = 0, + /// Bytes it keeps at most. + cap: u64, + + pub const name = "cache.xitdb"; + + /// The cache's place, its directory made: null with no home for it. + pub fn open(io: std.Io) ?Cache { + var c: Cache = .{ .cap = (if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024) << 20 }; + const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else ""; + const where = (if (base.len > 0 and base[0] == '/') + std.fmt.bufPrint(&c.path_buf, "{s}/pardes/" ++ name, .{base}) + else if (std.c.getenv("HOME")) |home| + std.fmt.bufPrint(&c.path_buf, "{s}/.cache/pardes/" ++ name, .{std.mem.span(home)}) + else + return null) catch return null; + std.Io.Dir.cwd().createDirPath(io, std.fs.path.dirname(where).?) catch return null; + c.path_len = where.len; + return c; + } + + /// The cache at `where` (a test's). + pub fn of(where: []const u8, cap: u64) Cache { + var c: Cache = .{ .cap = cap, .path_len = where.len }; + @memcpy(c.path_buf[0..where.len], where); + return c; + } + + pub fn path(c: *const Cache) []const u8 { + return c.path_buf[0..c.path_len]; + } + + /// The cache open for one use. + const Use = struct { + file: std.Io.File, + buffer: std.Io.Writer.Allocating, + db: Disk, + }; + + fn use(c: *const Cache, io: std.Io, u: *Use) !void { + const cwd = std.Io.Dir.cwd(); + u.buffer = .init(std.heap.c_allocator); + errdefer u.buffer.deinit(); + for (0..2) |_| { + u.file = try cwd.createFile(io, c.path(), .{ .read = true, .truncate = false }); + u.db = Disk.init(io, .{ .file = u.file, .buffer = &u.buffer, .fsync = false, .hash_id = hash_id }) catch |err| { + u.file.close(io); + // No database: started afresh, as nothing in it is precious. + cwd.deleteFile(io, c.path()) catch return err; + continue; + }; + return; + } + return error.InvalidDatabase; + } + + fn done(u: *Use, io: std.Io) void { + u.file.close(io); + u.buffer.deinit(); + } + + fn mapAt(root: Disk.HashMap(.read_only), field: []const u8) !?Disk.HashMap(.read_only) { + const c = (try root.getCursor(key(field))) orelse return null; + return try Disk.HashMap(.read_only).init(c); + } + + /// The text of `hash` read into `out` (its length) and hashed again. + /// One of another length or content is a miss, and goes. + pub fn get(c: *const Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const ok = ok: { + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch break :ok false; + const blobs = (mapAt(root, "blobs") catch break :ok false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch break :ok false) orelse return false; + if ((at.count() catch break :ok false) != out.len) break :ok false; + _ = at.readBytes(out) catch break :ok false; + var d: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(out, &d, .{}); + break :ok std.mem.eql(u8, &d, hash); + }; + if (!ok) drop(&u, io, hash); + return ok; + } + + /// Whether a text of `len` bytes is there under `hash` (not read). + pub fn has(c: *const Cache, io: std.Io, hash: *const [32]u8, len: usize) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch return false; + const blobs = (mapAt(root, "blobs") catch return false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch return false) orelse return false; + return (at.count() catch return false) == len; + } + + /// The text of `hash`, gone (one read back wrong). + fn drop(u: *Use, io: std.Io, hash: *const [32]u8) void { + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + hash: *const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const blobs = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const at = (try blobs.getCursor(blobKey(ctx.hash))) orelse return; + const len = try at.count(); + _ = try blobs.remove(blobKey(ctx.hash)); + const size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + try root.put(key("size"), .{ .uint = size -| len }); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + if (try seqs.getCursor(blobKey(ctx.hash))) |s| { + const was = try s.readUint(); + _ = try seqs.remove(blobKey(ctx.hash)); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + _ = try seen.remove(&seqKey(was)); + } + } + }{ .hash = hash }) catch {}; + } + + fn seqKey(seq: u64) [8]u8 { + var k: [8]u8 = undefined; + std.mem.writeInt(u64, &k, seq, .big); + return k; + } + + /// `blobs` kept (each text not there yet, under its hash) and `hits`, + /// texts read from here, made the most recently used: one transaction, + /// the file's lock waited for (a storing thread's, a client's end's). + pub fn put(c: *const Cache, io: std.Io, blobs: []const pardes.walk.Blob, hits: []const [32]u8) void { + if (blobs.len == 0 and hits.len == 0) return; + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + blobs: []const pardes.walk.Blob, + hits: []const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + var next = if (try root.getCursor(key("next"))) |n| try n.readUint() else 0; + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + for (ctx.blobs) |*b| { + if (try texts.getCursor(blobKey(&b.hash)) != null) continue; + try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes }); + try seqs.put(blobKey(&b.hash), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = &b.hash }); + next += 1; + size += b.bytes.len; + } + for (ctx.hits) |*h| { + const at = (try seqs.getCursor(blobKey(h))) orelse continue; + _ = try seen.remove(&seqKey(try at.readUint())); + try seqs.put(blobKey(h), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = h }); + next += 1; + } + try root.put(key("next"), .{ .uint = next }); + try root.put(key("size"), .{ .uint = size }); + } + }{ .blobs = blobs, .hits = hits }) catch {}; + } + + /// Under the cap: the least recently used texts gone, in one + /// transaction, and the file compacted into `<path>.new` and renamed + /// over it once any went (what no moment holds is the file's growth). + pub fn trim(c: *const Cache, io: std.Io) void { + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + var went = false; + u.db.runContext(struct { + cap: u64, + went: *bool, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + if (size <= ctx.cap) return; + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + // The oldest numbers, read before any goes. + var gone: std.ArrayList(struct { seq: [8]u8, hash: [32]u8 }) = .empty; + defer gone.deinit(std.heap.c_allocator); + var left = size; + var it = try seen.iterator(); + while (left > ctx.cap) { + const kv_at = (try it.next()) orelse break; + const kv = try kv_at.readKeyValuePair(); + var g: @TypeOf(gone.items[0]) = undefined; + if ((try kv.key_cursor.readBytes(&g.seq)).len != 8 or (try kv.value_cursor.readBytes(&g.hash)).len != 32) continue; + if (try texts.getCursor(blobKey(&g.hash))) |at| left -|= try at.count(); + try gone.append(std.heap.c_allocator, g); + } + for (gone.items) |*g| { + if (try texts.getCursor(blobKey(&g.hash))) |at| size -|= try at.count(); + _ = try texts.remove(blobKey(&g.hash)); + _ = try seqs.remove(blobKey(&g.hash)); + _ = try seen.remove(&g.seq); + } + try root.put(key("size"), .{ .uint = size }); + ctx.went.* = gone.items.len > 0; + } + }{ .cap = c.cap, .went = &went }) catch return; + if (!went) return; + var new_buf: [4200]u8 = undefined; + const new = std.fmt.bufPrint(&new_buf, "{s}.new", .{c.path()}) catch return; + const cwd = std.Io.Dir.cwd(); + const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return; + var buffer: std.Io.Writer.Allocating = .init(std.heap.c_allocator); + defer buffer.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(std.heap.c_allocator); + defer offsets.deinit(); + _ = u.db.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = false, .hash_id = hash_id }, &offsets) catch { + file.close(io); + cwd.deleteFile(io, new) catch {}; + return; + }; + file.close(io); + cwd.rename(new, cwd, c.path(), io) catch cwd.deleteFile(io, new) catch {}; + } +}; + // ---- tests ---- const testing = std.testing; |
