summaryrefslogtreecommitdiff
path: root/src/Db.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/Db.zig')
-rw-r--r--src/Db.zig255
1 files changed, 255 insertions, 0 deletions
diff --git a/src/Db.zig b/src/Db.zig
index a91a5054..846c8d96 100644
--- a/src/Db.zig
+++ b/src/Db.zig
@@ -762,6 +762,261 @@ fn liveSize(db: *Disk) !u64 {
return live;
}
+// ---- a far client's blob cache ----
+
+/// A far replica client's blob cache (replica.zig `Have`): `cache.xitdb` in
+/// $XDG_CACHE_HOME/pardes (else ~/.cache/pardes), the texts a join was
+/// sent, by their BLAKE3 hash, so a client started again or a second one on
+/// this machine is not sent them again. Expendable, and up to a gigabyte, so
+/// a file of its own, never the state file's:
+///
+/// blobs/<hash> the text
+/// seqs/<hash> its number in `seen`
+/// seen a sorted map, a number (big endian) to its blob's hash:
+/// the least recently used first (a hit takes a new number)
+/// size, next the texts' bytes, the next number
+///
+/// Past the cap (PARDES_BLOB_CACHE_MB, else 1 GiB) the oldest go, and the
+/// file is compacted into `cache.xitdb.new` and renamed over it: one
+/// transaction and one compaction, at a far client's end (`trim`), never on
+/// a frame. A text read back is hashed again: one torn or changed is a miss
+/// and goes, never wrong content; a file that is no database is started
+/// afresh (it holds nothing a join cannot send again). Each use opens the
+/// file for itself: a client's reader, its storing thread and its end each
+/// have a database of their own, as xitdb asks.
+pub const Cache = struct {
+ path_buf: [4096]u8 = undefined,
+ path_len: usize = 0,
+ /// Bytes it keeps at most.
+ cap: u64,
+
+ pub const name = "cache.xitdb";
+
+ /// The cache's place, its directory made: null with no home for it.
+ pub fn open(io: std.Io) ?Cache {
+ var c: Cache = .{ .cap = (if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024) << 20 };
+ const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else "";
+ const where = (if (base.len > 0 and base[0] == '/')
+ std.fmt.bufPrint(&c.path_buf, "{s}/pardes/" ++ name, .{base})
+ else if (std.c.getenv("HOME")) |home|
+ std.fmt.bufPrint(&c.path_buf, "{s}/.cache/pardes/" ++ name, .{std.mem.span(home)})
+ else
+ return null) catch return null;
+ std.Io.Dir.cwd().createDirPath(io, std.fs.path.dirname(where).?) catch return null;
+ c.path_len = where.len;
+ return c;
+ }
+
+ /// The cache at `where` (a test's).
+ pub fn of(where: []const u8, cap: u64) Cache {
+ var c: Cache = .{ .cap = cap, .path_len = where.len };
+ @memcpy(c.path_buf[0..where.len], where);
+ return c;
+ }
+
+ pub fn path(c: *const Cache) []const u8 {
+ return c.path_buf[0..c.path_len];
+ }
+
+ /// The cache open for one use.
+ const Use = struct {
+ file: std.Io.File,
+ buffer: std.Io.Writer.Allocating,
+ db: Disk,
+ };
+
+ fn use(c: *const Cache, io: std.Io, u: *Use) !void {
+ const cwd = std.Io.Dir.cwd();
+ u.buffer = .init(std.heap.c_allocator);
+ errdefer u.buffer.deinit();
+ for (0..2) |_| {
+ u.file = try cwd.createFile(io, c.path(), .{ .read = true, .truncate = false });
+ u.db = Disk.init(io, .{ .file = u.file, .buffer = &u.buffer, .fsync = false, .hash_id = hash_id }) catch |err| {
+ u.file.close(io);
+ // No database: started afresh, as nothing in it is precious.
+ cwd.deleteFile(io, c.path()) catch return err;
+ continue;
+ };
+ return;
+ }
+ return error.InvalidDatabase;
+ }
+
+ fn done(u: *Use, io: std.Io) void {
+ u.file.close(io);
+ u.buffer.deinit();
+ }
+
+ fn mapAt(root: Disk.HashMap(.read_only), field: []const u8) !?Disk.HashMap(.read_only) {
+ const c = (try root.getCursor(key(field))) orelse return null;
+ return try Disk.HashMap(.read_only).init(c);
+ }
+
+ /// The text of `hash` read into `out` (its length) and hashed again.
+ /// One of another length or content is a miss, and goes.
+ pub fn get(c: *const Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool {
+ var u: Use = undefined;
+ c.use(io, &u) catch return false;
+ defer done(&u, io);
+ const ok = ok: {
+ const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch break :ok false;
+ const blobs = (mapAt(root, "blobs") catch break :ok false) orelse return false;
+ const at = (blobs.getCursor(blobKey(hash)) catch break :ok false) orelse return false;
+ if ((at.count() catch break :ok false) != out.len) break :ok false;
+ _ = at.readBytes(out) catch break :ok false;
+ var d: [32]u8 = undefined;
+ std.crypto.hash.Blake3.hash(out, &d, .{});
+ break :ok std.mem.eql(u8, &d, hash);
+ };
+ if (!ok) drop(&u, io, hash);
+ return ok;
+ }
+
+ /// Whether a text of `len` bytes is there under `hash` (not read).
+ pub fn has(c: *const Cache, io: std.Io, hash: *const [32]u8, len: usize) bool {
+ var u: Use = undefined;
+ c.use(io, &u) catch return false;
+ defer done(&u, io);
+ const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch return false;
+ const blobs = (mapAt(root, "blobs") catch return false) orelse return false;
+ const at = (blobs.getCursor(blobKey(hash)) catch return false) orelse return false;
+ return (at.count() catch return false) == len;
+ }
+
+ /// The text of `hash`, gone (one read back wrong).
+ fn drop(u: *Use, io: std.Io, hash: *const [32]u8) void {
+ u.file.lock(io, .exclusive) catch return;
+ defer u.file.unlock(io);
+ u.db.runContext(struct {
+ hash: *const [32]u8,
+ pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void {
+ const root = try Disk.HashMap(.read_write).init(cursor.*);
+ const blobs = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const at = (try blobs.getCursor(blobKey(ctx.hash))) orelse return;
+ const len = try at.count();
+ _ = try blobs.remove(blobKey(ctx.hash));
+ const size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0;
+ try root.put(key("size"), .{ .uint = size -| len });
+ const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs")));
+ if (try seqs.getCursor(blobKey(ctx.hash))) |s| {
+ const was = try s.readUint();
+ _ = try seqs.remove(blobKey(ctx.hash));
+ const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen")));
+ _ = try seen.remove(&seqKey(was));
+ }
+ }
+ }{ .hash = hash }) catch {};
+ }
+
+ fn seqKey(seq: u64) [8]u8 {
+ var k: [8]u8 = undefined;
+ std.mem.writeInt(u64, &k, seq, .big);
+ return k;
+ }
+
+ /// `blobs` kept (each text not there yet, under its hash) and `hits`,
+ /// texts read from here, made the most recently used: one transaction,
+ /// the file's lock waited for (a storing thread's, a client's end's).
+ pub fn put(c: *const Cache, io: std.Io, blobs: []const pardes.walk.Blob, hits: []const [32]u8) void {
+ if (blobs.len == 0 and hits.len == 0) return;
+ var u: Use = undefined;
+ c.use(io, &u) catch return;
+ defer done(&u, io);
+ u.file.lock(io, .exclusive) catch return;
+ defer u.file.unlock(io);
+ u.db.runContext(struct {
+ blobs: []const pardes.walk.Blob,
+ hits: []const [32]u8,
+ pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void {
+ const root = try Disk.HashMap(.read_write).init(cursor.*);
+ const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs")));
+ const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen")));
+ var next = if (try root.getCursor(key("next"))) |n| try n.readUint() else 0;
+ var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0;
+ for (ctx.blobs) |*b| {
+ if (try texts.getCursor(blobKey(&b.hash)) != null) continue;
+ try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes });
+ try seqs.put(blobKey(&b.hash), .{ .uint = next });
+ try seen.put(&seqKey(next), .{ .bytes = &b.hash });
+ next += 1;
+ size += b.bytes.len;
+ }
+ for (ctx.hits) |*h| {
+ const at = (try seqs.getCursor(blobKey(h))) orelse continue;
+ _ = try seen.remove(&seqKey(try at.readUint()));
+ try seqs.put(blobKey(h), .{ .uint = next });
+ try seen.put(&seqKey(next), .{ .bytes = h });
+ next += 1;
+ }
+ try root.put(key("next"), .{ .uint = next });
+ try root.put(key("size"), .{ .uint = size });
+ }
+ }{ .blobs = blobs, .hits = hits }) catch {};
+ }
+
+ /// Under the cap: the least recently used texts gone, in one
+ /// transaction, and the file compacted into `<path>.new` and renamed
+ /// over it once any went (what no moment holds is the file's growth).
+ pub fn trim(c: *const Cache, io: std.Io) void {
+ var u: Use = undefined;
+ c.use(io, &u) catch return;
+ defer done(&u, io);
+ u.file.lock(io, .exclusive) catch return;
+ defer u.file.unlock(io);
+ var went = false;
+ u.db.runContext(struct {
+ cap: u64,
+ went: *bool,
+ pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void {
+ const root = try Disk.HashMap(.read_write).init(cursor.*);
+ var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0;
+ if (size <= ctx.cap) return;
+ const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs")));
+ const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen")));
+ // The oldest numbers, read before any goes.
+ var gone: std.ArrayList(struct { seq: [8]u8, hash: [32]u8 }) = .empty;
+ defer gone.deinit(std.heap.c_allocator);
+ var left = size;
+ var it = try seen.iterator();
+ while (left > ctx.cap) {
+ const kv_at = (try it.next()) orelse break;
+ const kv = try kv_at.readKeyValuePair();
+ var g: @TypeOf(gone.items[0]) = undefined;
+ if ((try kv.key_cursor.readBytes(&g.seq)).len != 8 or (try kv.value_cursor.readBytes(&g.hash)).len != 32) continue;
+ if (try texts.getCursor(blobKey(&g.hash))) |at| left -|= try at.count();
+ try gone.append(std.heap.c_allocator, g);
+ }
+ for (gone.items) |*g| {
+ if (try texts.getCursor(blobKey(&g.hash))) |at| size -|= try at.count();
+ _ = try texts.remove(blobKey(&g.hash));
+ _ = try seqs.remove(blobKey(&g.hash));
+ _ = try seen.remove(&g.seq);
+ }
+ try root.put(key("size"), .{ .uint = size });
+ ctx.went.* = gone.items.len > 0;
+ }
+ }{ .cap = c.cap, .went = &went }) catch return;
+ if (!went) return;
+ var new_buf: [4200]u8 = undefined;
+ const new = std.fmt.bufPrint(&new_buf, "{s}.new", .{c.path()}) catch return;
+ const cwd = std.Io.Dir.cwd();
+ const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return;
+ var buffer: std.Io.Writer.Allocating = .init(std.heap.c_allocator);
+ defer buffer.deinit();
+ var offsets: std.AutoHashMap(u64, u64) = .init(std.heap.c_allocator);
+ defer offsets.deinit();
+ _ = u.db.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = false, .hash_id = hash_id }, &offsets) catch {
+ file.close(io);
+ cwd.deleteFile(io, new) catch {};
+ return;
+ };
+ file.close(io);
+ cwd.rename(new, cwd, c.path(), io) catch cwd.deleteFile(io, new) catch {};
+ }
+};
+
// ---- tests ----
const testing = std.testing;