summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/Db.zig255
-rw-r--r--src/replica.zig272
-rw-r--r--test/daemon.py50
3 files changed, 373 insertions, 204 deletions
diff --git a/src/Db.zig b/src/Db.zig
index a91a5054..846c8d96 100644
--- a/src/Db.zig
+++ b/src/Db.zig
@@ -762,6 +762,261 @@ fn liveSize(db: *Disk) !u64 {
return live;
}
+// ---- a far client's blob cache ----
+
+/// A far replica client's blob cache (replica.zig `Have`): `cache.xitdb` in
+/// $XDG_CACHE_HOME/pardes (else ~/.cache/pardes), the texts a join was
+/// sent, by their BLAKE3 hash, so a client started again or a second one on
+/// this machine is not sent them again. Expendable, and up to a gigabyte, so
+/// a file of its own, never the state file's:
+///
+/// blobs/<hash> the text
+/// seqs/<hash> its number in `seen`
+/// seen a sorted map, a number (big endian) to its blob's hash:
+/// the least recently used first (a hit takes a new number)
+/// size, next the texts' bytes, the next number
+///
+/// Past the cap (PARDES_BLOB_CACHE_MB, else 1 GiB) the oldest go, and the
+/// file is compacted into `cache.xitdb.new` and renamed over it: one
+/// transaction and one compaction, at a far client's end (`trim`), never on
+/// a frame. A text read back is hashed again: one torn or changed is a miss
+/// and goes, never wrong content; a file that is no database is started
+/// afresh (it holds nothing a join cannot send again). Each use opens the
+/// file for itself: a client's reader, its storing thread and its end each
+/// have a database of their own, as xitdb asks.
+pub const Cache = struct {
+ path_buf: [4096]u8 = undefined,
+ path_len: usize = 0,
+ /// Bytes it keeps at most.
+ cap: u64,
+
+ pub const name = "cache.xitdb";
+
+ /// The cache's place, its directory made: null with no home for it.
+ pub fn open(io: std.Io) ?Cache {
+ var c: Cache = .{ .cap = (if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024) << 20 };
+ const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else "";
+ const where = (if (base.len > 0 and base[0] == '/')
+ std.fmt.bufPrint(&c.path_buf, "{s}/pardes/" ++ name, .{base})
+ else if (std.c.getenv("HOME")) |home|
+ std.fmt.bufPrint(&c.path_buf, "{s}/.cache/pardes/" ++ name, .{std.mem.span(home)})
+ else
+ return null) catch return null;
+ std.Io.Dir.cwd().createDirPath(io, std.fs.path.dirname(where).?) catch return null;
+ c.path_len = where.len;
+ return c;
+ }
+
+ /// The cache at `where` (a test's).
+ pub fn of(where: []const u8, cap: u64) Cache {
+ var c: Cache = .{ .cap = cap, .path_len = where.len };
+ @memcpy(c.path_buf[0..where.len], where);
+ return c;
+ }
+
+ pub fn path(c: *const Cache) []const u8 {
+ return c.path_buf[0..c.path_len];
+ }
+
+ /// The cache open for one use.
+ const Use = struct {
+ file: std.Io.File,
+ buffer: std.Io.Writer.Allocating,
+ db: Disk,
+ };
+
+ fn use(c: *const Cache, io: std.Io, u: *Use) !void {
+ const cwd = std.Io.Dir.cwd();
+ u.buffer = .init(std.heap.c_allocator);
+ errdefer u.buffer.deinit();
+ for (0..2) |_| {
+ u.file = try cwd.createFile(io, c.path(), .{ .read = true, .truncate = false });
+ u.db = Disk.init(io, .{ .file = u.file, .buffer = &u.buffer, .fsync = false, .hash_id = hash_id }) catch |err| {
+ u.file.close(io);
+ // No database: started afresh, as nothing in it is precious.
+ cwd.deleteFile(io, c.path()) catch return err;
+ continue;
+ };
+ return;
+ }
+ return error.InvalidDatabase;
+ }
+
+ fn done(u: *Use, io: std.Io) void {
+ u.file.close(io);
+ u.buffer.deinit();
+ }
+
+ fn mapAt(root: Disk.HashMap(.read_only), field: []const u8) !?Disk.HashMap(.read_only) {
+ const c = (try root.getCursor(key(field))) orelse return null;
+ return try Disk.HashMap(.read_only).init(c);
+ }
+
+ /// The text of `hash` read into `out` (its length) and hashed again.
+ /// One of another length or content is a miss, and goes.
+ pub fn get(c: *const Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool {
+ var u: Use = undefined;
+ c.use(io, &u) catch return false;
+ defer done(&u, io);
+ const ok = ok: {
+ const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch break :ok false;
+ const blobs = (mapAt(root, "blobs") catch break :ok false) orelse return false;
+ const at = (blobs.getCursor(blobKey(hash)) catch break :ok false) orelse return false;
+ if ((at.count() catch break :ok false) != out.len) break :ok false;
+ _ = at.readBytes(out) catch break :ok false;
+ var d: [32]u8 = undefined;
+ std.crypto.hash.Blake3.hash(out, &d, .{});
+ break :ok std.mem.eql(u8, &d, hash);
+ };
+ if (!ok) drop(&u, io, hash);
+ return ok;
+ }
+
+ /// Whether a text of `len` bytes is there under `hash` (not read).
+ pub fn has(c: *const Cache, io: std.Io, hash: *const [32]u8, len: usize) bool {
+ var u: Use = undefined;
+ c.use(io, &u) catch return false;
+ defer done(&u, io);
+ const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch return false;
+ const blobs = (mapAt(root, "blobs") catch return false) orelse return false;
+ const at = (blobs.getCursor(blobKey(hash)) catch return false) orelse return false;
+ return (at.count() catch return false) == len;
+ }
+
+ /// The text of `hash`, gone (one read back wrong).
+ fn drop(u: *Use, io: std.Io, hash: *const [32]u8) void {
+ u.file.lock(io, .exclusive) catch return;
+ defer u.file.unlock(io);
+ u.db.runContext(struct {
+ hash: *const [32]u8,
+ pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void {
+ const root = try Disk.HashMap(.read_write).init(cursor.*);
+ const blobs = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const at = (try blobs.getCursor(blobKey(ctx.hash))) orelse return;
+ const len = try at.count();
+ _ = try blobs.remove(blobKey(ctx.hash));
+ const size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0;
+ try root.put(key("size"), .{ .uint = size -| len });
+ const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs")));
+ if (try seqs.getCursor(blobKey(ctx.hash))) |s| {
+ const was = try s.readUint();
+ _ = try seqs.remove(blobKey(ctx.hash));
+ const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen")));
+ _ = try seen.remove(&seqKey(was));
+ }
+ }
+ }{ .hash = hash }) catch {};
+ }
+
+ fn seqKey(seq: u64) [8]u8 {
+ var k: [8]u8 = undefined;
+ std.mem.writeInt(u64, &k, seq, .big);
+ return k;
+ }
+
+ /// `blobs` kept (each text not there yet, under its hash) and `hits`,
+ /// texts read from here, made the most recently used: one transaction,
+ /// the file's lock waited for (a storing thread's, a client's end's).
+ pub fn put(c: *const Cache, io: std.Io, blobs: []const pardes.walk.Blob, hits: []const [32]u8) void {
+ if (blobs.len == 0 and hits.len == 0) return;
+ var u: Use = undefined;
+ c.use(io, &u) catch return;
+ defer done(&u, io);
+ u.file.lock(io, .exclusive) catch return;
+ defer u.file.unlock(io);
+ u.db.runContext(struct {
+ blobs: []const pardes.walk.Blob,
+ hits: []const [32]u8,
+ pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void {
+ const root = try Disk.HashMap(.read_write).init(cursor.*);
+ const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs")));
+ const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen")));
+ var next = if (try root.getCursor(key("next"))) |n| try n.readUint() else 0;
+ var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0;
+ for (ctx.blobs) |*b| {
+ if (try texts.getCursor(blobKey(&b.hash)) != null) continue;
+ try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes });
+ try seqs.put(blobKey(&b.hash), .{ .uint = next });
+ try seen.put(&seqKey(next), .{ .bytes = &b.hash });
+ next += 1;
+ size += b.bytes.len;
+ }
+ for (ctx.hits) |*h| {
+ const at = (try seqs.getCursor(blobKey(h))) orelse continue;
+ _ = try seen.remove(&seqKey(try at.readUint()));
+ try seqs.put(blobKey(h), .{ .uint = next });
+ try seen.put(&seqKey(next), .{ .bytes = h });
+ next += 1;
+ }
+ try root.put(key("next"), .{ .uint = next });
+ try root.put(key("size"), .{ .uint = size });
+ }
+ }{ .blobs = blobs, .hits = hits }) catch {};
+ }
+
+ /// Under the cap: the least recently used texts gone, in one
+ /// transaction, and the file compacted into `<path>.new` and renamed
+ /// over it once any went (what no moment holds is the file's growth).
+ pub fn trim(c: *const Cache, io: std.Io) void {
+ var u: Use = undefined;
+ c.use(io, &u) catch return;
+ defer done(&u, io);
+ u.file.lock(io, .exclusive) catch return;
+ defer u.file.unlock(io);
+ var went = false;
+ u.db.runContext(struct {
+ cap: u64,
+ went: *bool,
+ pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void {
+ const root = try Disk.HashMap(.read_write).init(cursor.*);
+ var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0;
+ if (size <= ctx.cap) return;
+ const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs")));
+ const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs")));
+ const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen")));
+ // The oldest numbers, read before any goes.
+ var gone: std.ArrayList(struct { seq: [8]u8, hash: [32]u8 }) = .empty;
+ defer gone.deinit(std.heap.c_allocator);
+ var left = size;
+ var it = try seen.iterator();
+ while (left > ctx.cap) {
+ const kv_at = (try it.next()) orelse break;
+ const kv = try kv_at.readKeyValuePair();
+ var g: @TypeOf(gone.items[0]) = undefined;
+ if ((try kv.key_cursor.readBytes(&g.seq)).len != 8 or (try kv.value_cursor.readBytes(&g.hash)).len != 32) continue;
+ if (try texts.getCursor(blobKey(&g.hash))) |at| left -|= try at.count();
+ try gone.append(std.heap.c_allocator, g);
+ }
+ for (gone.items) |*g| {
+ if (try texts.getCursor(blobKey(&g.hash))) |at| size -|= try at.count();
+ _ = try texts.remove(blobKey(&g.hash));
+ _ = try seqs.remove(blobKey(&g.hash));
+ _ = try seen.remove(&g.seq);
+ }
+ try root.put(key("size"), .{ .uint = size });
+ ctx.went.* = gone.items.len > 0;
+ }
+ }{ .cap = c.cap, .went = &went }) catch return;
+ if (!went) return;
+ var new_buf: [4200]u8 = undefined;
+ const new = std.fmt.bufPrint(&new_buf, "{s}.new", .{c.path()}) catch return;
+ const cwd = std.Io.Dir.cwd();
+ const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return;
+ var buffer: std.Io.Writer.Allocating = .init(std.heap.c_allocator);
+ defer buffer.deinit();
+ var offsets: std.AutoHashMap(u64, u64) = .init(std.heap.c_allocator);
+ defer offsets.deinit();
+ _ = u.db.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = false, .hash_id = hash_id }, &offsets) catch {
+ file.close(io);
+ cwd.deleteFile(io, new) catch {};
+ return;
+ };
+ file.close(io);
+ cwd.rename(new, cwd, c.path(), io) catch cwd.deleteFile(io, new) catch {};
+ }
+};
+
// ---- tests ----
const testing = std.testing;
diff --git a/src/replica.zig b/src/replica.zig
index 5bcc0a53..b98d792b 100644
--- a/src/replica.zig
+++ b/src/replica.zig
@@ -2421,6 +2421,8 @@ fn joinOn(gpa: Allocator, fd: c_int, session: []const u8, why: []u8, have: ?*Hav
pub const Have = struct {
io: std.Io,
cache: ?Cache = null,
+ /// The blobs a join found on disk: the most recently used once stored.
+ hits: std.ArrayListUnmanaged([32]u8) = .empty,
/// The core it leaves (`leaving`), each blob hashed once when one of its
/// length is looked for. ponytail: a scan a lookup, fine to some
/// thousands of blobs; a map by length would take over past that.
@@ -2438,7 +2440,7 @@ pub const Have = struct {
h.left();
h.old.deinit(std.heap.c_allocator);
h.old_hash.deinit(std.heap.c_allocator);
- if (h.cache) |c| c.dir.close(h.io);
+ h.hits.deinit(std.heap.c_allocator);
}
fn wait(h: *Have) void {
@@ -2475,189 +2477,83 @@ pub const Have = struct {
@memcpy(out, b);
return .memory;
}
- if (h.cache) |c| if (c.get(h.io, hash, out)) return .disk;
+ if (h.cache) |*c| if (c.get(h.io, hash, out)) {
+ h.hits.append(std.heap.c_allocator, hash.*) catch {};
+ return .disk;
+ };
return null;
}
/// The blobs a join was sent, written to the cache on a thread of their
- /// own, which frees the dump (`gpa`'s) and the list after: the first
- /// frame waits for none of it.
+ /// own (with those it found there, made the most recently used), which
+ /// frees the dump (`gpa`'s) and the lists after: the first frame waits
+ /// for none of it.
pub fn store(h: *Have, gpa: Allocator, j: *Joined) void {
const dump = j.dump;
const got = j.got;
j.dump = &.{};
j.got = &.{};
h.wait();
+ var hits = h.hits;
+ h.hits = .empty;
const Store = struct {
- fn run(c: Cache, io: std.Io, a: Allocator, bytes: []u8, list: []Got) void {
+ fn run(c: Cache, io: std.Io, a: Allocator, bytes: []u8, list: []Got, found: std.ArrayListUnmanaged([32]u8)) void {
+ var seen = found;
defer {
a.free(bytes);
std.heap.c_allocator.free(list);
+ seen.deinit(std.heap.c_allocator);
}
- var incoming: u64 = 0;
- for (list) |g| incoming += g.len;
- c.trim(io, @min(incoming, c.cap));
+ var blobs: std.ArrayListUnmanaged(walk.Blob) = .empty;
+ defer blobs.deinit(std.heap.c_allocator);
var put: u64 = 0;
for (list) |g| {
if (put + g.len > c.cap) continue;
put += g.len;
- c.put(io, &g.hash, bytes[g.pos..][0..g.len]);
+ blobs.append(std.heap.c_allocator, .{ .hash = g.hash, .bytes = bytes[g.pos..][0..g.len] }) catch return;
}
+ c.put(io, blobs.items, seen.items);
}
};
const c = h.cache orelse {
gpa.free(dump);
std.heap.c_allocator.free(got);
+ hits.deinit(std.heap.c_allocator);
return;
};
- h.storing = std.Thread.spawn(.{}, Store.run, .{ c, h.io, gpa, dump, got }) catch {
- Store.run(c, h.io, gpa, dump, got);
+ h.storing = std.Thread.spawn(.{}, Store.run, .{ c, h.io, gpa, dump, got, hits }) catch {
+ Store.run(c, h.io, gpa, dump, got, hits);
return;
};
}
/// `core`'s blobs the cache lacks, written there: what this client made
- /// since it joined (an edit's text, its undo), for its next start.
+ /// since it joined (an edit's text, its undo), for its next start; then
+ /// the cache trimmed under its cap, at this client's end.
pub fn keep(h: *Have, core: *const Pardes) void {
h.wait();
const c = h.cache orelse return;
var all: std.ArrayListUnmanaged([]const u8) = .empty;
defer all.deinit(std.heap.c_allocator);
walk.blobs(Pardes, core, &all);
- var missing: std.ArrayListUnmanaged(struct { bytes: []const u8, hash: [32]u8 }) = .empty;
+ var missing: std.ArrayListUnmanaged(walk.Blob) = .empty;
defer missing.deinit(std.heap.c_allocator);
- var incoming: u64 = 0;
+ var put: u64 = 0;
for (all.items) |b| {
var d: [32]u8 = undefined;
std.crypto.hash.Blake3.hash(b, &d, .{});
- if (c.has(h.io, &d, b.len)) continue;
+ if (put + b.len > c.cap or c.has(h.io, &d, b.len)) continue;
+ put += b.len;
missing.append(std.heap.c_allocator, .{ .bytes = b, .hash = d }) catch return;
- incoming += b.len;
- }
- if (missing.items.len == 0) return;
- c.trim(h.io, @min(incoming, c.cap));
- var put: u64 = 0;
- for (missing.items) |m| {
- if (put + m.bytes.len > c.cap) continue;
- put += m.bytes.len;
- c.put(h.io, &m.hash, m.bytes);
}
+ c.put(h.io, missing.items, h.hits.items);
+ h.hits.clearRetainingCapacity();
+ c.trim(h.io);
}
};
-/// A far client's blobs on disk, a file each named by its BLAKE3 hash in
-/// hex, in $XDG_CACHE_HOME/pardes/blobs (else ~/.cache/pardes/blobs): what
-/// a client started again, or a second one on this machine, is not sent
-/// again. The cache directory and not the runtime one: the blobs are worth
-/// keeping across a logout or a reboot and safe to lose, and they can be
-/// hundreds of megabytes, which the runtime directory (a tmpfs, often
-/// capped at a tenth of memory and emptied at logout) is not for. A file
-/// read back is hashed again: one cut short, corrupted or planted is a
-/// miss and goes, never wrong content.
-pub const Cache = struct {
- dir: std.Io.Dir,
- /// Bytes it keeps at most: PARDES_BLOB_CACHE_MB, else 1 GiB.
- cap: u64,
-
- pub fn open(io: std.Io) ?Cache {
- var buf: [4096]u8 = undefined;
- const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else "";
- const path = if (base.len > 0 and base[0] == '/')
- std.fmt.bufPrint(&buf, "{s}/pardes/blobs", .{base}) catch return null
- else if (std.c.getenv("HOME")) |home|
- std.fmt.bufPrint(&buf, "{s}/.cache/pardes/blobs", .{std.mem.span(home)}) catch return null
- else
- return null;
- const cwd = std.Io.Dir.cwd();
- cwd.createDirPath(io, std.fs.path.dirname(path).?) catch return null;
- cwd.createDir(io, path, .fromMode(0o700)) catch |err| if (err != error.PathAlreadyExists) return null;
- const mb = if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024;
- return .{ .dir = cwd.openDir(io, path, .{ .iterate = true }) catch return null, .cap = mb << 20 };
- }
-
- fn name(hash: *const [32]u8) [64]u8 {
- return std.fmt.bytesToHex(hash.*, .lower);
- }
-
- /// The blob of `hash` read into `out` (its length) and hashed again.
- /// A file of another length or content is a miss, and goes; a hit's
- /// time is made now, so the least recently used go first (`trim`).
- pub fn get(c: Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool {
- const file = name(hash);
- var f = c.dir.openFile(io, &file, .{}) catch return false;
- const ok = ok: {
- defer f.close(io);
- const st = f.stat(io) catch break :ok false;
- if (st.size != out.len) break :ok false;
- var reader = f.reader(io, &.{});
- const n = reader.interface.readSliceShort(out) catch break :ok false;
- if (n != out.len) break :ok false;
- var d: [32]u8 = undefined;
- std.crypto.hash.Blake3.hash(out, &d, .{});
- break :ok std.mem.eql(u8, &d, hash);
- };
- if (!ok) {
- c.dir.deleteFile(io, &file) catch {};
- return false;
- }
- c.dir.setTimestampsNow(io, &file, .{}) catch {};
- return true;
- }
-
- /// Whether a file of `len` bytes is there under `hash` (not read).
- pub fn has(c: Cache, io: std.Io, hash: *const [32]u8, len: usize) bool {
- const file = name(hash);
- const st = c.dir.statFile(io, &file, .{}) catch return false;
- return st.size == len;
- }
-
- var tmp_serial: std.atomic.Value(u32) = .init(0);
-
- /// `bytes` kept under `hash`: written aside and renamed into place, so
- /// that a reader never meets half a file (and would hash it wrong).
- pub fn put(c: Cache, io: std.Io, hash: *const [32]u8, bytes: []const u8) void {
- const file = name(hash);
- if (c.has(io, hash, bytes.len)) return;
- var buf: [96]u8 = undefined;
- const tmp = std.fmt.bufPrint(&buf, ".tmp-{d}-{d}-{s}", .{ std.c.getpid(), tmp_serial.fetchAdd(1, .monotonic), file[0..16] }) catch return;
- c.dir.writeFile(io, .{ .sub_path = tmp, .data = bytes }) catch {
- c.dir.deleteFile(io, tmp) catch {};
- return;
- };
- c.dir.rename(tmp, c.dir, &file, io) catch c.dir.deleteFile(io, tmp) catch {};
- }
-
- /// Room for `incoming` more bytes under the cap, the least recently
- /// used files deleted first (a hit makes its file's time now).
- /// ponytail: the directory scanned and sorted at each join's store;
- /// fine to tens of thousands of blobs, where an index would take over.
- pub fn trim(c: Cache, io: std.Io, incoming: u64) void {
- const File = struct { name: [80]u8, len: u8, size: u64, mtime: i96 };
- var files: std.ArrayListUnmanaged(File) = .empty;
- defer files.deinit(std.heap.c_allocator);
- var total: u64 = 0;
- var it = c.dir.iterate();
- while (it.next(io) catch null) |entry| {
- if (entry.kind != .file or entry.name.len > 80) continue;
- const st = c.dir.statFile(io, entry.name, .{}) catch continue;
- var f: File = .{ .name = undefined, .len = @intCast(entry.name.len), .size = st.size, .mtime = st.mtime.nanoseconds };
- @memcpy(f.name[0..f.len], entry.name);
- files.append(std.heap.c_allocator, f) catch return;
- total += st.size;
- }
- if (total + incoming <= c.cap) return;
- std.mem.sort(File, files.items, {}, struct {
- fn older(_: void, a: File, b: File) bool {
- return a.mtime < b.mtime;
- }
- }.older);
- for (files.items) |f| {
- if (total + incoming <= c.cap) break;
- c.dir.deleteFile(io, f.name[0..f.len]) catch continue;
- total -= f.size;
- }
- }
-};
+/// A far client's blobs on disk (Db.zig): cache.xitdb, by BLAKE3 hash.
+pub const Cache = pardes.Db.Cache;
fn said(why: []u8, comptime fmt: []const u8, args: anytype) ?Joined {
_ = std.fmt.bufPrint(why, fmt ++ "\x00", args) catch {};
@@ -3619,75 +3515,93 @@ test "a join's dump, its blobs put back where they go, is the session's dump" {
try std.testing.expectEqualSlices(u8, whole, joined.items);
}
-test "a cached blob is hashed again as it is read: one cut short, corrupted, planted or longer is a miss, and goes" {
+/// The blob cache of a test, in its own directory (never the user's).
+fn testCache(tmp: *std.testing.TmpDir, buf: *[4200]u8, cap: u64) !Cache {
+ var dir_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const dir = dir_buf[0..try tmp.dir.realPath(std.testing.io, &dir_buf)];
+ return Cache.of(try std.fmt.bufPrint(buf, "{s}/" ++ Cache.name, .{dir}), cap);
+}
+
+test "a cached blob is hashed again as it is read: one kept cut short, corrupted, planted or longer is a miss, and goes" {
const io = std.testing.io;
- var tmp = std.testing.tmpDir(.{ .iterate = true });
+ var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
- const c: Cache = .{ .dir = tmp.dir, .cap = 1 << 20 };
+ var path: [4200]u8 = undefined;
+ const c = try testCache(&tmp, &path, 1 << 20);
var text: [6000]u8 = undefined;
for (&text, 0..) |*b, i| b.* = @truncate(i *% 31 +% i / 7);
var hash: [32]u8 = undefined;
std.crypto.hash.Blake3.hash(&text, &hash, .{});
- const file = Cache.name(&hash);
var out: [6000]u8 = undefined;
try std.testing.expect(!c.get(io, &hash, &out));
- c.put(io, &hash, &text);
- try std.testing.expect(c.has(io, &hash, text.len));
- try std.testing.expect(c.get(io, &hash, &out));
- try std.testing.expectEqualSlices(u8, &text, &out);
var corrupted = text;
corrupted[4000] ^= 0x20;
const planted: [6000]u8 = @splat('x');
var longer: [6001]u8 = undefined;
@memcpy(longer[0..6000], &text);
longer[6000] = '\n';
+ // Each kept under the text's hash, as a torn write or another hand
+ // would leave it: read, it is a miss, and gone.
for ([_][]const u8{ text[0..3000], &corrupted, &planted, &longer }) |bad| {
- try tmp.dir.writeFile(io, .{ .sub_path = &file, .data = bad });
+ c.put(io, &.{.{ .hash = hash, .bytes = bad }}, &.{});
+ try std.testing.expect(c.has(io, &hash, bad.len));
@memset(&out, 0);
try std.testing.expect(!c.get(io, &hash, &out));
- try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(io, &file, .{}));
+ try std.testing.expect(!c.has(io, &hash, bad.len));
}
- // ...and put back, it is read again.
- c.put(io, &hash, &text);
+ // ...and put right, it is read again.
+ c.put(io, &.{.{ .hash = hash, .bytes = &text }}, &.{});
try std.testing.expect(c.get(io, &hash, &out));
try std.testing.expectEqualSlices(u8, &text, &out);
}
-test "the blob cache stays under its cap, the least recently used going first" {
+test "the blob cache stays under its cap, the least recently used going first, the file compacted" {
const io = std.testing.io;
- var tmp = std.testing.tmpDir(.{ .iterate = true });
+ var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
- const c: Cache = .{ .dir = tmp.dir, .cap = 3 * 5000 };
- var blobs: [3][5000]u8 = undefined;
- var hashes: [3][32]u8 = undefined;
+ var path: [4200]u8 = undefined;
+ const c = try testCache(&tmp, &path, 3 * 5000);
+ var blobs: [4][5000]u8 = undefined;
+ var hashes: [4][32]u8 = undefined;
for (&blobs, &hashes, 0..) |*b, *h, k| {
@memset(b, @intCast('a' + k));
std.crypto.hash.Blake3.hash(b, h, .{});
- c.put(io, h, b);
- // Each older than the next, a thousand seconds apart.
- const name = Cache.name(h);
- try tmp.dir.setTimestamps(io, &name, .{ .modify_timestamp = .{ .new = .{ .nanoseconds = @as(i96, @intCast(k + 1)) * 1000 * std.time.ns_per_s } } });
}
+ for (0..3) |k| c.put(io, &.{.{ .hash = hashes[k], .bytes = &blobs[k] }}, &.{});
// The oldest read: it is the newest now.
var out: [5000]u8 = undefined;
try std.testing.expect(c.get(io, &hashes[0], &out));
- c.trim(io, 5000);
+ c.put(io, &.{}, &.{hashes[0]});
+ // Under the cap: nothing goes.
+ c.trim(io);
+ for (hashes[0..3]) |*h| try std.testing.expect(c.has(io, h, 5000));
+ // One more is past it: the least recently used goes.
+ c.put(io, &.{.{ .hash = hashes[3], .bytes = &blobs[3] }}, &.{});
+ const before = (try std.Io.Dir.cwd().statFile(io, c.path(), .{})).size;
+ c.trim(io);
try std.testing.expect(c.has(io, &hashes[0], 5000));
try std.testing.expect(!c.has(io, &hashes[1], 5000));
- try std.testing.expect(c.has(io, &hashes[2], 5000));
- // A file not a blob's counts, and goes by its time as well.
- const junk: [9000]u8 = @splat(0);
- try tmp.dir.writeFile(io, .{ .sub_path = ".tmp-1-1-0123456789abcdef", .data = &junk });
- try tmp.dir.setTimestamps(io, ".tmp-1-1-0123456789abcdef", .{ .modify_timestamp = .{ .new = .{ .nanoseconds = 500 * std.time.ns_per_s } } });
- c.trim(io, 5000);
- try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(io, ".tmp-1-1-0123456789abcdef", .{}));
- try std.testing.expect(c.has(io, &hashes[0], 5000) and c.has(io, &hashes[2], 5000));
- // Under the cap with room to spare: nothing goes.
- c.trim(io, 0);
- try std.testing.expect(c.has(io, &hashes[0], 5000) and c.has(io, &hashes[2], 5000));
- // Room for the whole cap: everything goes, the older first.
- c.trim(io, c.cap);
- try std.testing.expect(!c.has(io, &hashes[0], 5000) and !c.has(io, &hashes[2], 5000));
+ try std.testing.expect(c.has(io, &hashes[2], 5000) and c.has(io, &hashes[3], 5000));
+ try std.testing.expect((try std.Io.Dir.cwd().statFile(io, c.path(), .{})).size < before);
+ // Room for nothing: everything goes.
+ Cache.of(c.path(), 0).trim(io);
+ for (&hashes) |*h| try std.testing.expect(!c.has(io, h, 5000));
+}
+
+test "a blob cache that is no database is started afresh" {
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var path: [4200]u8 = undefined;
+ const c = try testCache(&tmp, &path, 1 << 20);
+ try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = c.path(), .data = "xit but not a database, cut short" });
+ const text: [5000]u8 = @splat('t');
+ var hash: [32]u8 = undefined;
+ std.crypto.hash.Blake3.hash(&text, &hash, .{});
+ var out: [5000]u8 = undefined;
+ try std.testing.expect(!c.get(io, &hash, &out));
+ c.put(io, &.{.{ .hash = hash, .bytes = &text }}, &.{});
+ try std.testing.expect(c.get(io, &hash, &out));
}
/// A far join over a socket pair, the session's writer and reader threads
@@ -3785,9 +3699,10 @@ test "a far join is sent each blob once: joined again with them on disk or in th
}
try std.testing.expect(distinct.count() >= 3);
- var tmp = std.testing.tmpDir(.{ .iterate = true });
+ var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
- var have: Have = .{ .io = io, .cache = .{ .dir = tmp.dir, .cap = 64 << 20 } };
+ var path: [4200]u8 = undefined;
+ var have: Have = .{ .io = io, .cache = try testCache(&tmp, &path, 64 << 20) };
defer {
have.cache = null;
have.deinit();
@@ -3813,12 +3728,13 @@ test "a far join is sent each blob once: joined again with them on disk or in th
try std.testing.expectEqual(distinct.count(), j.from_disk);
try std.testing.expectEqual(@as(u64, 0), j.sent_bytes);
endJoin(&j, &have);
- // The biggest's file corrupted: it alone is sent again.
- const name = Cache.name(&biggest);
+ // The biggest kept wrong (read back once as another length, it goes;
+ // then junk under its hash): it alone is sent again.
const junk = try gpa.alloc(u8, biggest_len);
defer gpa.free(junk);
@memset(junk, 'z');
- try tmp.dir.writeFile(io, .{ .sub_path = &name, .data = junk });
+ try std.testing.expect(!have.cache.?.get(io, &biggest, junk[0 .. biggest_len - 1]));
+ have.cache.?.put(io, &.{.{ .hash = biggest, .bytes = junk }}, &.{});
j = try farJoinOnce(&ps, a.core, &have);
try std.testing.expectEqualSlices(u8, whole, j.dump);
try std.testing.expectEqual(distinct.count() - 1, j.from_disk);
diff --git a/test/daemon.py b/test/daemon.py
index 0c4c2f3c..074c860c 100644
--- a/test/daemon.py
+++ b/test/daemon.py
@@ -820,10 +820,15 @@ def joins(path):
return out
+def cache_file(cache):
+ """The blob cache (a far client's, XDG_CACHE_HOME/pardes/cache.xitdb)."""
+ return cache / 'pardes' / 'cache.xitdb'
+
+
def cached(cache):
- """The blob cache's files (a far client's, XDG_CACHE_HOME/pardes/blobs)."""
- d = cache / 'pardes' / 'blobs'
- return {p.name: p.stat().st_size for p in d.iterdir() if p.is_file()} if d.exists() else {}
+ """The blob cache's size in bytes, 0 with none."""
+ f = cache_file(cache)
+ return f.stat().st_size if f.exists() else 0
def far(binary):
@@ -832,9 +837,10 @@ def far(binary):
blob and keeps them; a second on the same machine, and one put off the
session's state that joins again, are sent none they hold, from disk
and from the core they leave; one started after the first ended gets
- what it made from disk; one after the cache was corrupted and cut short
- by hand, and one with the cache full, are sent what they lack. Every
- client's digest is the session's once quiet."""
+ what it made from disk; one after the cache was cut short by hand (a
+ file that is no database: started afresh), and one with the cache full,
+ are sent what they lack. Every client's digest is the session's once
+ quiet."""
scratch = Path(__file__).resolve().parent.parent.parent / '.scratch' / 'daemon' / 'dmn'
scratch.mkdir(parents=True, exist_ok=True)
root = Path(tempfile.mkdtemp(prefix='rf-', dir=scratch))
@@ -895,7 +901,8 @@ def far(binary):
# A: the first, its cache empty: every blob sent, and kept.
ta, a = client('a')
typed('a', ta, b'from-A')
- wait_for('client A\'s blobs on disk', lambda: len(cached(cache)) >= 2)
+ # The notes and the big file, a megabyte and more between them.
+ wait_for('client A\'s blobs on disk', lambda: cached(cache) > 1 << 20)
time.sleep(.5)
# B: a second client on this machine, and put off the session's
# state once quiet: its first join from disk, its rejoin from the
@@ -921,37 +928,28 @@ def far(binary):
detach('c', tc, c)
c_join = joins(root / 'c.stats')[0]
assert c_join['from_disk'] >= c_join['blobs'] - 1 and c_join['sent_kb'] < 64, c_join
- # D: the cache corrupted and cut short by hand: a miss each, sent again.
- files = sorted(cached(cache).items(), key=lambda f: -f[1])
- for k, (name, size) in enumerate(files):
- p = cache / 'pardes' / 'blobs' / name
- if k % 2:
- p.write_bytes(p.read_bytes()[:size // 2])
- else:
- data = bytearray(p.read_bytes())
- data[size // 2] ^= 0x55
- p.write_bytes(bytes(data))
+ # D: the cache cut short by hand: no database, started afresh, every
+ # blob sent again.
+ whole = cache_file(cache).read_bytes()
+ cache_file(cache).write_bytes(whole[:len(whole) // 2])
td, d = client('d')
typed('d', td, b'from-D')
detach('d', td, d)
d_join = joins(root / 'd.stats')[0]
assert d_join['from_disk'] == 0 and d_join['sent_kb'] > 100, d_join
- # E: the cache full (a 1 MiB cap, held by older files): they go first,
- # and the cache stays under its cap.
- for k in range(2):
- p = cache / 'pardes' / 'blobs' / f'junk-{k}'
- p.write_bytes(b'j' * (512 << 10))
- os.utime(p, (1000 + k, 1000 + k))
+ # E: the cache full (D's, past a 1 MiB cap): the oldest go at its end,
+ # the file compacted, under its cap but for xitdb's own few KB.
+ before_e = cached(cache)
+ assert before_e > 1 << 20, before_e
te, e = client('e', {'PARDES_BLOB_CACHE_MB': '1'})
typed('e', te, b'from-E')
detach('e', te, e)
- left = cached(cache)
- assert not [n for n in left if n.startswith('junk')] and sum(left.values()) <= 1 << 20, left
+ assert cached(cache) <= (1 << 20) + (64 << 10), cached(cache)
tf, f = client('f', {'PARDES_BLOB_CACHE_MB': '1'})
typed('f', tf, b'from-F')
detach('f', tf, f)
f_join = joins(root / 'f.stats')[0]
- assert f_join['from_disk'] > 0 and sum(cached(cache).values()) <= 1 << 20, (f_join, cached(cache))
+ assert f_join['from_disk'] > 0 and cached(cache) <= (1 << 20) + (64 << 10), (f_join, cached(cache))
far_joins(root / 'pardes-replica.sock', 'files', binary, 200)
# Through ssh (a stand-in forwarding as `ssh -L` does): a client
# refused and one detached each end their ssh and its socket (one