diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Db.zig | 255 | ||||
| -rw-r--r-- | src/replica.zig | 272 |
2 files changed, 349 insertions, 178 deletions
@@ -762,6 +762,261 @@ fn liveSize(db: *Disk) !u64 { return live; } +// ---- a far client's blob cache ---- + +/// A far replica client's blob cache (replica.zig `Have`): `cache.xitdb` in +/// $XDG_CACHE_HOME/pardes (else ~/.cache/pardes), the texts a join was +/// sent, by their BLAKE3 hash, so a client started again or a second one on +/// this machine is not sent them again. Expendable, and up to a gigabyte, so +/// a file of its own, never the state file's: +/// +/// blobs/<hash> the text +/// seqs/<hash> its number in `seen` +/// seen a sorted map, a number (big endian) to its blob's hash: +/// the least recently used first (a hit takes a new number) +/// size, next the texts' bytes, the next number +/// +/// Past the cap (PARDES_BLOB_CACHE_MB, else 1 GiB) the oldest go, and the +/// file is compacted into `cache.xitdb.new` and renamed over it: one +/// transaction and one compaction, at a far client's end (`trim`), never on +/// a frame. A text read back is hashed again: one torn or changed is a miss +/// and goes, never wrong content; a file that is no database is started +/// afresh (it holds nothing a join cannot send again). Each use opens the +/// file for itself: a client's reader, its storing thread and its end each +/// have a database of their own, as xitdb asks. +pub const Cache = struct { + path_buf: [4096]u8 = undefined, + path_len: usize = 0, + /// Bytes it keeps at most. + cap: u64, + + pub const name = "cache.xitdb"; + + /// The cache's place, its directory made: null with no home for it. + pub fn open(io: std.Io) ?Cache { + var c: Cache = .{ .cap = (if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024) << 20 }; + const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else ""; + const where = (if (base.len > 0 and base[0] == '/') + std.fmt.bufPrint(&c.path_buf, "{s}/pardes/" ++ name, .{base}) + else if (std.c.getenv("HOME")) |home| + std.fmt.bufPrint(&c.path_buf, "{s}/.cache/pardes/" ++ name, .{std.mem.span(home)}) + else + return null) catch return null; + std.Io.Dir.cwd().createDirPath(io, std.fs.path.dirname(where).?) catch return null; + c.path_len = where.len; + return c; + } + + /// The cache at `where` (a test's). + pub fn of(where: []const u8, cap: u64) Cache { + var c: Cache = .{ .cap = cap, .path_len = where.len }; + @memcpy(c.path_buf[0..where.len], where); + return c; + } + + pub fn path(c: *const Cache) []const u8 { + return c.path_buf[0..c.path_len]; + } + + /// The cache open for one use. + const Use = struct { + file: std.Io.File, + buffer: std.Io.Writer.Allocating, + db: Disk, + }; + + fn use(c: *const Cache, io: std.Io, u: *Use) !void { + const cwd = std.Io.Dir.cwd(); + u.buffer = .init(std.heap.c_allocator); + errdefer u.buffer.deinit(); + for (0..2) |_| { + u.file = try cwd.createFile(io, c.path(), .{ .read = true, .truncate = false }); + u.db = Disk.init(io, .{ .file = u.file, .buffer = &u.buffer, .fsync = false, .hash_id = hash_id }) catch |err| { + u.file.close(io); + // No database: started afresh, as nothing in it is precious. + cwd.deleteFile(io, c.path()) catch return err; + continue; + }; + return; + } + return error.InvalidDatabase; + } + + fn done(u: *Use, io: std.Io) void { + u.file.close(io); + u.buffer.deinit(); + } + + fn mapAt(root: Disk.HashMap(.read_only), field: []const u8) !?Disk.HashMap(.read_only) { + const c = (try root.getCursor(key(field))) orelse return null; + return try Disk.HashMap(.read_only).init(c); + } + + /// The text of `hash` read into `out` (its length) and hashed again. + /// One of another length or content is a miss, and goes. + pub fn get(c: *const Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const ok = ok: { + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch break :ok false; + const blobs = (mapAt(root, "blobs") catch break :ok false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch break :ok false) orelse return false; + if ((at.count() catch break :ok false) != out.len) break :ok false; + _ = at.readBytes(out) catch break :ok false; + var d: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(out, &d, .{}); + break :ok std.mem.eql(u8, &d, hash); + }; + if (!ok) drop(&u, io, hash); + return ok; + } + + /// Whether a text of `len` bytes is there under `hash` (not read). + pub fn has(c: *const Cache, io: std.Io, hash: *const [32]u8, len: usize) bool { + var u: Use = undefined; + c.use(io, &u) catch return false; + defer done(&u, io); + const root = Disk.HashMap(.read_only).init(u.db.rootCursor().readOnly()) catch return false; + const blobs = (mapAt(root, "blobs") catch return false) orelse return false; + const at = (blobs.getCursor(blobKey(hash)) catch return false) orelse return false; + return (at.count() catch return false) == len; + } + + /// The text of `hash`, gone (one read back wrong). + fn drop(u: *Use, io: std.Io, hash: *const [32]u8) void { + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + hash: *const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const blobs = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const at = (try blobs.getCursor(blobKey(ctx.hash))) orelse return; + const len = try at.count(); + _ = try blobs.remove(blobKey(ctx.hash)); + const size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + try root.put(key("size"), .{ .uint = size -| len }); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + if (try seqs.getCursor(blobKey(ctx.hash))) |s| { + const was = try s.readUint(); + _ = try seqs.remove(blobKey(ctx.hash)); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + _ = try seen.remove(&seqKey(was)); + } + } + }{ .hash = hash }) catch {}; + } + + fn seqKey(seq: u64) [8]u8 { + var k: [8]u8 = undefined; + std.mem.writeInt(u64, &k, seq, .big); + return k; + } + + /// `blobs` kept (each text not there yet, under its hash) and `hits`, + /// texts read from here, made the most recently used: one transaction, + /// the file's lock waited for (a storing thread's, a client's end's). + pub fn put(c: *const Cache, io: std.Io, blobs: []const pardes.walk.Blob, hits: []const [32]u8) void { + if (blobs.len == 0 and hits.len == 0) return; + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + u.db.runContext(struct { + blobs: []const pardes.walk.Blob, + hits: []const [32]u8, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + var next = if (try root.getCursor(key("next"))) |n| try n.readUint() else 0; + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + for (ctx.blobs) |*b| { + if (try texts.getCursor(blobKey(&b.hash)) != null) continue; + try texts.put(blobKey(&b.hash), .{ .bytes = b.bytes }); + try seqs.put(blobKey(&b.hash), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = &b.hash }); + next += 1; + size += b.bytes.len; + } + for (ctx.hits) |*h| { + const at = (try seqs.getCursor(blobKey(h))) orelse continue; + _ = try seen.remove(&seqKey(try at.readUint())); + try seqs.put(blobKey(h), .{ .uint = next }); + try seen.put(&seqKey(next), .{ .bytes = h }); + next += 1; + } + try root.put(key("next"), .{ .uint = next }); + try root.put(key("size"), .{ .uint = size }); + } + }{ .blobs = blobs, .hits = hits }) catch {}; + } + + /// Under the cap: the least recently used texts gone, in one + /// transaction, and the file compacted into `<path>.new` and renamed + /// over it once any went (what no moment holds is the file's growth). + pub fn trim(c: *const Cache, io: std.Io) void { + var u: Use = undefined; + c.use(io, &u) catch return; + defer done(&u, io); + u.file.lock(io, .exclusive) catch return; + defer u.file.unlock(io); + var went = false; + u.db.runContext(struct { + cap: u64, + went: *bool, + pub fn run(ctx: @This(), cursor: *Disk.Cursor(.read_write)) !void { + const root = try Disk.HashMap(.read_write).init(cursor.*); + var size = if (try root.getCursor(key("size"))) |s| try s.readUint() else 0; + if (size <= ctx.cap) return; + const texts = try Disk.HashMap(.read_write).init(try root.putCursor(key("blobs"))); + const seqs = try Disk.HashMap(.read_write).init(try root.putCursor(key("seqs"))); + const seen = try Disk.SortedMap(.read_write).init(try root.putCursor(key("seen"))); + // The oldest numbers, read before any goes. + var gone: std.ArrayList(struct { seq: [8]u8, hash: [32]u8 }) = .empty; + defer gone.deinit(std.heap.c_allocator); + var left = size; + var it = try seen.iterator(); + while (left > ctx.cap) { + const kv_at = (try it.next()) orelse break; + const kv = try kv_at.readKeyValuePair(); + var g: @TypeOf(gone.items[0]) = undefined; + if ((try kv.key_cursor.readBytes(&g.seq)).len != 8 or (try kv.value_cursor.readBytes(&g.hash)).len != 32) continue; + if (try texts.getCursor(blobKey(&g.hash))) |at| left -|= try at.count(); + try gone.append(std.heap.c_allocator, g); + } + for (gone.items) |*g| { + if (try texts.getCursor(blobKey(&g.hash))) |at| size -|= try at.count(); + _ = try texts.remove(blobKey(&g.hash)); + _ = try seqs.remove(blobKey(&g.hash)); + _ = try seen.remove(&g.seq); + } + try root.put(key("size"), .{ .uint = size }); + ctx.went.* = gone.items.len > 0; + } + }{ .cap = c.cap, .went = &went }) catch return; + if (!went) return; + var new_buf: [4200]u8 = undefined; + const new = std.fmt.bufPrint(&new_buf, "{s}.new", .{c.path()}) catch return; + const cwd = std.Io.Dir.cwd(); + const file = cwd.createFile(io, new, .{ .read = true, .truncate = true }) catch return; + var buffer: std.Io.Writer.Allocating = .init(std.heap.c_allocator); + defer buffer.deinit(); + var offsets: std.AutoHashMap(u64, u64) = .init(std.heap.c_allocator); + defer offsets.deinit(); + _ = u.db.compact(.buffered_file, .{ .file = file, .buffer = &buffer, .fsync = false, .hash_id = hash_id }, &offsets) catch { + file.close(io); + cwd.deleteFile(io, new) catch {}; + return; + }; + file.close(io); + cwd.rename(new, cwd, c.path(), io) catch cwd.deleteFile(io, new) catch {}; + } +}; + // ---- tests ---- const testing = std.testing; diff --git a/src/replica.zig b/src/replica.zig index 5bcc0a53..b98d792b 100644 --- a/src/replica.zig +++ b/src/replica.zig @@ -2421,6 +2421,8 @@ fn joinOn(gpa: Allocator, fd: c_int, session: []const u8, why: []u8, have: ?*Hav pub const Have = struct { io: std.Io, cache: ?Cache = null, + /// The blobs a join found on disk: the most recently used once stored. + hits: std.ArrayListUnmanaged([32]u8) = .empty, /// The core it leaves (`leaving`), each blob hashed once when one of its /// length is looked for. ponytail: a scan a lookup, fine to some /// thousands of blobs; a map by length would take over past that. @@ -2438,7 +2440,7 @@ pub const Have = struct { h.left(); h.old.deinit(std.heap.c_allocator); h.old_hash.deinit(std.heap.c_allocator); - if (h.cache) |c| c.dir.close(h.io); + h.hits.deinit(std.heap.c_allocator); } fn wait(h: *Have) void { @@ -2475,189 +2477,83 @@ pub const Have = struct { @memcpy(out, b); return .memory; } - if (h.cache) |c| if (c.get(h.io, hash, out)) return .disk; + if (h.cache) |*c| if (c.get(h.io, hash, out)) { + h.hits.append(std.heap.c_allocator, hash.*) catch {}; + return .disk; + }; return null; } /// The blobs a join was sent, written to the cache on a thread of their - /// own, which frees the dump (`gpa`'s) and the list after: the first - /// frame waits for none of it. + /// own (with those it found there, made the most recently used), which + /// frees the dump (`gpa`'s) and the lists after: the first frame waits + /// for none of it. pub fn store(h: *Have, gpa: Allocator, j: *Joined) void { const dump = j.dump; const got = j.got; j.dump = &.{}; j.got = &.{}; h.wait(); + var hits = h.hits; + h.hits = .empty; const Store = struct { - fn run(c: Cache, io: std.Io, a: Allocator, bytes: []u8, list: []Got) void { + fn run(c: Cache, io: std.Io, a: Allocator, bytes: []u8, list: []Got, found: std.ArrayListUnmanaged([32]u8)) void { + var seen = found; defer { a.free(bytes); std.heap.c_allocator.free(list); + seen.deinit(std.heap.c_allocator); } - var incoming: u64 = 0; - for (list) |g| incoming += g.len; - c.trim(io, @min(incoming, c.cap)); + var blobs: std.ArrayListUnmanaged(walk.Blob) = .empty; + defer blobs.deinit(std.heap.c_allocator); var put: u64 = 0; for (list) |g| { if (put + g.len > c.cap) continue; put += g.len; - c.put(io, &g.hash, bytes[g.pos..][0..g.len]); + blobs.append(std.heap.c_allocator, .{ .hash = g.hash, .bytes = bytes[g.pos..][0..g.len] }) catch return; } + c.put(io, blobs.items, seen.items); } }; const c = h.cache orelse { gpa.free(dump); std.heap.c_allocator.free(got); + hits.deinit(std.heap.c_allocator); return; }; - h.storing = std.Thread.spawn(.{}, Store.run, .{ c, h.io, gpa, dump, got }) catch { - Store.run(c, h.io, gpa, dump, got); + h.storing = std.Thread.spawn(.{}, Store.run, .{ c, h.io, gpa, dump, got, hits }) catch { + Store.run(c, h.io, gpa, dump, got, hits); return; }; } /// `core`'s blobs the cache lacks, written there: what this client made - /// since it joined (an edit's text, its undo), for its next start. + /// since it joined (an edit's text, its undo), for its next start; then + /// the cache trimmed under its cap, at this client's end. pub fn keep(h: *Have, core: *const Pardes) void { h.wait(); const c = h.cache orelse return; var all: std.ArrayListUnmanaged([]const u8) = .empty; defer all.deinit(std.heap.c_allocator); walk.blobs(Pardes, core, &all); - var missing: std.ArrayListUnmanaged(struct { bytes: []const u8, hash: [32]u8 }) = .empty; + var missing: std.ArrayListUnmanaged(walk.Blob) = .empty; defer missing.deinit(std.heap.c_allocator); - var incoming: u64 = 0; + var put: u64 = 0; for (all.items) |b| { var d: [32]u8 = undefined; std.crypto.hash.Blake3.hash(b, &d, .{}); - if (c.has(h.io, &d, b.len)) continue; + if (put + b.len > c.cap or c.has(h.io, &d, b.len)) continue; + put += b.len; missing.append(std.heap.c_allocator, .{ .bytes = b, .hash = d }) catch return; - incoming += b.len; - } - if (missing.items.len == 0) return; - c.trim(h.io, @min(incoming, c.cap)); - var put: u64 = 0; - for (missing.items) |m| { - if (put + m.bytes.len > c.cap) continue; - put += m.bytes.len; - c.put(h.io, &m.hash, m.bytes); } + c.put(h.io, missing.items, h.hits.items); + h.hits.clearRetainingCapacity(); + c.trim(h.io); } }; -/// A far client's blobs on disk, a file each named by its BLAKE3 hash in -/// hex, in $XDG_CACHE_HOME/pardes/blobs (else ~/.cache/pardes/blobs): what -/// a client started again, or a second one on this machine, is not sent -/// again. The cache directory and not the runtime one: the blobs are worth -/// keeping across a logout or a reboot and safe to lose, and they can be -/// hundreds of megabytes, which the runtime directory (a tmpfs, often -/// capped at a tenth of memory and emptied at logout) is not for. A file -/// read back is hashed again: one cut short, corrupted or planted is a -/// miss and goes, never wrong content. -pub const Cache = struct { - dir: std.Io.Dir, - /// Bytes it keeps at most: PARDES_BLOB_CACHE_MB, else 1 GiB. - cap: u64, - - pub fn open(io: std.Io) ?Cache { - var buf: [4096]u8 = undefined; - const base = if (std.c.getenv("XDG_CACHE_HOME")) |x| std.mem.span(x) else ""; - const path = if (base.len > 0 and base[0] == '/') - std.fmt.bufPrint(&buf, "{s}/pardes/blobs", .{base}) catch return null - else if (std.c.getenv("HOME")) |home| - std.fmt.bufPrint(&buf, "{s}/.cache/pardes/blobs", .{std.mem.span(home)}) catch return null - else - return null; - const cwd = std.Io.Dir.cwd(); - cwd.createDirPath(io, std.fs.path.dirname(path).?) catch return null; - cwd.createDir(io, path, .fromMode(0o700)) catch |err| if (err != error.PathAlreadyExists) return null; - const mb = if (std.c.getenv("PARDES_BLOB_CACHE_MB")) |v| std.fmt.parseInt(u64, std.mem.span(v), 10) catch 1024 else 1024; - return .{ .dir = cwd.openDir(io, path, .{ .iterate = true }) catch return null, .cap = mb << 20 }; - } - - fn name(hash: *const [32]u8) [64]u8 { - return std.fmt.bytesToHex(hash.*, .lower); - } - - /// The blob of `hash` read into `out` (its length) and hashed again. - /// A file of another length or content is a miss, and goes; a hit's - /// time is made now, so the least recently used go first (`trim`). - pub fn get(c: Cache, io: std.Io, hash: *const [32]u8, out: []u8) bool { - const file = name(hash); - var f = c.dir.openFile(io, &file, .{}) catch return false; - const ok = ok: { - defer f.close(io); - const st = f.stat(io) catch break :ok false; - if (st.size != out.len) break :ok false; - var reader = f.reader(io, &.{}); - const n = reader.interface.readSliceShort(out) catch break :ok false; - if (n != out.len) break :ok false; - var d: [32]u8 = undefined; - std.crypto.hash.Blake3.hash(out, &d, .{}); - break :ok std.mem.eql(u8, &d, hash); - }; - if (!ok) { - c.dir.deleteFile(io, &file) catch {}; - return false; - } - c.dir.setTimestampsNow(io, &file, .{}) catch {}; - return true; - } - - /// Whether a file of `len` bytes is there under `hash` (not read). - pub fn has(c: Cache, io: std.Io, hash: *const [32]u8, len: usize) bool { - const file = name(hash); - const st = c.dir.statFile(io, &file, .{}) catch return false; - return st.size == len; - } - - var tmp_serial: std.atomic.Value(u32) = .init(0); - - /// `bytes` kept under `hash`: written aside and renamed into place, so - /// that a reader never meets half a file (and would hash it wrong). - pub fn put(c: Cache, io: std.Io, hash: *const [32]u8, bytes: []const u8) void { - const file = name(hash); - if (c.has(io, hash, bytes.len)) return; - var buf: [96]u8 = undefined; - const tmp = std.fmt.bufPrint(&buf, ".tmp-{d}-{d}-{s}", .{ std.c.getpid(), tmp_serial.fetchAdd(1, .monotonic), file[0..16] }) catch return; - c.dir.writeFile(io, .{ .sub_path = tmp, .data = bytes }) catch { - c.dir.deleteFile(io, tmp) catch {}; - return; - }; - c.dir.rename(tmp, c.dir, &file, io) catch c.dir.deleteFile(io, tmp) catch {}; - } - - /// Room for `incoming` more bytes under the cap, the least recently - /// used files deleted first (a hit makes its file's time now). - /// ponytail: the directory scanned and sorted at each join's store; - /// fine to tens of thousands of blobs, where an index would take over. - pub fn trim(c: Cache, io: std.Io, incoming: u64) void { - const File = struct { name: [80]u8, len: u8, size: u64, mtime: i96 }; - var files: std.ArrayListUnmanaged(File) = .empty; - defer files.deinit(std.heap.c_allocator); - var total: u64 = 0; - var it = c.dir.iterate(); - while (it.next(io) catch null) |entry| { - if (entry.kind != .file or entry.name.len > 80) continue; - const st = c.dir.statFile(io, entry.name, .{}) catch continue; - var f: File = .{ .name = undefined, .len = @intCast(entry.name.len), .size = st.size, .mtime = st.mtime.nanoseconds }; - @memcpy(f.name[0..f.len], entry.name); - files.append(std.heap.c_allocator, f) catch return; - total += st.size; - } - if (total + incoming <= c.cap) return; - std.mem.sort(File, files.items, {}, struct { - fn older(_: void, a: File, b: File) bool { - return a.mtime < b.mtime; - } - }.older); - for (files.items) |f| { - if (total + incoming <= c.cap) break; - c.dir.deleteFile(io, f.name[0..f.len]) catch continue; - total -= f.size; - } - } -}; +/// A far client's blobs on disk (Db.zig): cache.xitdb, by BLAKE3 hash. +pub const Cache = pardes.Db.Cache; fn said(why: []u8, comptime fmt: []const u8, args: anytype) ?Joined { _ = std.fmt.bufPrint(why, fmt ++ "\x00", args) catch {}; @@ -3619,75 +3515,93 @@ test "a join's dump, its blobs put back where they go, is the session's dump" { try std.testing.expectEqualSlices(u8, whole, joined.items); } -test "a cached blob is hashed again as it is read: one cut short, corrupted, planted or longer is a miss, and goes" { +/// The blob cache of a test, in its own directory (never the user's). +fn testCache(tmp: *std.testing.TmpDir, buf: *[4200]u8, cap: u64) !Cache { + var dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir = dir_buf[0..try tmp.dir.realPath(std.testing.io, &dir_buf)]; + return Cache.of(try std.fmt.bufPrint(buf, "{s}/" ++ Cache.name, .{dir}), cap); +} + +test "a cached blob is hashed again as it is read: one kept cut short, corrupted, planted or longer is a miss, and goes" { const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - const c: Cache = .{ .dir = tmp.dir, .cap = 1 << 20 }; + var path: [4200]u8 = undefined; + const c = try testCache(&tmp, &path, 1 << 20); var text: [6000]u8 = undefined; for (&text, 0..) |*b, i| b.* = @truncate(i *% 31 +% i / 7); var hash: [32]u8 = undefined; std.crypto.hash.Blake3.hash(&text, &hash, .{}); - const file = Cache.name(&hash); var out: [6000]u8 = undefined; try std.testing.expect(!c.get(io, &hash, &out)); - c.put(io, &hash, &text); - try std.testing.expect(c.has(io, &hash, text.len)); - try std.testing.expect(c.get(io, &hash, &out)); - try std.testing.expectEqualSlices(u8, &text, &out); var corrupted = text; corrupted[4000] ^= 0x20; const planted: [6000]u8 = @splat('x'); var longer: [6001]u8 = undefined; @memcpy(longer[0..6000], &text); longer[6000] = '\n'; + // Each kept under the text's hash, as a torn write or another hand + // would leave it: read, it is a miss, and gone. for ([_][]const u8{ text[0..3000], &corrupted, &planted, &longer }) |bad| { - try tmp.dir.writeFile(io, .{ .sub_path = &file, .data = bad }); + c.put(io, &.{.{ .hash = hash, .bytes = bad }}, &.{}); + try std.testing.expect(c.has(io, &hash, bad.len)); @memset(&out, 0); try std.testing.expect(!c.get(io, &hash, &out)); - try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(io, &file, .{})); + try std.testing.expect(!c.has(io, &hash, bad.len)); } - // ...and put back, it is read again. - c.put(io, &hash, &text); + // ...and put right, it is read again. + c.put(io, &.{.{ .hash = hash, .bytes = &text }}, &.{}); try std.testing.expect(c.get(io, &hash, &out)); try std.testing.expectEqualSlices(u8, &text, &out); } -test "the blob cache stays under its cap, the least recently used going first" { +test "the blob cache stays under its cap, the least recently used going first, the file compacted" { const io = std.testing.io; - var tmp = std.testing.tmpDir(.{ .iterate = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - const c: Cache = .{ .dir = tmp.dir, .cap = 3 * 5000 }; - var blobs: [3][5000]u8 = undefined; - var hashes: [3][32]u8 = undefined; + var path: [4200]u8 = undefined; + const c = try testCache(&tmp, &path, 3 * 5000); + var blobs: [4][5000]u8 = undefined; + var hashes: [4][32]u8 = undefined; for (&blobs, &hashes, 0..) |*b, *h, k| { @memset(b, @intCast('a' + k)); std.crypto.hash.Blake3.hash(b, h, .{}); - c.put(io, h, b); - // Each older than the next, a thousand seconds apart. - const name = Cache.name(h); - try tmp.dir.setTimestamps(io, &name, .{ .modify_timestamp = .{ .new = .{ .nanoseconds = @as(i96, @intCast(k + 1)) * 1000 * std.time.ns_per_s } } }); } + for (0..3) |k| c.put(io, &.{.{ .hash = hashes[k], .bytes = &blobs[k] }}, &.{}); // The oldest read: it is the newest now. var out: [5000]u8 = undefined; try std.testing.expect(c.get(io, &hashes[0], &out)); - c.trim(io, 5000); + c.put(io, &.{}, &.{hashes[0]}); + // Under the cap: nothing goes. + c.trim(io); + for (hashes[0..3]) |*h| try std.testing.expect(c.has(io, h, 5000)); + // One more is past it: the least recently used goes. + c.put(io, &.{.{ .hash = hashes[3], .bytes = &blobs[3] }}, &.{}); + const before = (try std.Io.Dir.cwd().statFile(io, c.path(), .{})).size; + c.trim(io); try std.testing.expect(c.has(io, &hashes[0], 5000)); try std.testing.expect(!c.has(io, &hashes[1], 5000)); - try std.testing.expect(c.has(io, &hashes[2], 5000)); - // A file not a blob's counts, and goes by its time as well. - const junk: [9000]u8 = @splat(0); - try tmp.dir.writeFile(io, .{ .sub_path = ".tmp-1-1-0123456789abcdef", .data = &junk }); - try tmp.dir.setTimestamps(io, ".tmp-1-1-0123456789abcdef", .{ .modify_timestamp = .{ .new = .{ .nanoseconds = 500 * std.time.ns_per_s } } }); - c.trim(io, 5000); - try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(io, ".tmp-1-1-0123456789abcdef", .{})); - try std.testing.expect(c.has(io, &hashes[0], 5000) and c.has(io, &hashes[2], 5000)); - // Under the cap with room to spare: nothing goes. - c.trim(io, 0); - try std.testing.expect(c.has(io, &hashes[0], 5000) and c.has(io, &hashes[2], 5000)); - // Room for the whole cap: everything goes, the older first. - c.trim(io, c.cap); - try std.testing.expect(!c.has(io, &hashes[0], 5000) and !c.has(io, &hashes[2], 5000)); + try std.testing.expect(c.has(io, &hashes[2], 5000) and c.has(io, &hashes[3], 5000)); + try std.testing.expect((try std.Io.Dir.cwd().statFile(io, c.path(), .{})).size < before); + // Room for nothing: everything goes. + Cache.of(c.path(), 0).trim(io); + for (&hashes) |*h| try std.testing.expect(!c.has(io, h, 5000)); +} + +test "a blob cache that is no database is started afresh" { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path: [4200]u8 = undefined; + const c = try testCache(&tmp, &path, 1 << 20); + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = c.path(), .data = "xit but not a database, cut short" }); + const text: [5000]u8 = @splat('t'); + var hash: [32]u8 = undefined; + std.crypto.hash.Blake3.hash(&text, &hash, .{}); + var out: [5000]u8 = undefined; + try std.testing.expect(!c.get(io, &hash, &out)); + c.put(io, &.{.{ .hash = hash, .bytes = &text }}, &.{}); + try std.testing.expect(c.get(io, &hash, &out)); } /// A far join over a socket pair, the session's writer and reader threads @@ -3785,9 +3699,10 @@ test "a far join is sent each blob once: joined again with them on disk or in th } try std.testing.expect(distinct.count() >= 3); - var tmp = std.testing.tmpDir(.{ .iterate = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - var have: Have = .{ .io = io, .cache = .{ .dir = tmp.dir, .cap = 64 << 20 } }; + var path: [4200]u8 = undefined; + var have: Have = .{ .io = io, .cache = try testCache(&tmp, &path, 64 << 20) }; defer { have.cache = null; have.deinit(); @@ -3813,12 +3728,13 @@ test "a far join is sent each blob once: joined again with them on disk or in th try std.testing.expectEqual(distinct.count(), j.from_disk); try std.testing.expectEqual(@as(u64, 0), j.sent_bytes); endJoin(&j, &have); - // The biggest's file corrupted: it alone is sent again. - const name = Cache.name(&biggest); + // The biggest kept wrong (read back once as another length, it goes; + // then junk under its hash): it alone is sent again. const junk = try gpa.alloc(u8, biggest_len); defer gpa.free(junk); @memset(junk, 'z'); - try tmp.dir.writeFile(io, .{ .sub_path = &name, .data = junk }); + try std.testing.expect(!have.cache.?.get(io, &biggest, junk[0 .. biggest_len - 1])); + have.cache.?.put(io, &.{.{ .hash = biggest, .bytes = junk }}, &.{}); j = try farJoinOnce(&ps, a.core, &have); try std.testing.expectEqualSlices(u8, whole, j.dump); try std.testing.expectEqual(distinct.count() - 1, j.from_disk); |
