From 1eae252b80ddab0a06574de700b6b2957d3b64fc Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Thu, 5 Nov 2026 15:20:35 -0300 Subject: blob cache: a far client's blobs are kept in `cache.xitdb` (src/Db.zig's Cache, in $XDG_CACHE_HOME/pardes, else ~/.cache/pardes), a file of its own and never the state file (expendable, and up to a gigabyte): `blobs/` the texts, `seqs/` and `seen` (a sorted map, a number to its hash, the least recently used first) their order, `size` and `next`; a join's blobs are put in one transaction by the client's storing thread, with those it found there made the most recently used (a hit takes a new number), and at the client's end what it made since is put and the cache trimmed: past the cap (PARDES_BLOB_CACHE_MB, 1 GiB) the oldest go in one transaction and the file is compacted into `cache.xitdb.new` and renamed over it, where a directory of files was scanned, sorted and deleted from at each join; each use opens the file for itself (a database a thread, as xitdb asks), a text read back is hashed again (one kept wrong is a miss, and goes), and a file that is no database is started afresh; replica.zig's Cache (its directory, the hex names, put by rename, trim by scan) goes for Db.Cache; tests: replica.zig's 'a cached blob is hashed again as it is read: one cut short, corrupted, planted or longer is a miss, and goes' and 'the blob cache stays under its cap, the least recently used going first' are rewritten for the file (texts kept wrong under a hash through put, the order made by hits, and the file seen compacted), 'a blob cache that is no database is started afresh' is new, the far join test keeps its biggest blob wrong through the cache rather than a file, and daemon.py's far clients read the cache file's size, cut it short by hand where they corrupted files, and check the cap after a client's end where they planted junk files Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016Y9MNAtZenFv4weoMD3cge --- test/daemon.py | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) (limited to 'test') diff --git a/test/daemon.py b/test/daemon.py index 0c4c2f3c..074c860c 100644 --- a/test/daemon.py +++ b/test/daemon.py @@ -820,10 +820,15 @@ def joins(path): return out +def cache_file(cache): + """The blob cache (a far client's, XDG_CACHE_HOME/pardes/cache.xitdb).""" + return cache / 'pardes' / 'cache.xitdb' + + def cached(cache): - """The blob cache's files (a far client's, XDG_CACHE_HOME/pardes/blobs).""" - d = cache / 'pardes' / 'blobs' - return {p.name: p.stat().st_size for p in d.iterdir() if p.is_file()} if d.exists() else {} + """The blob cache's size in bytes, 0 with none.""" + f = cache_file(cache) + return f.stat().st_size if f.exists() else 0 def far(binary): @@ -832,9 +837,10 @@ def far(binary): blob and keeps them; a second on the same machine, and one put off the session's state that joins again, are sent none they hold, from disk and from the core they leave; one started after the first ended gets - what it made from disk; one after the cache was corrupted and cut short - by hand, and one with the cache full, are sent what they lack. Every - client's digest is the session's once quiet.""" + what it made from disk; one after the cache was cut short by hand (a + file that is no database: started afresh), and one with the cache full, + are sent what they lack. Every client's digest is the session's once + quiet.""" scratch = Path(__file__).resolve().parent.parent.parent / '.scratch' / 'daemon' / 'dmn' scratch.mkdir(parents=True, exist_ok=True) root = Path(tempfile.mkdtemp(prefix='rf-', dir=scratch)) @@ -895,7 +901,8 @@ def far(binary): # A: the first, its cache empty: every blob sent, and kept. ta, a = client('a') typed('a', ta, b'from-A') - wait_for('client A\'s blobs on disk', lambda: len(cached(cache)) >= 2) + # The notes and the big file, a megabyte and more between them. + wait_for('client A\'s blobs on disk', lambda: cached(cache) > 1 << 20) time.sleep(.5) # B: a second client on this machine, and put off the session's # state once quiet: its first join from disk, its rejoin from the @@ -921,37 +928,28 @@ def far(binary): detach('c', tc, c) c_join = joins(root / 'c.stats')[0] assert c_join['from_disk'] >= c_join['blobs'] - 1 and c_join['sent_kb'] < 64, c_join - # D: the cache corrupted and cut short by hand: a miss each, sent again. - files = sorted(cached(cache).items(), key=lambda f: -f[1]) - for k, (name, size) in enumerate(files): - p = cache / 'pardes' / 'blobs' / name - if k % 2: - p.write_bytes(p.read_bytes()[:size // 2]) - else: - data = bytearray(p.read_bytes()) - data[size // 2] ^= 0x55 - p.write_bytes(bytes(data)) + # D: the cache cut short by hand: no database, started afresh, every + # blob sent again. + whole = cache_file(cache).read_bytes() + cache_file(cache).write_bytes(whole[:len(whole) // 2]) td, d = client('d') typed('d', td, b'from-D') detach('d', td, d) d_join = joins(root / 'd.stats')[0] assert d_join['from_disk'] == 0 and d_join['sent_kb'] > 100, d_join - # E: the cache full (a 1 MiB cap, held by older files): they go first, - # and the cache stays under its cap. - for k in range(2): - p = cache / 'pardes' / 'blobs' / f'junk-{k}' - p.write_bytes(b'j' * (512 << 10)) - os.utime(p, (1000 + k, 1000 + k)) + # E: the cache full (D's, past a 1 MiB cap): the oldest go at its end, + # the file compacted, under its cap but for xitdb's own few KB. + before_e = cached(cache) + assert before_e > 1 << 20, before_e te, e = client('e', {'PARDES_BLOB_CACHE_MB': '1'}) typed('e', te, b'from-E') detach('e', te, e) - left = cached(cache) - assert not [n for n in left if n.startswith('junk')] and sum(left.values()) <= 1 << 20, left + assert cached(cache) <= (1 << 20) + (64 << 10), cached(cache) tf, f = client('f', {'PARDES_BLOB_CACHE_MB': '1'}) typed('f', tf, b'from-F') detach('f', tf, f) f_join = joins(root / 'f.stats')[0] - assert f_join['from_disk'] > 0 and sum(cached(cache).values()) <= 1 << 20, (f_join, cached(cache)) + assert f_join['from_disk'] > 0 and cached(cache) <= (1 << 20) + (64 << 10), (f_join, cached(cache)) far_joins(root / 'pardes-replica.sock', 'files', binary, 200) # Through ssh (a stand-in forwarding as `ssh -L` does): a client # refused and one detached each end their ssh and its socket (one -- cgit v1.3