diff options
| author | Gabriel Schneider <[email protected]> | 2026-10-30 05:58:13 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-10-30 10:44:19 -0300 |
| commit | ddb2554fff073eed0b73a43825b2c007c895bc4d (patch) | |
| tree | 77db8d82afd23dc6e0395bc4ba21525bc6c8dfcd /test | |
| parent | 541ebfa62ed966d74e1b7e93eb0d62e536d50e7d (diff) | |
| download | pardes-ddb2554fff073eed0b73a43825b2c007c895bc4d.tar.gz pardes-ddb2554fff073eed0b73a43825b2c007c895bc4d.zip | |
replica: a far client's content cache and zstd: the session lists its join's blobs by BLAKE3 hash first (wire 16 `blobs`), the client finds each in the core it leaves or on disk ($XDG_CACHE_HOME/pardes/blobs, hashed again on load, a bad file a miss, the least recently used evicted past 1 GiB) and asks for the rest (`want`), and everything after the welcome goes in zstd envelopes (level 6 over 8 MiB for the join, an undo snapshot being a near copy of its text, then level 3), so `ssh -C` leaves the default command; a far client (through ssh, or PARDES_REPLICA_FAR) keeps what it was sent and, at its end, what it made; a hangup ends a remote client the normal way; through the box a client holding the blobs is drawn in 2.4 s where main took 7.0 to 11.9, and a first join of a session with undo in 7.1 s where main took 11.9; libzstd 1.5.7 vendored, its allocations on c_heap, a second copy refused by shared-libs-check; tests for the cache's verify-on-load, its eviction and a far join's blobs sent once; test/daemon.py's far clients
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Diffstat (limited to 'test')
| -rw-r--r-- | test/daemon.py | 190 |
1 files changed, 187 insertions, 3 deletions
diff --git a/test/daemon.py b/test/daemon.py index 8b0a20bd..fb4f3013 100644 --- a/test/daemon.py +++ b/test/daemon.py @@ -23,6 +23,8 @@ private HOME and runtime directory: `ssh -L` forwards, say), timed python3 -B test/daemon.py --clients <pardes> the replica clients alone + python3 -B test/daemon.py --far <pardes> + the far clients and their blob cache alone """ import fcntl import os @@ -65,6 +67,7 @@ class Term: self.out = bytearray() self.lock = threading.Lock() self.closing = False + self.tail = b'' self.reader = threading.Thread(target=self.drain, daemon=True) self.reader.start() @@ -83,7 +86,9 @@ class Term: return with self.lock: self.out += got - if b'\x1b[5n' in got: + # A request split across two reads is answered too. + seen, self.tail = self.tail + got, got[-3:] + if b'\x1b[5n' in seen: try: os.write(self.master, b'\x1b[0n') except OSError: @@ -183,8 +188,9 @@ def semantics(binary): class Replica: """A replica's connection (src/detached/wire.zig): length-prefixed - messages, a tag byte and a little-endian u32 length.""" - WIRE = 15 + messages, a tag byte and a little-endian u32 length. It is a near + client: its first word after the hello says so.""" + WIRE = 16 def __init__(self, path, timeout=10): self.s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -212,6 +218,7 @@ class Replica: is read, whole.""" name = session.encode() self.send(0x01, struct.pack('<HQH', self.WIRE, build, len(name)) + name) + self.send(0x06, b'\x00') tag, body = self.read() if tag == 0x01: pid, n = struct.unpack_from('<IH', body) @@ -445,6 +452,180 @@ def clients(binary): shutil.rmtree(root, ignore_errors=True) +def joins(path): + """A replica client's joins (PARDES_REPLICA_STATS, a line a link): how + many blobs each listed, how many it held already from the core it left + and from disk, and the KB it was sent.""" + out = [] + for line in checks(path): + words = line.split() + if 'blobs' not in words: + continue + got = {words[i]: int(words[i + 1]) for i in range(0, len(words) - 1, 2) if words[i + 1].isdigit()} + out.append(got) + return out + + +def cached(cache): + """The blob cache's files (a far client's, XDG_CACHE_HOME/pardes/blobs).""" + d = cache / 'pardes' / 'blobs' + return {p.name: p.stat().st_size for p in d.iterdir() if p.is_file()} if d.exists() else {} + + +def far(binary): + """Far replica clients (PARDES_REPLICA_FAR, as through ssh) of a + detached session and their blob cache: the first one is sent every + blob and keeps them; a second on the same machine, and one put off the + session's state that joins again, are sent none they hold, from disk + and from the core they leave; one started after the first ended gets + what it made from disk; one after the cache was corrupted and cut short + by hand, and one with the cache full, are sent what they lack. Every + client's digest is the session's once quiet.""" + scratch = Path(__file__).resolve().parent.parent.parent / '.scratch' / 'daemon' / 'dmn' + scratch.mkdir(parents=True, exist_ok=True) + root = Path(tempfile.mkdtemp(prefix='rf-', dir=scratch)) + cache = root / 'cache' + env = {k: v for k, v in os.environ.items() if not k.startswith('PARDES_') and 'PAGER' not in k} + env.update(HOME=str(root), XDG_RUNTIME_DIR=str(root), XDG_CONFIG_HOME=str(root / 'config'), + XDG_CACHE_HOME=str(cache), TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', + PARDES_NOTIME='1', PARDES_REPLICA_FAR='1') + (root / 'config' / 'pardes').mkdir(parents=True) + (root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n') + notes = root / 'notes.txt' + notes.write_text(''.join(f'line {i} of the notes\n' for i in range(300))) + big = root / 'big.txt' + big.write_text(''.join(f'{i:06d} the quick brown fox jumps over the lazy dog {i * 7919 % 10007}\n' for i in range(20000))) + started = [] + terms = [] + + def client(name, extra=None): + t = Term(24, 90) + terms.append(t) + c = t.spawn([binary, f'--attach={root / "pardes-replica.sock"}:files'], + dict(env, PARDES_REPLICA_CHECK=str(root / f'{name}.check'), + PARDES_REPLICA_STATS=str(root / f'{name}.stats'), **(extra or {}))) + started.append(c.pid) + wait_for(f'client {name}\'s first frame', lambda: b'line 3 of the notes' in t.text(), timeout=30) + # Its frame's request answered before a key goes: an answer behind + # an Esc, in one read, is taken as Alt with it. + time.sleep(.3) + return t, c + + def texts(): + return b''.join(nine.read(f'/pane/{row.split()[0].decode()}/body') for row in nine.read('/index').splitlines() + if row.split()[1] == b'text') + + def typed(name, t, words): + seen = len(checks(root / f'{name}.check')) + os.write(t.master, b'O' + words) + wait_for(f'client {name}\'s keys in the session\'s text', lambda: words in texts()) + time.sleep(.2) + os.write(t.master, b'\x1b') + wait_for(f'client {name} out of insert mode', lambda: normal_again(t)) + wait_for(f'client {name} agrees once quiet', lambda: agreed(root / f'{name}.check', seen)) + + def detach(name, t, c): + os.write(t.master, b' sD') + wait_for(f'client {name} let go', lambda: c.poll() is not None, timeout=20) + + try: + sess = subprocess.Popen([binary, '--detach=files', str(notes)], env=env, cwd=root, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) + started.append(sess.pid) + wait_for('the replica socket', (root / 'pardes-replica.sock').exists) + wait_for('the session\'s 9P socket', (root / 'pardes-9p-files.sock').exists) + nine = Client(root / 'pardes-9p-files.sock') + pane = next(int(row.split()[0]) for row in nine.read('/index').splitlines() if row.split()[1] == b'text') + nine.write(f'/pane/{pane}/exec', f'Look {big}\n'.encode()) + nine.write(f'/pane/{pane}/exec', b'Edit 3a/ an edit the session made /\n') + # A: the first, its cache empty: every blob sent, and kept. + ta, a = client('a') + typed('a', ta, b'from-A') + wait_for('client A\'s blobs on disk', lambda: len(cached(cache)) >= 2) + time.sleep(.5) + # B: a second client on this machine, and put off the session's + # state once quiet: its first join from disk, its rejoin from the + # core it left. + tb, b = client('b', {'PARDES_TEST_DIVERGE': '1'}) + os.write(tb.master, b'jj') + time.sleep(.5) + os.write(ta.master, b'jj') + wait_for('client B diverged and joined again', lambda: b'joined again' in tb.text(), timeout=30) + typed('b', tb, b'from-B') + typed('a', ta, b'again-A') + detach('b', tb, b) + detach('a', ta, a) + a_joins, b_joins = joins(root / 'a.stats'), joins(root / 'b.stats') + assert len(a_joins) == 1 and len(b_joins) == 2, (a_joins, b_joins) + assert a_joins[0]['from_disk'] == 0 and a_joins[0]['from_memory'] == 0 and a_joins[0]['sent_kb'] > 100, a_joins + assert b_joins[0]['from_disk'] >= b_joins[0]['blobs'] - 2 and b_joins[0]['sent_kb'] < a_joins[0]['sent_kb'], b_joins + assert b_joins[1]['from_memory'] >= b_joins[1]['blobs'] - 1 and b_joins[1]['sent_kb'] < 64, b_joins + # C: a client started again after them: what they made since they + # joined is on disk too (Have.keep at their end). + tc, c = client('c') + typed('c', tc, b'from-C') + detach('c', tc, c) + c_join = joins(root / 'c.stats')[0] + assert c_join['from_disk'] >= c_join['blobs'] - 1 and c_join['sent_kb'] < 64, c_join + # D: the cache corrupted and cut short by hand: a miss each, sent again. + files = sorted(cached(cache).items(), key=lambda f: -f[1]) + for k, (name, size) in enumerate(files): + p = cache / 'pardes' / 'blobs' / name + if k % 2: + p.write_bytes(p.read_bytes()[:size // 2]) + else: + data = bytearray(p.read_bytes()) + data[size // 2] ^= 0x55 + p.write_bytes(bytes(data)) + td, d = client('d') + typed('d', td, b'from-D') + detach('d', td, d) + d_join = joins(root / 'd.stats')[0] + assert d_join['from_disk'] == 0 and d_join['sent_kb'] > 100, d_join + # E: the cache full (a 1 MiB cap, held by older files): they go first, + # and the cache stays under its cap. + for k in range(2): + p = cache / 'pardes' / 'blobs' / f'junk-{k}' + p.write_bytes(b'j' * (512 << 10)) + os.utime(p, (1000 + k, 1000 + k)) + te, e = client('e', {'PARDES_BLOB_CACHE_MB': '1'}) + typed('e', te, b'from-E') + detach('e', te, e) + left = cached(cache) + assert not [n for n in left if n.startswith('junk')] and sum(left.values()) <= 1 << 20, left + tf, f = client('f', {'PARDES_BLOB_CACHE_MB': '1'}) + typed('f', tf, b'from-F') + detach('f', tf, f) + f_join = joins(root / 'f.stats')[0] + assert f_join['from_disk'] > 0 and sum(cached(cache).values()) <= 1 << 20, (f_join, cached(cache)) + body = texts() + for words in (b'from-A', b'from-B', b'again-A', b'from-C', b'from-D', b'from-E', b'from-F'): + assert words in body, words + print('daemon: far clients: the first sent {} KB; a second from disk ({} of {} blobs), its rejoin from memory ' + '({} of {}); one started again from disk ({} of {}); after the cache was spoilt by hand sent {} KB; with ' + 'the cache full it stayed under its cap; every digest agreed ({} checks)'.format( + a_joins[0]['sent_kb'], b_joins[0]['from_disk'], b_joins[0]['blobs'], b_joins[1]['from_memory'], + b_joins[1]['blobs'], c_join['from_disk'], c_join['blobs'], d_join['sent_kb'], + sum(len(checks(root / f'{n}.check')) for n in 'abcdef'))) + try: + nine.write(f'/pane/{pane}/exec', b'Exit\n') + except OSError as err: + assert 'Exit again' in str(err), err + nine.write(f'/pane/{pane}/exec', b'Exit\n') + sess.wait(timeout=10) + nine.socket.close() + wait_for('nothing left', lambda: not [p for p in processes(root) if alive(p)], timeout=10) + finally: + for t in terms: + t.close() + for pid in started + processes(root): + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + shutil.rmtree(root, ignore_errors=True) + + def terminals(client): return [int(row.split()[0]) for row in client.read('/index').splitlines() if row.split()[1] == b'term'] @@ -478,6 +659,7 @@ def main(binary, cycles): started.append(daemon) replicas(root, binary, session, daemon) clients(binary) + far(binary) shell = terminals(client)[0] shell_pid = int(run(client, shell, b'echo $$\n').split(b'\n')[1]) started.append(shell_pid) @@ -1078,6 +1260,8 @@ if __name__ == '__main__': upgrades(Path(sys.argv[2]).resolve(), Path(sys.argv[3]).resolve()) elif sys.argv[1:2] == ['--clients']: clients(sys.argv[2]) + elif sys.argv[1:2] == ['--far']: + far(sys.argv[2]) elif sys.argv[1:2] == ['--ping']: ping(sys.argv[2], sys.argv[3], sys.argv[4], int(sys.argv[5]) if len(sys.argv) > 5 else 50) elif '--gui' in sys.argv: |
