From 484baaf3d237c91a6ba5f7e2016c988e8aee00d3 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 14:44:53 -0300 Subject: test/soak.py: a soak of replica clients, what a person does in them through each link, measured: a session on a directory and two shells (here, or on the FreeBSD box over ssh), terminal clients in a pty kept by pyte and offscreen window clients through their test feed, each a seeded monkey of moves, edits, undo, saves, Looks, pickers, shells with programs, panes and columns, clicks, the wheel, the pointer resting, pastes and idle gaps, beside resizes, detaches and attaches, an Upgrade and a script's 9P edits; its table a client (rejoins, checks that differ, exits and cores, freezes at quiet points, a terminal client's screen and cursor against the session's render of its view, visible rollbacks, keystroke to output or frame p50/p99) and the session's worst boundary, exit 1 on any breach; test/delay_relay.py takes --jitter (order kept) Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 1318 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1318 insertions(+) create mode 100644 test/soak.py (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py new file mode 100644 index 00000000..f75371ef --- /dev/null +++ b/test/soak.py @@ -0,0 +1,1318 @@ +#!/usr/bin/env python3 +"""A soak of replica clients: what a person does in them, through each link, +measured (docs/next-daemon.md, the replica and GUI replica notes). + + python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10] + [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] + python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] + [--pardes P] [--gui G] [--seed 1] [--trace] + +`local`: a session on this machine (`pardes --detach=soak `, the +shape of the user's: a directory pane, then two shells), and at each RTT its +clients through test/delay_relay.py (with jitter), far as through ssh +(PARDES_REPLICA_FAR=1: the blobs by hash, the stream in zstd). One cell an +RTT, `--minutes` each. `box`: the session on the FreeBSD box under +/root/pardes-test (`--remote-bin`, by default bin/pardes-hunt there, run in +run-hunt and home-hunt), the clients here through ssh, as the user's window +was; its 9P through an `ssh -L` of the session's socket. + +The clients: terminals (`pardes --attach`, in a pty here whose screen pyte +keeps, frame requests answered) and windows (`pardes-gui --attach`, SDL's +offscreen driver, keys and the mouse through its test feed on a pty, so it +can be resized). Each runs a seeded monkey of a person's actions: moves of +every kind (keys, clicks, jumps, search), scrolls (keys, the wheel), edits, +undo and save, Look at files and directories, the Panes picker, Find, new +terminals with programs (ls, a long output, less, vi, top, Ctrl-C), panes and +columns opened, moved and closed, the pointer resting and moving, pastes, +idle gaps. Between them, at random: a client resized, a client detached and +another attached, an Upgrade, a 9P script's edit. `--seed` makes a run +again (the timing differs, the actions do not). + +Measured (a row a client, then the session's): + - rejoins: links past the first (PARDES_REPLICA_STATS writes one a link) + and `joined again` said by it; checks (PARDES_REPLICA_CHECK) that differ; + - exits nobody asked for, a signal or a non-zero code, the session's death, + a panic or `crashes` line; + - freezes: a quiet point where a client gives no fresh check within 8 s, + and a terminal client's key with no output within 2 s while it was not + held; the session's worst boundary (its stats at its end); + - the view: at each quiet point, a terminal client's cursor and screen + (pyte) against what the session renders for its view (/screen read right + after its own input made its view the live one), and the panes on it; + - blink: visible rollbacks (`visible`), mispredicted inputs, a terminal + client's full clears, a minute; + - keystroke to output (terminal) and to a frame (window, PARDES_TEST_LATENCY) + p50 / p99. + +Prints a table, then each breach, and exits 1 if there is any. Everything +started is ended by pid (and found by its environment); the run's files are +kept under $TMPDIR/soak-* (`--keep`) or removed. +""" +import argparse +import fcntl +import json +import os +import random +import re +import shutil +import signal +import socket +import statistics +import struct +import subprocess +import sys +import tempfile +import termios +import threading +import time +import tty as ttymod +from pathlib import Path + +import pyte + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from ninep import Client # noqa: E402 +from daemon import processes, alive, wait_for, checks # noqa: E402 + +BOX = 'root@65.109.81.103' +SSH = ['/usr/bin/ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', + '-o', 'UserKnownHostsFile=/home/goblin/.ssh/known_hosts', '-i', '/home/goblin/.ssh/id_ed25519'] +BOX_T = '/root/pardes-test' +NAME = 'soak' +ESC = b'\x1b' + + +def now(): + return time.monotonic() + + +def pct(xs, p): + if not xs: + return None + xs = sorted(xs) + return xs[min(len(xs) - 1, len(xs) * p // 100)] + + +# ---- the fixture: a directory such as a person works in ---- + +WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa ' + 'quebec romeo sierra tango uniform victor whiskey xray yankee zulu').split() + + +def fixture(at, rng): + at.mkdir(parents=True, exist_ok=True) + (at / 'notes.txt').write_text(''.join(f'line {i} of the notes: {rng.choice(WORDS)} {rng.choice(WORDS)} see code.zig:{i % 90 + 1}\n' + for i in range(300))) + (at / 'code.zig').write_text(''.join(f'pub fn f{i}(x: u32) u32 {{\n return x * {i} + {rng.randrange(1000)}; // {rng.choice(WORDS)}\n}}\n\n' + for i in range(120))) + (at / 'big.txt').write_text(''.join(f'{i:06d} {" ".join(rng.choice(WORDS) for _ in range(9))}\n' for i in range(8000))) + (at / 'README').write_text('A directory to soak in.\nnotes.txt code.zig big.txt sub/\nsub/inner.txt:3\n') + sub = at / 'sub' + sub.mkdir(exist_ok=True) + (sub / 'inner.txt').write_text(''.join(f'inner {i} {rng.choice(WORDS)}\n' for i in range(60))) + (sub / 'more.md').write_text('# More\n\n' + ''.join(f'- item {i} {rng.choice(WORDS)}\n' for i in range(40))) + + +# ---- terminals ---- + +class Term: + """A terminal: a pty whose output is kept, timed and fed to pyte, its + frame requests (CSI 5 n) answered as a terminal emulator does. An Esc is + written alone and nothing follows it for 80 ms, an answer included: the + client's parser takes an Esc and what follows within 50 ms as Alt.""" + + def __init__(self, rows, cols): + self.master, self.slave = os.openpty() + self.rows, self.cols = rows, cols + fcntl.ioctl(self.slave, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.lock = threading.Lock() + self.wlock = threading.Lock() + self.screen = pyte.Screen(cols, rows) + self.stream = pyte.ByteStream(self.screen) + self.out_bytes = 0 + self.last_out = 0.0 + self.chunks = [] # (time, bytes) of the last minute or so, for latencies + self.clears = 0 + self.joined_again = 0 + self.tail = b'' + self.esc_until = 0.0 + self.closing = False + self.log = None + self.reader = threading.Thread(target=self.drain, daemon=True) + self.reader.start() + + def drain(self): + import select + while not self.closing: + try: + if not select.select([self.master], [], [], 0.05)[0]: + continue + got = os.read(self.master, 65536) + except OSError: + return + if not got: + return + t = now() + carry, self.tail = self.tail, (self.tail + got)[-11:] + seen = carry[-3:] + got + with self.lock: + self.out_bytes += len(got) + self.last_out = t + self.chunks.append((t, len(got))) + if len(self.chunks) > 4096: + del self.chunks[:2048] + # The carry is shorter than either word: a match holds a new byte. + self.clears += seen.count(b'\x1b[2J') + self.joined_again += (carry + got).count(b'joined again') + self.feed(got) + if self.log: + self.log.write(got) + if b'\x1b[5n' in seen: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + with self.wlock: + try: + os.write(self.master, b'\x1b[0n') + except OSError: + return + + SGR = re.compile(rb'\x1b\[[0-9:;]*m') + + def feed(self, got): + """To pyte, without its colours (pyte reads no colon sub-parameters) + and holding back a sequence a read cut in two.""" + data = getattr(self, 'held_back', b'') + got + cut = data.rfind(b'\x1b') + self.held_back = b'' + if cut >= 0 and not re.match(rb'\x1b(\[[0-9:;?<>=! ]*[@-~]|[^\[\]]|\][^\x07\x1b]*(\x07|\x1b\\))', data[cut:]): + data, self.held_back = data[:cut], data[cut:] + try: + self.stream.feed(self.SGR.sub(b'', data)) + except Exception: + pass + + def write(self, data): + """Keys as typed: each Esc alone, 80 ms clear after it.""" + parts = re.split(b'(\x1b(?![\\[O\\]n0-9a-z]))', data) + for part in parts: + if not part: + continue + if part == ESC: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, ESC) + self.esc_until = now() + 0.08 + time.sleep(0.08) + else: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, part) + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + with self.lock: + self.screen.resize(rows, cols) + + def view(self): + with self.lock: + return [line.rstrip() for line in self.screen.display], (self.screen.cursor.y, self.screen.cursor.x), self.screen.cursor.hidden + + def spawn(self, argv, env, cwd, err): + slave = self.slave + return subprocess.Popen(argv, env=env, cwd=cwd, stdin=slave, stdout=slave, stderr=err, start_new_session=True, + preexec_fn=lambda: fcntl.ioctl(0, termios.TIOCSCTTY, 0)) + + def close(self): + self.closing = True + self.reader.join(timeout=2) + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- clients ---- + +class Base: + def __init__(self, run, kind, label, link): + self.run = run + self.kind = kind + self.label = label + self.link = link # where it dials: a socket path, or the box + self.dir = run.root / label + self.dir.mkdir() + self.check = self.dir / 'check' + self.stats = self.dir / 'stats' + self.err = self.dir / 'err' + self.p = None + self.started = None + self.asked_to_end = False + self.gone = None # how it ended, if it did + self.inputs = 0 + self.lat = [] # ms, input to output/frame + self.no_output = 0 + self.held = threading.Event() # the monkey waits while set + self.paused = threading.Event() # set by the monkey once it waits + self.lock = threading.Lock() + + def env(self): + e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) + if self.run.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + if self.run.far: + e['PARDES_REPLICA_FAR'] = '1' + return e + + def attach_arg(self): + return f'--attach={self.link}:{NAME}' + + def links(self): + return [line for line in checks(self.stats) if line.startswith('inputs ')] + + def stats_sum(self): + keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible') + tot = dict.fromkeys(keys, 0) + for line in self.links(): + w = line.split() + d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)} + for k in keys: + try: + tot[k] += int(d.get(k, 0)) + except ValueError: + pass + return tot + + def check_lines(self): + return checks(self.check) + + def poll(self): + if self.gone is None and self.p is not None and self.p.poll() is not None: + self.gone = self.p.returncode + return self.gone + + +class TtyClient(Base): + def __init__(self, run, label, link, rows=30, cols=100): + super().__init__(run, 'tty', label, link) + self.term = Term(rows, cols) + self.term.log = open(self.dir / 'out', 'wb', buffering=0) if run.args.keep else None + + def start(self): + argv = [self.run.pardes, self.attach_arg()] + # Its stderr a file: a panic's words and the replica's trace. + self.p = self.term.spawn(argv, self.env(), self.run.root, open(self.err, 'w')) + self.started = now() + + def ready(self): + return self.term.out_bytes > 2000 and now() - self.term.last_out > 0.3 + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + before = self.term.out_bytes + t = now() + self.term.write(data) + threading.Thread(target=self._latency, args=(t, before), daemon=True).start() + + def _latency(self, t, before): + deadline = t + 2.0 + while now() < deadline: + with self.term.lock: + got = [c for c in self.term.chunks if c[0] >= t] + if got: + with self.lock: + self.lat.append((got[0][0] - t) * 1000) + return + time.sleep(0.005) + if not self.held.is_set(): + with self.lock: + self.no_output += 1 + + def mouse(self, kind, button, x, y): + # SGR: cells from 1; x, y here are fractions of the screen. + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + b = {'down': button - 1, 'up': button - 1, 'motion': 35, 'wheel': 64 if button > 0 else 65}[kind] + end = 'm' if kind == 'up' else 'M' + self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) + + def resize(self, rows, cols): + self.term.resize(rows, cols) + if self.p: + try: + os.killpg(self.p.pid, signal.SIGWINCH) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.term.joined_again + + def hangup(self): + """As its terminal closing: the normal way out (its stats written).""" + try: + os.killpg(self.p.pid, signal.SIGHUP) + except OSError: + pass + + def close(self): + self.term.close() + + +class WinClient(Base): + """A window client in test mode: offscreen, its keys and pointer through + its test feed on a pty (Escape as CSI 27 u: a lone ESC waits for more).""" + + def __init__(self, run, label, link, rows=34, cols=110): + super().__init__(run, 'win', label, link) + self.cap = self.dir / 'cap' + self.cap.mkdir() + self.latency = self.dir / 'latency' + self.master, self.slave = os.openpty() + ttymod.setraw(self.slave) + self.rows, self.cols = rows, cols + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.wlock = threading.Lock() + self.px = (cols * 16, rows * 33) # test mode's 27 px font, about + + def start(self): + e = dict(self.env(), PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(self.cap), PARDES_TEST_LATENCY=str(self.latency), + SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(self.cols), PARDES_TEST_ROWS=str(self.rows)) + self.p = subprocess.Popen([self.run.gui, self.attach_arg()], env=e, cwd=self.run.root, stdin=self.slave, + stdout=subprocess.DEVNULL, stderr=open(self.err, 'w'), start_new_session=True) + self.started = now() + + def ready(self): + # Software presenting (offscreen) captures no ppm: its first frame is + # its latency trace's first F. + try: + return b'F ' in self.latency.read_bytes() and self.p.poll() is None + except FileNotFoundError: + return False + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + with self.wlock: + try: + os.write(self.master, data.replace(ESC + b'[', b'\x00CSI').replace(ESC, b'\x1b[27u').replace(b'\x00CSI', ESC + b'[')) + except OSError: + pass + + def mouse(self, kind, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + if kind == 'wheel': + seq = f'\x1b]777;mouse;wheel;{button};{px};{py}\x07' + else: + seq = f'\x1b]777;mouse;{kind};{button};{px};{py}\x07' + self.inputs += 1 + self.run.note(f'{self.label} < mouse {kind} {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, seq.encode()) + except OSError: + pass + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + + def hangup(self): + """Its test feed's end: the normal way out (its stats written).""" + try: + os.close(self.master) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.said().count('joined again') + + def frame_size(self): + try: + with open(self.cap / 'latest.ppm', 'rb') as f: + head = f.read(32).split() + self.px = (int(head[1]), int(head[2])) + except (OSError, IndexError, ValueError): + pass + + def latencies(self): + """Each stdin read to the end of the first frame after it, ms; and the + reads no frame followed within 2 s (a frame came after a later one).""" + try: + lines = self.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads, frames = [], [] + for line in lines: + w = line.split() + if len(w) >= 2 and w[0] == 'R': + reads.append(int(w[1])) + elif len(w) >= 3 and w[0] == 'F': + frames.append(int(w[2])) + out, missed, j = [], 0, 0 + for r in reads: + while j < len(frames) and frames[j] < r: + j += 1 + if j < len(frames): + d = (frames[j] - r) / 1e6 + if d <= 2000: + out.append(d) + else: + missed += 1 + return out, missed + + def close(self): + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- what a person does ---- + +def words(rng, n): + return ' '.join(rng.choice(WORDS) for _ in range(n)).encode() + + +def burst(c, rng): + """One action of a person's, in whatever state the client is.""" + k = c.keys + acts = c.run.actions + choice = rng.choices(acts, weights=[a[1] for a in acts])[0] + choice[2](c, rng, k) + return choice[0] + + +def act_move(c, rng, k): + for _ in range(rng.randint(1, 6)): + k(rng.choice([b'j', b'k', b'h', b'l', b'w', b'b', b'e', b'5j', b'3k', b'gg', b'ge', b'\x04', b'\x15', + b'\x06', b'\x02', b'}', b'{', b'%', b'x', b'12G', b'gl', b'gh'])) + time.sleep(rng.uniform(0.02, 0.2)) + + +def act_search(c, rng, k): + k(b'/' + rng.choice(WORDS).encode() + b'\r') + for _ in range(rng.randint(0, 3)): + time.sleep(rng.uniform(0.1, 0.4)) + k(rng.choice([b'n', b'N'])) + + +def act_jump(c, rng, k): + k(rng.choice([b'\x0f', b' jo', b' jj', b'gd', b'\x0f\x0f'])) + + +def act_insert(c, rng, k): + k(rng.choice([b'i', b'a', b'o', b'O', b'A'])) + for w in words(rng, rng.randint(1, 5)).split(): + k(w + b' ') + time.sleep(rng.uniform(0.03, 0.15)) + k(ESC) + + +def act_delete(c, rng, k): + k(rng.choice([b'xd', b'd', b'wd', b'xxd', b'c' + words(rng, 1) + ESC])) + + +def act_undo(c, rng, k): + k(rng.choice([b'u', b'uu', b'U', b'u'])) + + +def act_save(c, rng, k): + k(b' fs') + + +def act_look(c, rng, k): + # Enter is Look: a directory's row opens it, a word finds its next place. + k(rng.choice([b'\r', b'j\r', b'jj\r', b'gg\r'])) + + +def act_panes(c, rng, k): + k(b' bb') + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'\r', ESC, b'jj\r', b'not\r'])) + + +def act_find(c, rng, k): + k(rng.choice([b' ff', b' fr', b' fg'])) + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'notes\r', ESC, b'j\r', b'inner\r'])) + + +def act_shell(c, rng, k): + # A terminal of its own, a program in it, and back out to normal mode. + k(b' nt') + time.sleep(rng.uniform(0.8, 1.5)) + for _ in range(rng.randint(1, 3)): + prog = rng.choice(['ls -la\r', 'seq 1 3000\r', 'less big.txt\r', 'vi notes.txt\r', 'top\r', 'sleep 30\r', + 'echo ' + rng.choice(WORDS) + '\r', 'cat code.zig\r']) + k(prog.encode()) + time.sleep(rng.uniform(0.5, 2.0)) + if prog.startswith('less'): + # Never SPC then q: that is Exit, should the keys reach no less. + k(b'jjjj\x06') + time.sleep(0.3) + k(b'q') + elif prog.startswith('vi'): + k(b'jjjjx') + k(ESC) + k(b':q!\r') + elif prog.startswith(('top', 'sleep', 'seq')): + k(b'\x03' if not prog.startswith('top') else b'q') + time.sleep(rng.uniform(0.2, 0.8)) + k(b'\x02') # Ctrl-b: normal mode + if rng.random() < 0.5 and c.run.pane_count() >= 4: + time.sleep(0.3) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b''])) + + +def act_window(c, rng, k): + k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l'])) + + +def act_column(c, rng, k): + few = c.run.pane_count() < 5 + k(rng.choice([b' cn', b' cn'] if few else [b' cn', b' cn', b' cd', b' cj'])) + + +def act_del(c, rng, k): + if c.run.pane_count() < 4: + return act_move(c, rng, k) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b'', b' d'])) + + +def act_new(c, rng, k): + k(b' fn') + time.sleep(0.4) + k(b'i' + words(rng, 4)) + k(ESC) + + +def act_esc(c, rng, k): + k(rng.choice([ESC, b'\x02', b':', b':'])) + time.sleep(0.3) + k(ESC) + + +def act_click(c, rng, k): + x, y = rng.random(), rng.uniform(0.05, 1.0) + b = rng.choice([1, 1, 1, 3]) + c.mouse('down', b, x, y) + time.sleep(rng.uniform(0.03, 0.12)) + if rng.random() < 0.3: # a drag + c.mouse('motion', b, min(1, x + rng.uniform(0, 0.2)), min(1, y + rng.uniform(0, 0.1))) + c.mouse('up', b, x, y) + + +def act_wheel(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + d = rng.choice([1, -1]) + for _ in range(rng.randint(1, 8)): + c.mouse('wheel', d, x, y) + time.sleep(rng.uniform(0.01, 0.08)) + + +def act_hover(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + for _ in range(rng.randint(2, 12)): + x = min(1, max(0, x + rng.uniform(-0.05, 0.05))) + y = min(1, max(0.05, y + rng.uniform(-0.03, 0.03))) + c.mouse('motion', 0, x, y) + time.sleep(rng.uniform(0.01, 0.06)) + time.sleep(rng.uniform(0.05, 0.6)) # resting: a Look preview + + +def act_paste(c, rng, k): + k(b'i\x1b[200~' + b'\n'.join(words(rng, 6) for _ in range(rng.randint(1, 4))) + b'\x1b[201~') + k(ESC) + + +def act_settings(c, rng, k): + k(rng.choice([b' tw', b' ?', b' hm', b' tt'])) + time.sleep(0.5) + k(ESC) + + +def act_idle(c, rng, k): + time.sleep(rng.choice([0.5, 1, 2, 4, 8])) + + +ACTIONS = [ + ('move', 30, act_move), ('search', 5, act_search), ('jump', 4, act_jump), ('insert', 12, act_insert), + ('delete', 4, act_delete), ('undo', 5, act_undo), ('save', 3, act_save), ('look', 6, act_look), + ('panes', 3, act_panes), ('find', 3, act_find), ('shell', 3, act_shell), ('window', 4, act_window), + ('column', 1, act_column), ('del', 2, act_del), ('new', 2, act_new), ('esc', 4, act_esc), + ('click', 8, act_click), ('wheel', 8, act_wheel), ('hover', 6, act_hover), ('paste', 2, act_paste), + ('settings', 1, act_settings), ('idle', 6, act_idle), +] + + +def monkey(c, rng, stop): + while not stop.is_set(): + if c.held.is_set(): + c.paused.set() + time.sleep(0.05) + continue + c.paused.clear() + if c.poll() is not None: + return + try: + what = burst(c, rng) + c.run.note(f'{c.label} {what}') + except OSError: + return + time.sleep(rng.uniform(0.05, 0.6)) + + +# ---- the run ---- + +class Run: + def __init__(self, args, cell): + self.args = args + self.cell = cell + self.root = Path(tempfile.mkdtemp(prefix=f'soak-{cell}-', dir=os.environ.get('TMPDIR') or '/tmp')) + self.pardes = str(Path(args.pardes).resolve()) + self.gui = str(Path(args.gui).resolve()) + self.far = True + self.pids = [] + self.clients = [] + self.relays = [] + self.events = [] + self.breaches = [] + self.notes = open(self.root / 'actions', 'w') + only = set(args.only.split(',')) if args.only else None + skip = set(args.skip.split(',')) if args.skip else set() + self.actions = [a for a in ACTIONS if (only is None or a[0] in only) and a[0] not in skip] + self.nlock = threading.Lock() + self.t0 = now() + env = {k: v for k, v in os.environ.items() if not k.startswith(('PARDES_', 'WAYLAND', 'DBUS_', 'DISPLAY', 'BROWSER')) + and 'PAGER' not in k and k not in ('winid', 'EDITOR', 'NINE_MOUNT')} + env.update(HOME=str(self.root), XDG_RUNTIME_DIR=str(self.root), XDG_CONFIG_HOME=str(self.root / 'config'), + XDG_CACHE_HOME=str(self.root / 'cache'), XDG_STATE_HOME=str(self.root / 'state'), + TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', PARDES_NOTIME='1') + (self.root / 'config' / 'pardes').mkdir(parents=True) + (self.root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n') + self.env = env + self.work = self.root / 'work' + self.nine = None + self.nine_lock = threading.RLock() + self.session_pid = None + + def note(self, what): + with self.nlock: + self.notes.write(f'{now() - self.t0:9.3f} {what}\n') + self.notes.flush() + + def breach(self, what): + self.note('BREACH ' + what) + self.breaches.append(what) + + def spawn(self, argv, **kw): + p = subprocess.Popen(argv, **kw) + self.pids.append(p.pid) + return p + + def relay(self, rtt): + if not rtt: + return self.root / 'pardes-replica.sock' + at = self.root / f'relay{rtt}.sock' + if not at.exists(): + self.relays.append(self.spawn([sys.executable, '-B', str(HERE / 'delay_relay.py'), str(at), str(self.root / 'pardes-replica.sock'), + '--delay', str(rtt / 2), '--jitter', str(self.args.jitter)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) + wait_for('the relay', at.exists) + return at + + def pane_count(self): + """The session's panes (a script's view of /index), cached a second: + a Del that closed the last one would end the session.""" + t = now() + if t - getattr(self, 'counted_at', 0) > 1: + rows = self.nine_read('/index') or b'' + self.counted, self.counted_at = len(rows.splitlines()), t + return self.counted + + def nine_read(self, path): + with self.nine_lock: + return self._nine_read(path) + + def nine_write(self, path, data): + with self.nine_lock: + return self._nine_write(path, data) + + def _nine_read(self, path): + for _ in range(3): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.read(path) + except (OSError, EOFError): + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + time.sleep(0.5) + return None + + def _nine_write(self, path, data): + for _ in range(2): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.write(path, data) + except EOFError: + self.nine = None + time.sleep(0.5) + except OSError as e: + return e + + # The session: here. + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) + if self.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / 'session.err', 'w'), + start_new_session=True) + self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' + wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20) + wait_for('the replica socket', (self.root / 'pardes-replica.sock').exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + self.pids.append(self.session_pid) + # Two shells beside the directory, as the user's had. + for _ in range(2): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(0.5) + + def session_alive(self): + return alive(self.session_pid) if self.session_pid else False + + def end_session(self): + """Exit (twice: unsaved text asks once), then --kill if it stays.""" + for _ in range(3): + self.nine_write('/ctl', b'Exit\n') + try: + wait_for('the session\'s end', lambda: not self.session_alive(), timeout=3) + return True + except TimeoutError: + pass + self.note('the session did not Exit: --kill') + self.kill_session() + return False + + def kill_session(self): + subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30) + + def session_said(self): + try: + return (self.root / 'session.err').read_text(errors='replace') + except FileNotFoundError: + return '' + + def crashes(self): + p = self.root / 'config' / 'pardes' / 'crashes' + return p.read_text() if p.exists() else '' + + def cleanup(self): + for c in self.clients: + if c.p and c.p.poll() is None: + c.p.kill() + c.close() + for pid in self.pids + processes(self.root): + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + + +class BoxRun(Run): + """The session on the FreeBSD box; the clients here through ssh.""" + + def __init__(self, args, cell): + super().__init__(args, cell) + self.far = False # an ssh client is far by itself + self.run_dir = f'{BOX_T}/run-hunt' + self.home = f'{BOX_T}/home-hunt' + self.env['PARDES_SSH'] = ' '.join(SSH) + self.env['PARDES_REMOTE_RUNTIME'] = self.run_dir + self.fwd = None + + def box(self, script, timeout=60): + return subprocess.run(SSH + [BOX, script], capture_output=True, timeout=timeout) + + def benv(self): + return (f'HOME={self.home} XDG_RUNTIME_DIR={self.run_dir} XDG_CONFIG_HOME={self.home}/config TMPDIR={BOX_T}/tmp-hunt ' + f'XDG_CACHE_HOME={self.home}/cache XDG_STATE_HOME={self.home}/state PATH={BOX_T}/bin:/usr/local/bin:/usr/bin:/bin ' + f'SHELL=/usr/local/bin/bash TERM=xterm-256color PARDES_NOTIME=1 PARDES_REPLICA_STATS={self.home}/session.stats' + + (' PARDES_REPLICA_TRACE=1' if self.args.trace else '')) + + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout + bin_ = self.args.remote_bin + script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' + f'rm -rf {self.home}; mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' + f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' + f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work {self.home}/session.err 2>&1 & ' + f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' + f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err; exit 4; }}') + got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) + if got.returncode != 0: + raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}') + self.nine_sock = self.root / 'box9p.sock' + self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + wait_for('the 9P forward', self.nine_sock.exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + for _ in range(2): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + + def session_alive(self): + got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) + return got.stdout.strip() == b'yes' + + def session_said(self): + got = self.box(f'cat {self.home}/session.err; ls {self.home}/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) + return got.stdout.decode(errors='replace') + + def crashes(self): + return '' + + def kill_session(self): + self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) + + def relay(self, rtt): + return BOX + + def cleanup(self): + super().cleanup() + # Everything started under run-hunt, by its environment: ps -e prints it. + self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' + f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) + + +def start_client(run, kind, rtt, n): + label = f'{kind}{rtt}-{n}' + link = run.relay(rtt) + c = (TtyClient if kind == 'tty' else WinClient)(run, label, link) + c.rtt = rtt + c.start() + run.pids.append(c.p.pid) + run.clients.append(c) + run.note(f'{label} started (pid {c.p.pid})') + wait_for(f'{label}\'s first frame', c.ready, timeout=60 if isinstance(run, BoxRun) else 30) + if isinstance(c, WinClient): + c.frame_size() + time.sleep(0.3) + return c + + +def hold(run, clients, timeout=30): + for c in clients: + c.held.set() + deadline = now() + timeout + for c in clients: + while not c.paused.is_set() and c.poll() is None and now() < deadline: + time.sleep(0.05) + + +def release(clients): + for c in clients: + c.held.clear() + + +def quiet_point(run, live): + """Every monkey held: each client must give a fresh check within 8 s + (a freeze otherwise), and none may differ; a terminal client's view + against the session's.""" + hold(run, live) + time.sleep(1.5) + for c in live: + if c.poll() is not None: + continue + before = len(c.check_lines()) + # A key that moves nothing a digest sees but the view's cursor, so the + # client's input is the session's last: its view is the live one. + c.keys(b'l') + t = now() + fresh = False + while now() - t < 8 + 2 * c.rtt / 1000: + if len(c.check_lines()) > before: + fresh = True + break + time.sleep(0.1) + if not fresh and c.poll() is None: + run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') + if isinstance(c, TtyClient) and fresh: + view_check(run, c) + release(live) + + +def view_check(run, c): + """The terminal client's screen and cursor against what the session renders + for the same view (its input was the last, so 9P reads that view).""" + time.sleep(0.4) + raw = run.nine_read('/screen') + if not raw: + return + try: + s = json.loads(raw) + except ValueError: + return + lines, cursor, hidden = c.term.view() + cols, rows = s.get('cols'), s.get('rows') + if (cols, rows) != (c.term.cols, c.term.rows): + c.view_skips = getattr(c, 'view_skips', 0) + 1 + return + theirs = [] + cells = s.get('cells', []) + for r in range(rows): + row = cells[r * cols:(r + 1) * cols] if len(cells) == rows * cols else (cells[r] if r < len(cells) else []) + theirs.append(''.join(cell[0] if isinstance(cell, list) else str(cell) for cell in row).rstrip()) + sc = s.get('cursor') + differ = [r for r in range(min(rows, len(lines))) if lines[r] != theirs[r]] + c.views = getattr(c, 'views', 0) + 1 + want = None + if isinstance(sc, dict): + want = (sc.get('row', sc.get('y')), sc.get('col', sc.get('x'))) + elif isinstance(sc, list) and len(sc) >= 2: + want = (sc[1], sc[0]) + bad_cursor = want is not None and None not in want and tuple(want) != cursor and not hidden + if differ or bad_cursor: + c.view_bad = getattr(c, 'view_bad', 0) + 1 + d = c.dir / f'view-{c.view_bad}.txt' + d.write_text(f'cursor client {cursor} hidden {hidden} session {sc}\nrows differing {differ[:12]}\n' + + ''.join(f'client {r:2} |{lines[r]}|\nsession {r:2} |{theirs[r]}|\n' for r in differ[:12])) + run.breach(f'{c.label}: its screen is not the session\'s render of its view ({len(differ)} rows' + f'{", cursor " + str(cursor) + " vs " + str(want) if bad_cursor else ""}): {d}') + + +def global_event(run, rng, live, made): + """Something happens beside the typing: a resize, a client detached and + another attached, an Upgrade, a script's edit over 9P.""" + kinds = [e for e in run.args.events.split(',') if e] + if not kinds: + return None + if run.args.upgrade and not getattr(run, 'upgraded', False): + kinds.append('upgrade') + what = rng.choice(kinds) + if what == 'resize': + c = rng.choice(live) + rows, cols = rng.randint(18, 50), rng.randint(60, 160) + run.note(f'{c.label} resized to {cols}x{rows}') + c.resize(rows, cols) + elif what == 'detach': + c = rng.choice(live) + run.note(f'{c.label} detached') + hold(run, [c]) + c.asked_to_end = True + c.keys(ESC) + c.keys(b'\x02') + time.sleep(0.2) + c.keys(ESC) + c.keys(b' sD') + try: + wait_for(f'{c.label} let go', lambda: c.poll() is not None, timeout=15) + except TimeoutError: + # Its keys went to a program or a prompt: it is ended as a window closes. + run.note(f'{c.label} did not detach by keys: hung up') + c.hangup() + wait_for(f'{c.label} hung up', lambda: c.poll() is not None, timeout=15) + live.remove(c) + made[0] += 1 + n = start_client(run, c.kind, c.rtt, made[0]) + live.append(n) + return n + elif what == 'upgrade': + run.note('Upgrade') + run.upgraded = True + run.nine_write('/ctl', b'Upgrade\n') + time.sleep(3) + if run.nine: + try: + run.nine.socket.close() + except Exception: + pass + run.nine = None + elif what == 'script': + rows = (run.nine_read('/index') or b'').splitlines() + texts = [r.split()[0].decode() for r in rows if len(r.split()) > 1 and r.split()[1] == b'text'] + if texts: + n = rng.choice(texts) + run.note(f'9P append to pane {n}') + run.nine_write(f'/pane/{n}/body', b'a line a script wrote\n') + return None + + +def soak(run, kinds, rtt, minutes, seed): + rng = random.Random(seed) + stop = threading.Event() + live = [] + threads = [] + made = [0] + try: + soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made) + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + stop.set() + report_cell(run, rtt) + run.cleanup() + + +def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): + run.start_session() + for kind in kinds: + made[0] += 1 + live.append(start_client(run, kind, rtt, made[0])) + for c in list(live): + t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + end = now() + minutes * 60 + next_quiet = now() + 45 + next_event = now() + rng.uniform(20, 60) + while now() < end: + time.sleep(0.5) + for c in list(live): + if c.poll() is not None: + run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') + live.remove(c) + if not run.session_alive(): + run.breach('the session died') + break + if not live: + break + if now() > next_event: + n = global_event(run, rng, live, made) + if n: + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + next_event = now() + rng.uniform(20, 70) + if now() > next_quiet: + quiet_point(run, live) + next_quiet = now() + rng.uniform(40, 80) + stop.set() + for t in threads: + t.join(timeout=20) + if run.session_alive() and live: + quiet_point(run, live) + # Each client's links written: a hangup ends a client the normal way. + for c in live: + c.asked_to_end = True + if c.p.poll() is None: + c.hangup() + for c in live: + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + run.breach(f'{c.label}: would not end on a hangup') + c.p.kill() + if run.session_alive(): + run.end_session() + + +def tail(c): + if isinstance(c, WinClient): + return c.said()[-300:].replace('\n', ' | ') + lines, _, _ = c.term.view() + return ' | '.join(line for line in lines if line.strip())[-300:] + + +ROWS = [] + + +def cores(run): + """systemd-coredump's cores since the cell began, of the pids it started.""" + if isinstance(run, BoxRun) or not shutil.which('coredumpctl'): + return [] + since = time.time() - (now() - run.t0) - 5 + got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True) + ours = set(run.pids) | {c.p.pid for c in run.clients if c.p} | ({run.session_pid} if run.session_pid else set()) + out = [] + for line in got.stdout.splitlines(): + w = line.split() + pid = next((int(x) for x in w if x.isdigit() and int(x) in ours), None) + if pid: + out.append(f'pid {pid} {" ".join(x for x in w if x.startswith("SIG"))} {w[-2] if len(w) > 1 else ""}') + return out + + +def report_cell(run, rtt): + for core in cores(run): + run.breach(f'core dumped: {core} (coredumpctl debug )') + said = run.session_said() + for word in ('panic', 'Segmentation', 'abort', 'reached unreachable', 'core dumped'): + if word in said: + run.breach(f'session said "{word}": {said[said.find(word) - 200:said.find(word) + 300]!r}') + break + if run.crashes(): + run.breach(f'crashes: {run.crashes()[-400:]}') + for c in run.clients: + st = c.stats_sum() + links = len(c.links()) + lines = c.check_lines() + differ = [line for line in lines if line.startswith('differ')] + said = c.said_joined() + if links > 1 or said: + run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})') + if differ: + run.breach(f'{c.label}: {len(differ)} checks differ') + if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): + run.breach(f'{c.label}: died of signal {-c.gone}') + if c.gone not in (None, 0) and not c.asked_to_end: + pass # said when found + elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: + run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') + joined = c.said() + for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): + if word in joined: + run.breach(f'{c.label} said "{word}": {joined[joined.find(word) - 200:joined.find(word) + 300]!r}') + break + if isinstance(c, WinClient): + lat, missed = c.latencies() + else: + lat, missed = c.lat, c.no_output + dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) + ROWS.append({ + 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, + 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], + 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), + 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone, + }) + st = (run.root / 'session.stats') + text = '' + if isinstance(run, BoxRun): + text = run.box(f'cat {run.home}/session.stats 2>/dev/null', timeout=30).stdout.decode() + elif st.exists(): + text = st.read_text() + for line in text.splitlines(): + if line.startswith('session inputs'): + w = line.split() + d = {w[i]: w[i + 1] for i in range(1, len(w) - 1, 2)} + worst = int(d.get('boundary_us', '0/0/0').split('/')[2]) + ROWS.append({'cell': run.cell, 'client': 'session', 'rtt': rtt, 'boundary_us': d.get('boundary_us'), + 'input_us': d.get('input_us'), 'cut': d.get('cut')}) + if worst > 250_000: + run.breach(f'session: a boundary took {worst / 1000:.0f} ms') + run.note('end of cell') + + +def table(rows): + cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', + 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit'] + def fmt(v): + if v is None: + return '-' + if isinstance(v, float): + return f'{v:.1f}' + return str(v) + out = [] + widths = {c: len(c) for c in cols} + body = [] + for r in rows: + if r['client'] == 'session': + body.append(f"{r['cell']:<8} session boundary_us p50/p99/max {r['boundary_us']} input_us {r['input_us']} cut {r['cut']}") + continue + vals = [fmt(r.get(c)) for c in cols] + for c, v in zip(cols, vals): + widths[c] = max(widths[c], len(v)) + body.append(vals) + out.append(' '.join(c.ljust(widths[c]) for c in cols)) + for b in body: + out.append(b if isinstance(b, str) else ' '.join(v.ljust(widths[c]) for c, v in zip(cols, b))) + return '\n'.join(out) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('where', choices=['local', 'box']) + ap.add_argument('--minutes', type=float, default=None) + ap.add_argument('--rtts', default='0,30,80,207') + ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') + ap.add_argument('--seed', type=int, default=1) + ap.add_argument('--clients', default='tty,win', help='kinds, one client each: tty,win,win...') + ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) + ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) + ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') + ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') + ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') + ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') + ap.add_argument('--keep', action='store_true', help='keep the run\'s files (and the terminals\' output)') + ap.add_argument('--only', default='', help='the monkey\'s actions, by name: move,insert,...') + ap.add_argument('--skip', default='', help='actions left out, by name') + ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') + args = ap.parse_args() + kinds = args.clients.split(',') + runs = [] + if args.where == 'local': + minutes = args.minutes if args.minutes is not None else 2 + for i, rtt in enumerate(int(x) for x in args.rtts.split(',')): + run = Run(args, f'local{rtt}') + run.far = not args.near + runs.append(run) + print(f'soak: {run.cell}: {kinds} for {minutes} min, seed {args.seed + i}, in {run.root}', flush=True) + soak(run, kinds, rtt, minutes, args.seed + i) + else: + minutes = args.minutes if args.minutes is not None else 20 + run = BoxRun(args, 'box') + runs.append(run) + print(f'soak: box: {kinds} for {minutes} min, seed {args.seed}, in {run.root}', flush=True) + soak(run, kinds, 207, minutes, args.seed) + print(table(ROWS)) + breaches = [(r.cell, b) for r in runs for b in r.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for r in runs: + if args.keep or r.breaches: + print(f'kept: {r.root}') + else: + shutil.rmtree(r.root, ignore_errors=True) + print('soak:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + +if __name__ == '__main__': + main() -- cgit v1.3 From 8e848932bc84ce261b1919692e40df2e9570d67a Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 15:12:59 -0300 Subject: test/soak.py: a cell measures to its end: a client that dies is started again as its user would, and the session too, its epochs counted; the panes on a client's view kept between quiet points (lost: still open, no longer on it); rejoins after a divergence told from an Upgrade's; each client its own XDG_STATE_HOME, as a client on another machine has; cores (systemd-coredump) and deaths by signal are breaches; a session that will not Exit is killed, not a lost table Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 18 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index f75371ef..e6626d53 100644 --- a/test/soak.py +++ b/test/soak.py @@ -166,6 +166,12 @@ class Term: self.clears += seen.count(b'\x1b[2J') self.joined_again += (carry + got).count(b'joined again') self.feed(got) + # A rejoin's notice on the screen, counted as it comes up. + for word, attr in ((' diverged from the session', 'shown_diverged'), ('is upgrading', 'shown_upgrade')): + up = any(word in line for line in self.screen.display) + if up and not getattr(self, attr + '_on', False): + setattr(self, attr, getattr(self, attr, 0) + 1) + setattr(self, attr + '_on', up) if self.log: self.log.write(got) if b'\x1b[5n' in seen: @@ -266,6 +272,9 @@ class Base: def env(self): e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) + # Its own machine's state (Recent, the store), as a client elsewhere has. + (self.dir / 'state').mkdir(exist_ok=True) + e['XDG_STATE_HOME'] = str(self.dir / 'state') if self.run.args.trace: e['PARDES_REPLICA_TRACE'] = '1' if self.run.far: @@ -362,6 +371,11 @@ class TtyClient(Base): def said_joined(self): return self.term.joined_again + def rejoins(self): + """Joins again: (after a divergence, after an Upgrade's refusal).""" + up = getattr(self.term, 'shown_upgrade', 0) + return max(getattr(self.term, 'shown_diverged', 0), len(self.links()) - 1 - up), up + def hangup(self): """As its terminal closing: the normal way out (its stats written).""" try: @@ -449,6 +463,10 @@ class WinClient(Base): def said_joined(self): return self.said().count('joined again') + def rejoins(self): + said = self.said() + return said.count('joined again: this client diverged'), said.count('joined again: the session is upgrading') + def frame_size(self): try: with open(self.cap / 'latest.ppm', 'rb') as f: @@ -723,6 +741,28 @@ class Run: self.nine = None self.nine_lock = threading.RLock() self.session_pid = None + self.epochs = 0 + self.made = [0] + + def reset_session(self): + """What a dead session left (its daemon, its shells, its sockets) gone, + for the next to start clean.""" + for pid in processes(self.root): + if pid not in [c.p.pid for c in self.clients if c.p and c.p.poll() is None]: + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + time.sleep(0.5) + for f in self.root.glob('pardes-*'): + if f.is_socket() or f.name.endswith('.lock'): + f.unlink(missing_ok=True) + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None def note(self, what): with self.nlock: @@ -920,11 +960,25 @@ class BoxRun(Run): def kill_session(self): self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) + def reset_session(self): + self.kill_remote() + if self.fwd and self.fwd.poll() is None: + self.fwd.kill() + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + def relay(self, rtt): return BOX def cleanup(self): super().cleanup() + self.kill_remote() + + def kill_remote(self): # Everything started under run-hunt, by its environment: ps -e prints it. self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) @@ -982,11 +1036,35 @@ def quiet_point(run, live): time.sleep(0.1) if not fresh and c.poll() is None: run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') + if fresh: + panes_check(run, c) if isinstance(c, TtyClient) and fresh: view_check(run, c) release(live) +def panes_check(run, c): + """The panes on the client's view (/layout, read while its view is the live + one) against the last quiet point's: one still in the session (/index) and no + longer on its view was lost (a rejoin's new view, a view gone wrong).""" + layout = run.nine_read('/layout') or b'' + index = run.nine_read('/index') or b'' + exist = {int(r.split()[0]) for r in index.splitlines() if r.split() and r.split()[0].isdigit()} + shown = set() + for line in layout.decode(errors='replace').splitlines(): + w = line.split() + if len(w) > 6 and w[0].isdigit(): + shown |= {int(x) for x in w[6:] if x.isdigit()} + before = getattr(c, 'shown', None) + c.shown = shown + if before is None: + return + lost = (before - shown) & exist + if lost: + c.lost = getattr(c, 'lost', 0) + len(lost) + run.breach(f'{c.label}: panes {sorted(lost)} left its view while still open (links {len(c.links())})') + + def view_check(run, c): """The terminal client's screen and cursor against what the session renders for the same view (its input was the last, so 9P reads that view).""" @@ -1084,23 +1162,32 @@ def global_event(run, rng, live, made): def soak(run, kinds, rtt, minutes, seed): + """The cell: its minutes of typing, the session started again (and every + client) whenever it dies, as its user would, so a cell measures to its end.""" rng = random.Random(seed) - stop = threading.Event() - live = [] - threads = [] - made = [0] + end = now() + minutes * 60 try: - soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made) + while True: + run.epochs += 1 + died = soak_body(run, kinds, rtt, end, rng) + if not died or now() > end - 30 or run.epochs >= 20 or not run.args.respawn: + break + run.note(f'the session started again (epoch {run.epochs + 1})') + run.reset_session() except Exception: import traceback run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) finally: - stop.set() report_cell(run, rtt) run.cleanup() -def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): +def soak_body(run, kinds, rtt, end, rng): + """One session's life in the cell: True if it died before `end`.""" + stop = threading.Event() + live = [] + threads = [] + made = run.made run.start_session() for kind in kinds: made[0] += 1 @@ -1109,7 +1196,6 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) t.start() threads.append(t) - end = now() + minutes * 60 next_quiet = now() + 45 next_event = now() + rng.uniform(20, 60) while now() < end: @@ -1118,9 +1204,31 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): if c.poll() is not None: run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') live.remove(c) + c.ended_at = now() + if run.session_alive() and run.args.respawn: + # As its user would: open it again, and go on. + made[0] += 1 + try: + n = start_client(run, c.kind, c.rtt, made[0]) + except TimeoutError as e: + run.breach(f'{c.kind}: a client started again gave no first frame: {e}') + continue + live.append(n) + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) if not run.session_alive(): - run.breach('the session died') - break + run.breach(f'the session died (epoch {run.epochs}, {now() - run.t0:.0f} s into the cell)') + stop.set() + for c in live: + c.asked_to_end = True + for c in live: + try: + wait_for(f'{c.label}\'s end with its session', lambda: c.poll() is not None, timeout=10) + except TimeoutError: + run.breach(f'{c.label}: still running 10 s after its session died') + c.p.kill() + return True if not live: break if now() > next_event: @@ -1151,6 +1259,7 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): c.p.kill() if run.session_alive(): run.end_session() + return False def tail(c): @@ -1194,16 +1303,16 @@ def report_cell(run, rtt): links = len(c.links()) lines = c.check_lines() differ = [line for line in lines if line.startswith('differ')] - said = c.said_joined() - if links > 1 or said: - run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})') + diverged, upgraded = c.rejoins() + if diverged: + run.breach(f'{c.label}: rejoined {diverged} times after a divergence (links {links}, {upgraded} after an Upgrade)') if differ: run.breach(f'{c.label}: {len(differ)} checks differ') if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): run.breach(f'{c.label}: died of signal {-c.gone}') if c.gone not in (None, 0) and not c.asked_to_end: pass # said when found - elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: + elif c.gone not in (None, 0, 1, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') joined = c.said() for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): @@ -1216,11 +1325,11 @@ def report_cell(run, rtt): lat, missed = c.lat, c.no_output dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) ROWS.append({ - 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, + 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged, 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), - 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone, + 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs, }) st = (run.root / 'session.stats') text = '' @@ -1241,8 +1350,8 @@ def report_cell(run, rtt): def table(rows): - cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', - 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit'] + cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', + 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs'] def fmt(v): if v is None: return '-' @@ -1277,6 +1386,7 @@ def main(): ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') + ap.add_argument('--no-respawn', dest='respawn', action='store_false', help='a client that dies stays dead') ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') -- cgit v1.3 From e7ae154d436991b4a9c7bbe72958262381e0b1d8 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 15:27:04 -0300 Subject: test/soak.py: the gate commands and the narrowing flags in its header; a client's speculation (at once of its inputs) and predicted echo in the table; each session's log kept an epoch, and a dead box session's core; the box's 9P forward waited for until it answers, its socket made afresh at a restart Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index e6626d53..8c73aa1e 100644 --- a/test/soak.py +++ b/test/soak.py @@ -3,9 +3,24 @@ measured (docs/next-daemon.md, the replica and GUI replica notes). python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10] - [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] + [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] [--keep] python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] - [--pardes P] [--gui G] [--seed 1] [--trace] + [--pardes P] [--gui G] [--seed 1] [--trace] [--keep] + +The gates (`source .scratch/daemon/hunt/env.sh` or any env without PARDES_*): + local, ~12 min: python3 -B test/soak.py local --minutes 2.5 + box, ~22 min: python3 -B test/soak.py box --minutes 20 +(the box's binary is deployed first as /root/pardes-test/bin/pardes-hunt, the +same build as --pardes/--gui here). Exit 0 only with no breach at all. + +Narrowing a breach down: `--trace` (PARDES_REPLICA_TRACE at both ends: per-field +digests, each unit and each divergence described in the clients' and the +session's logs), `--only move,insert` / `--skip click,hover` (the monkey's +actions by name, ACTIONS below), `--events resize,detach` ('' for none), +`--no-upgrade`, `--no-respawn` (a dead client stays dead, a dead session ends +the cell), `--near` (local clients without zstd and blobs by hash), `--keep` +(the run's files, with each terminal client's output and the actions log, a +line an input). A breach's files are kept whatever. `local`: a session on this machine (`pardes --detach=soak `, the shape of the user's: a directory pane, then two shells), and at each RTT its @@ -288,11 +303,15 @@ class Base: return [line for line in checks(self.stats) if line.startswith('inputs ')] def stats_sum(self): - keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible') + keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible', 'missed') tot = dict.fromkeys(keys, 0) + tot['echoed'] = tot['typed'] = 0 for line in self.links(): w = line.split() d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)} + a, _, b = d.get('echo', '0/0').partition('/') + tot['echoed'] += int(a or 0) + tot['typed'] += int(b or 0) for k in keys: try: tot[k] += int(d.get(k, 0)) @@ -841,7 +860,7 @@ class Run: if self.args.trace: e['PARDES_REPLICA_TRACE'] = '1' self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, - stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / 'session.err', 'w'), + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / f'session.err.{self.epochs}', 'w'), start_new_session=True) self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20) @@ -874,10 +893,7 @@ class Run: subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30) def session_said(self): - try: - return (self.root / 'session.err').read_text(errors='replace') - except FileNotFoundError: - return '' + return ''.join(f.read_text(errors='replace') for f in sorted(self.root.glob('session.err.*'))) def crashes(self): p = self.root / 'config' / 'pardes' / 'crashes' @@ -926,21 +942,23 @@ class BoxRun(Run): fixture(self.work, rng) tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout bin_ = self.args.remote_bin + fresh = f'rm -rf {self.home}; ' if self.epochs <= 1 else f'mkdir -p {self.home}/cores; mv {self.home}/work/*.core {self.home}/cores/ 2>/dev/null; ' script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' - f'rm -rf {self.home}; mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' + + fresh + f'mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' - f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work {self.home}/session.err 2>&1 & ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work {self.home}/session.err.{self.epochs} 2>&1 & ' f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' - f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err; exit 4; }}') + f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err.{self.epochs}; exit 4; }}') got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) if got.returncode != 0: raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}') self.nine_sock = self.root / 'box9p.sock' + self.nine_sock.unlink(missing_ok=True) self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) wait_for('the 9P forward', self.nine_sock.exists, timeout=20) - status = self.nine_read('/status').decode() + status = wait_for('the session over the 9P forward', lambda: self.nine_read('/status'), timeout=30).decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) for _ in range(2): self.nine_write('/pane/1/exec', b'Tty\n') @@ -951,7 +969,7 @@ class BoxRun(Run): return got.stdout.strip() == b'yes' def session_said(self): - got = self.box(f'cat {self.home}/session.err; ls {self.home}/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) + got = self.box(f'cat {self.home}/session.err.*; ls {self.home}/cores/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) return got.stdout.decode(errors='replace') def crashes(self): @@ -1274,7 +1292,7 @@ ROWS = [] def cores(run): """systemd-coredump's cores since the cell began, of the pids it started.""" - if isinstance(run, BoxRun) or not shutil.which('coredumpctl'): + if not shutil.which("coredumpctl"): return [] since = time.time() - (now() - run.t0) - 5 got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True) @@ -1327,7 +1345,7 @@ def report_cell(run, rtt): ROWS.append({ 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged, 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], - 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'vis/min': st['visible'] / dur, 'at_once': f"{st['at_once']}/{st['inputs']}", 'echo': f"{st['echoed']}/{st['typed']}", 'echo_missed': st['missed'], 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs, }) @@ -1350,7 +1368,7 @@ def report_cell(run, rtt): def table(rows): - cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', + cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'at_once', 'visible', 'mispred', 'vis/min', 'echo', 'echo_missed', 'clears/min', 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs'] def fmt(v): if v is None: -- cgit v1.3 From 1051f99dce57aa5dc389162a5fb34353cde1a718 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 15:36:15 -0300 Subject: test/soak.py: an Upgrade alone can be asked for (--events ''); a terminal client's rejoins after an Upgrade counted by the Upgrades made while it ran (its notice can be gone before a frame shows it); a client's visible rollbacks are its process's count, not summed over its links Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index 8c73aa1e..73419e92 100644 --- a/test/soak.py +++ b/test/soak.py @@ -314,7 +314,9 @@ class Base: tot['typed'] += int(b or 0) for k in keys: try: - tot[k] += int(d.get(k, 0)) + # `visible` is the link's (its process's) count so far, the + # rest each replica's own. + tot[k] = max(tot[k], int(d.get(k, 0))) if k == 'visible' else tot[k] + int(d.get(k, 0)) except ValueError: pass return tot @@ -391,8 +393,11 @@ class TtyClient(Base): return self.term.joined_again def rejoins(self): - """Joins again: (after a divergence, after an Upgrade's refusal).""" - up = getattr(self.term, 'shown_upgrade', 0) + """Joins again: (after a divergence, after an Upgrade's refusal). A + terminal's notice can be gone before a frame shows it: the Upgrades + made while it ran are counted as its refusals.""" + end = getattr(self, 'ended_at', None) or now() + up = max(getattr(self.term, 'shown_upgrade', 0), sum(1 for t in self.run.upgrade_times if self.started <= t <= end)) return max(getattr(self.term, 'shown_diverged', 0), len(self.links()) - 1 - up), up def hangup(self): @@ -762,6 +767,7 @@ class Run: self.session_pid = None self.epochs = 0 self.made = [0] + self.upgrade_times = [] def reset_session(self): """What a dead session left (its daemon, its shells, its sockets) gone, @@ -1126,10 +1132,10 @@ def global_event(run, rng, live, made): """Something happens beside the typing: a resize, a client detached and another attached, an Upgrade, a script's edit over 9P.""" kinds = [e for e in run.args.events.split(',') if e] - if not kinds: - return None if run.args.upgrade and not getattr(run, 'upgraded', False): kinds.append('upgrade') + if not kinds: + return None what = rng.choice(kinds) if what == 'resize': c = rng.choice(live) @@ -1161,6 +1167,7 @@ def global_event(run, rng, live, made): elif what == 'upgrade': run.note('Upgrade') run.upgraded = True + run.upgrade_times.append(now()) run.nine_write('/ctl', b'Upgrade\n') time.sleep(3) if run.nine: -- cgit v1.3 From d0b0c6ba0fb06ad4132b395b3bf9b18072638c33 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 15:50:30 -0300 Subject: test/soak.py instant: does each local action feel local: per action class (type in a file, at a shell prompt, move by keys and by click, scroll a file by keys and the wheel, a terminal's history by keys and the wheel, select, the Panes picker) and client kind, the share applied at once (the replica's trace of stopped inputs, and shown within half a round trip), keystroke to present p50/p99 against a local session's frontend measured in the same run, and the shell's predicted echo; breaches under 99% at once, over two frames past local, under 95% predicted; the soak's blink breach at a visible rollback a minute; clicks as one write; a box restart with no core to keep Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 347 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 339 insertions(+), 8 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index 73419e92..efe45b53 100644 --- a/test/soak.py +++ b/test/soak.py @@ -7,9 +7,27 @@ measured (docs/next-daemon.md, the replica and GUI replica notes). python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] [--pardes P] [--gui G] [--seed 1] [--trace] [--keep] + python3 -B test/soak.py instant [--rtts 0,30,80,207 | --box] [--clients tty,win] + The gates (`source .scratch/daemon/hunt/env.sh` or any env without PARDES_*): local, ~12 min: python3 -B test/soak.py local --minutes 2.5 box, ~22 min: python3 -B test/soak.py box --minutes 20 + instant, ~5 min a cell: python3 -B test/soak.py instant --rtts 207 + python3 -B test/soak.py instant --box + +`instant` asks whether each local action feels local (mosh's point): per action +class (type in a file, type at a shell prompt, move by keys and by click, scroll +a file by keys and the wheel, scroll a terminal's history by keys and the +wheel, select, type into the Panes picker) and client kind, 25 inputs each in +one client a kind, against a local session's own frontend measured first in the +same run. Its columns: `at once`, the share applied at once (none of them +stopped at a host call, by the replica's own trace, and each shown sooner than +half a round trip); p50/p99 keystroke to present; the local session's p99; +`none`, inputs that showed nothing in 2 s; for type_shell the keys whose echo +was predicted. Breaches: a local-only class under 99% at once, or its p99 more +than two frames over the local one; shell echo predicted under 95%; more than +one input showing nothing. The soak's own breach for blink: visible rollbacks at +1 a minute or more. (the box's binary is deployed first as /root/pardes-test/bin/pardes-hunt, the same build as --pardes/--gui here). Exit 0 only with no breach at all. @@ -235,6 +253,17 @@ class Term: time.sleep(wait) os.write(self.master, part) + def click(self, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + self.inputs += 1 + self.run.note(f'{self.label} < click {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, f'\x1b]777;mouse;down;{button};{px};{py}\x07\x1b]777;mouse;up;{button};{px};{py}\x07'.encode()) + except OSError: + pass + def resize(self, rows, cols): self.rows, self.cols = rows, cols self.px = (cols * 16, rows * 33) @@ -366,6 +395,7 @@ class TtyClient(Base): if not self.held.is_set(): with self.lock: self.no_output += 1 + self.run.note(f'{self.label} no output in 2 s after the key at {t - self.run.t0:.3f}') def mouse(self, kind, button, x, y): # SGR: cells from 1; x, y here are fractions of the screen. @@ -375,6 +405,11 @@ class TtyClient(Base): end = 'm' if kind == 'up' else 'M' self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) + def click(self, button, x, y): + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + self.keys(f'\x1b[<{button - 1};{col};{row}M\x1b[<{button - 1};{col};{row}m'.encode()) + def resize(self, rows, cols): self.term.resize(rows, cols) if self.p: @@ -466,6 +501,17 @@ class WinClient(Base): except OSError: pass + def click(self, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + self.inputs += 1 + self.run.note(f'{self.label} < click {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, f'\x1b]777;mouse;down;{button};{px};{py}\x07\x1b]777;mouse;up;{button};{px};{py}\x07'.encode()) + except OSError: + pass + def resize(self, rows, cols): self.rows, self.cols = rows, cols self.px = (cols * 16, rows * 33) @@ -768,6 +814,10 @@ class Run: self.epochs = 0 self.made = [0] self.upgrade_times = [] + # What the session opens (its work directory, or a file in it) and + # the shells made beside it. + self.opens = '' + self.shells = 2 def reset_session(self): """What a dead session left (its daemon, its shells, its sockets) gone, @@ -865,7 +915,7 @@ class Run: e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) if self.args.trace: e['PARDES_REPLICA_TRACE'] = '1' - self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, + self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work / self.opens)], env=e, cwd=self.work, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / f'session.err.{self.epochs}', 'w'), start_new_session=True) self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' @@ -875,7 +925,7 @@ class Run: self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) self.pids.append(self.session_pid) # Two shells beside the directory, as the user's had. - for _ in range(2): + for _ in range(self.shells): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(0.5) @@ -948,12 +998,12 @@ class BoxRun(Run): fixture(self.work, rng) tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout bin_ = self.args.remote_bin - fresh = f'rm -rf {self.home}; ' if self.epochs <= 1 else f'mkdir -p {self.home}/cores; mv {self.home}/work/*.core {self.home}/cores/ 2>/dev/null; ' + fresh = f'rm -rf {self.home}; ' if self.epochs <= 1 else f'mkdir -p {self.home}/cores; mv {self.home}/work/*.core {self.home}/cores/ 2>/dev/null || true; ' script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' + fresh + f'mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' - f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work {self.home}/session.err.{self.epochs} 2>&1 & ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work/{self.opens} {self.home}/session.err.{self.epochs} 2>&1 & ' f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err.{self.epochs}; exit 4; }}') got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) @@ -966,7 +1016,7 @@ class BoxRun(Run): wait_for('the 9P forward', self.nine_sock.exists, timeout=20) status = wait_for('the session over the 9P forward', lambda: self.nine_read('/status'), timeout=30).decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) - for _ in range(2): + for _ in range(self.shells): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(1) @@ -1331,6 +1381,9 @@ def report_cell(run, rtt): diverged, upgraded = c.rejoins() if diverged: run.breach(f'{c.label}: rejoined {diverged} times after a divergence (links {links}, {upgraded} after an Upgrade)') + mins = ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60 + if mins >= 1 and st['visible'] / mins >= 1: + run.breach(f'{c.label}: {st["visible"]} visible rollbacks in {mins:.1f} min (want under 1 a minute)') if differ: run.breach(f'{c.label}: {len(differ)} checks differ') if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): @@ -1352,7 +1405,7 @@ def report_cell(run, rtt): ROWS.append({ 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged, 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], - 'vis/min': st['visible'] / dur, 'at_once': f"{st['at_once']}/{st['inputs']}", 'echo': f"{st['echoed']}/{st['typed']}", 'echo_missed': st['missed'], 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'vis/min': st['visible'] / dur, 'spec/in': f"{st['at_once']}/{st['inputs']}", 'echo': f"{st['echoed']}/{st['typed']}", 'echo_missed': st['missed'], 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs, }) @@ -1375,7 +1428,7 @@ def report_cell(run, rtt): def table(rows): - cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'at_once', 'visible', 'mispred', 'vis/min', 'echo', 'echo_missed', 'clears/min', + cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'spec/in', 'visible', 'mispred', 'vis/min', 'echo', 'echo_missed', 'clears/min', 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs'] def fmt(v): if v is None: @@ -1400,9 +1453,284 @@ def table(rows): return '\n'.join(out) +# ---- instant: does each kind of local action feel local? ---- +# +# The user: "make sure to use the mosh tricks so that user interaction to +# type/scroll/move the cursor feels instant specially when they don't require +# a request going to the daemon". Per action class and client kind: the share +# of its inputs applied at once (not one of them waiting a round trip), its +# keystroke to present p50/p99 against a local session's measured in the same +# run, and for typing at a shell prompt the share of keys whose echo was +# predicted. A class runs in a link of its own (a client started for it), and +# again with only its setup, so its stats line less the setup's is the class's. + +INSTANT_N = 25 + + +def gen_letters(c, rng, i): + c.keys(bytes([97 + rng.randrange(26)])) + + +def gen_moves(c, rng, i): + c.keys([b'j', b'k', b'w', b'b', b'l', b'h', b'e', b'j'][i % 8]) + + +def gen_scroll_keys(c, rng, i): + c.keys(b'\x04' if i % 2 == 0 else b'\x15') + + +def gen_select(c, rng, i): + c.keys(b'x' if i % 3 else b'w') + + +def gen_term_keys(c, rng, i): + c.keys(b'k' if i % 4 else b'\x15') + + +def gen_click(c, rng, i): + c.click(1, rng.uniform(0.15, 0.9), rng.uniform(0.14, 0.4)) + + +def gen_wheel_file(c, rng, i): + c.mouse('wheel', 1 if i % 3 == 2 else -1, 0.5, 0.3) + + +def gen_wheel_term(c, rng, i): + c.mouse('wheel', 1 if i % 3 != 2 else -1, 0.5, 0.85) + + +# name, pane it is in, setup keys, one measured input, keys after, local only +CLASSES = [ + ('type_shell', 'term', b'', gen_letters, b'\x15', False), + ('type_file', 'file', b'i', gen_letters, ESC, True), + ('move_keys', 'file', b'', gen_moves, b'', True), + ('move_click', 'file', b'', gen_click, b'', True), + ('scroll_keys', 'file', b'', gen_scroll_keys, b'', True), + ('scroll_wheel', 'file', b'', gen_wheel_file, b'', True), + ('term_scroll_keys', 'term', b'\x02', gen_term_keys, b'\x02', True), + ('term_scroll_wheel', 'term', b'', gen_wheel_term, b'', True), + ('select', 'file', b'', gen_select, b';', True), + ('picker', 'file', b' bb', gen_letters, ESC, True), +] + + +def instant_panes(run): + rows = (run.nine_read('/index') or b'').decode().splitlines() + big = next(r.split()[0] for r in rows if r.split()[1] == 'text' and 'big.txt' in r) + term = next(r.split()[0] for r in rows if r.split()[1] == 'term') + return big, term + + +def instant_setup(run, where): + """The pane it starts in: big.txt at line 4000 (normal mode), or the shell + at its prompt with 3000 lines of scrollback above. 9P acts in the view + that typed last: the client's, its join's inputs.""" + big, term = run.instant_panes + run.nine_write('/focus', (big if where == 'file' else term).encode()) + if where == 'file': + run.nine_write(f'/pane/{big}/dot', b'4000') + time.sleep(0.4) + + +def present_ms(c, t0, t1): + """Each measured input's keystroke to present, ms, and how many showed + nothing within 2 s.""" + if isinstance(c, TtyClient): + with c.lock: + return list(c.lat), c.no_output + try: + lines = c.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads = [int(w[1]) for w in (line.split() for line in lines) if len(w) >= 2 and w[0] == 'R' and t0 * 1e9 <= int(w[1]) <= t1 * 1e9] + frames = sorted(int(w[2]) for w in (line.split() for line in lines) if len(w) >= 3 and w[0] == 'F') + out, missed = [], 0 + for r in reads: + f = next((x for x in frames if x >= r), None) + if f is not None and f - r <= 2e9: + out.append((f - r) / 1e6) + else: + missed += 1 + return out, missed + + +def instant_class(run, c, cls): + """One class in client `c` (or the local session's frontend): its setup, + its N inputs a frame apart or more, its keys after. Its present latencies, + the inputs that showed nothing in 2 s, and the replica's own count of the + class's events that stopped at a host call (its trace: PARDES_REPLICA_TRACE).""" + name, where, setup, gen, after, _ = cls + rng = random.Random(sum(name.encode())) + instant_setup(run, where) + if setup: + c.keys(setup) + time.sleep(0.8) + if isinstance(c, TtyClient): + with c.lock: + c.lat.clear() + c.no_output = 0 + said = len(c.said()) + t0 = time.monotonic() + for i in range(INSTANT_N): + gen(c, rng, i) + time.sleep(0.16 + rng.uniform(0, 0.06)) + t1 = time.monotonic() + time.sleep(2.2) + lat, missed = present_ms(c, t0, t1) + stops = c.said()[said:].count('stopped at a host call') + if after: + c.keys(after) + time.sleep(0.5) + return lat, missed, stops + + +def local_session(run, kind): + """A session of this machine's own, the regular path (not detached, no + replica): the baseline, in a terminal or a window, on big.txt and a shell.""" + name = f'local-{kind}' + e = dict(run.env) + big = run.work / 'big.txt' + if kind == 'tty': + c = TtyClient(run, name, None) + c.p = c.term.spawn([run.pardes, f'--9p={name}', str(big)], e, run.work, open(c.err, 'w')) + else: + c = WinClient(run, name, None) + c.p = subprocess.Popen([run.gui, f'--9p={name}', str(big)], env=dict(e, PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(c.cap), + PARDES_TEST_LATENCY=str(c.latency), SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(c.cols), + PARDES_TEST_ROWS=str(c.rows)), cwd=run.work, stdin=c.slave, stdout=subprocess.DEVNULL, + stderr=open(c.err, 'w'), start_new_session=True) + c.started = now() + run.pids.append(c.p.pid) + run.clients.append(c) + run.nine_sock = run.root / f'pardes-9p-{name}.sock' + run.nine = None + wait_for(f'{name}\'s 9P', run.nine_sock.exists, timeout=30) + wait_for(f'{name}\'s first frame', c.ready, timeout=30) + run.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + run.instant_panes = instant_panes(run) + run.nine_write(f'/pane/{run.instant_panes[1]}/body', b'seq 1 3000\n') + time.sleep(1.5) + return c + + +def instant(args): + kinds = args.clients.split(',') + cells = ['box'] if args.where == 'box' else [int(x) for x in args.rtts.split(',')] + results = [] + runs = [] + base = {} + # The baseline: each class in a local session's own frontend. + run = Run(args, 'instant-local') + runs.append(run) + fixture(run.work, random.Random(args.seed)) + try: + for kind in kinds: + c = local_session(run, kind) + for cls in CLASSES: + lat, missed, _ = instant_class(run, c, cls) + base[(kind, cls[0])] = lat + c.hangup() + time.sleep(1) + if c.p.poll() is None: + c.p.kill() + finally: + run.cleanup() + for cell in cells: + run = BoxRun(args, 'instant-box') if cell == 'box' else Run(args, f'instant{cell}') + rtt = 207 if cell == 'box' else cell + runs.append(run) + run.opens = 'big.txt' + run.shells = 1 + # The replica's own words for each input that waited (the client's + # trace alone; the session keeps its one-word digests). + run.env['PARDES_REPLICA_TRACE'] = '1' + try: + run.start_session() + run.instant_panes = instant_panes(run) + run.nine_write(f'/pane/{run.instant_panes[1]}/body', b'seq 1 3000\n') + time.sleep(2) + for kind in kinds: + c = None + for cls in CLASSES: + if not run.session_alive(): + run.breach(f'{kind} {cls[0]}: the session died') + break + if c is None or c.poll() is not None: + run.made[0] += 1 + c = start_client(run, kind, rtt, run.made[0]) + time.sleep(0.6) + lat, missed, stops = instant_class(run, c, cls) + echo = None + if cls[0] == 'type_shell' and c.poll() is None: + # Its link alone: its echo counts are this class's. + c.asked_to_end = True + c.hangup() + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + c.p.kill() + st = c.stats_sum() + echo = (st['echoed'], st['typed']) + c = None + died = c is not None and c.poll() is not None + if died: + run.breach(f'{kind} {cls[0]}: the client died ({c.gone}): {tail(c)[-200:]}') + events = INSTANT_N * (2 if cls[0] == 'move_click' else 1) + # At once: no stop, and shown sooner than a round trip could + # bring it (an input queued behind a stopped one says nothing). + quick = sum(1 for x in lat if x < max(rtt / 2, 10)) if rtt else len(lat) + frac = min(1 - stops / events, quick / INSTANT_N) + results.append((run.cell, kind, cls, frac, stops, quick, pct(lat, 50), pct(lat, 99), missed, + pct(base.get((kind, cls[0]), []), 99), died, echo)) + if c is not None and c.poll() is None: + c.asked_to_end = True + c.hangup() + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + c.p.kill() + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + run.cleanup() + frame = 1000 / 60 + print(f'{"cell":12} {"kind":4} {"class":18} {"at once":>8} {"stops":>5} {"quick":>5} {"p50 ms":>7} {"p99 ms":>7} {"local p99":>9} {"none":>4} {"echo pred":>9}') + for r in results: + cell, kind, cls, frac, stops, quick, p50, p99, missed, lp99, died, echo = r + name, local_only = cls[0], cls[5] + fmt = lambda v: '-' if v is None else f'{v:.1f}' + ep = f'{echo[0]}/{echo[1]}' if echo else '' + print(f'{cell:12} {kind:4} {name:18} {100 * frac:7.0f}% {stops:>5} {quick:>5} {fmt(p50):>7} {fmt(p99):>7} {fmt(lp99):>9} {missed:>4} {ep:>9}') + run = next(x for x in runs if x.cell == cell) + if local_only and frac < 0.99: + run.breach(f'{kind} {name}: {100 * frac:.0f}% applied at once ({stops} stopped, {quick} of {INSTANT_N} shown within half a round trip; want >= 99%)') + if local_only and p99 is not None and lp99 is not None and p99 > lp99 + 2 * frame: + run.breach(f'{kind} {name}: present p99 {p99:.1f} ms against a local session\'s {lp99:.1f} (want within 2 frames)') + if missed > 1: + run.breach(f'{kind} {name}: {missed} of {INSTANT_N} inputs showed nothing within 2 s') + if name == 'type_shell': + if not echo or echo[1] == 0: + run.breach(f'{kind} {name}: no key\'s echo predicted or counted') + elif echo[0] / echo[1] < 0.95: + run.breach(f'{kind} {name}: {echo[0]} of {echo[1]} keys\' echo predicted (want >= 95%)') + breaches = [(x.cell, b) for x in runs for b in x.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for x in runs: + if args.keep or x.breaches: + print(f'kept: {x.root}') + else: + shutil.rmtree(x.root, ignore_errors=True) + print('instant:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + def main(): ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) - ap.add_argument('where', choices=['local', 'box']) + ap.add_argument('where', choices=['local', 'box', 'instant']) + ap.add_argument('--box', action='store_true', help='instant: the session on the box (else local, at each --rtts)') ap.add_argument('--minutes', type=float, default=None) ap.add_argument('--rtts', default='0,30,80,207') ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') @@ -1420,6 +1748,9 @@ def main(): ap.add_argument('--skip', default='', help='actions left out, by name') ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') args = ap.parse_args() + if args.where == 'instant': + args.where = 'box' if args.box else 'local' + return instant(args) kinds = args.clients.split(',') runs = [] if args.where == 'local': -- cgit v1.3 From a50c7a6ebe1f877ed4bcda71d6843ffd8a1eba2d Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 16:23:57 -0300 Subject: test/soak.py instant: an input shown within two frames is at once at a short round trip too (a window's software present takes up to a frame); the shell's predicted echo counted over its letters, not the link's keys (its Ctrl-U is never predicted) Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index efe45b53..191408b0 100644 --- a/test/soak.py +++ b/test/soak.py @@ -1679,7 +1679,9 @@ def instant(args): events = INSTANT_N * (2 if cls[0] == 'move_click' else 1) # At once: no stop, and shown sooner than a round trip could # bring it (an input queued behind a stopped one says nothing). - quick = sum(1 for x in lat if x < max(rtt / 2, 10)) if rtt else len(lat) + # (Half a round trip, or two frames: a window's software present + # alone takes up to a frame or so.) + quick = sum(1 for x in lat if x < max(rtt / 2, 2000 / 60)) if rtt else len(lat) frac = min(1 - stops / events, quick / INSTANT_N) results.append((run.cell, kind, cls, frac, stops, quick, pct(lat, 50), pct(lat, 99), missed, pct(base.get((kind, cls[0]), []), 99), died, echo)) @@ -1713,8 +1715,10 @@ def instant(args): if name == 'type_shell': if not echo or echo[1] == 0: run.breach(f'{kind} {name}: no key\'s echo predicted or counted') - elif echo[0] / echo[1] < 0.95: - run.breach(f'{kind} {name}: {echo[0]} of {echo[1]} keys\' echo predicted (want >= 95%)') + elif echo[0] / INSTANT_N < 0.95: + # Of the class's letters (the link's count also has its Ctrl-U, + # which is never predicted). + run.breach(f'{kind} {name}: {echo[0]} of its {INSTANT_N} letters\' echo predicted (want >= 95%)') breaches = [(x.cell, b) for x in runs for b in x.breaches] for cell, b in breaches: print(f'BREACH {cell}: {b}') -- cgit v1.3 From 725e6d4abec56f41688bfb979fbe3b1eadad4729 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 19:38:27 -0300 Subject: test/soak.py: a pane that left a client's view says whether the client joined again since (its view new, the known gap) or not, with its /index row; the check compares serials (/layout's and /index's), never slots Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index 191408b0..b30254a1 100644 --- a/test/soak.py +++ b/test/soak.py @@ -1129,14 +1129,20 @@ def panes_check(run, c): w = line.split() if len(w) > 6 and w[0].isdigit(): shown |= {int(x) for x in w[6:] if x.isdigit()} + # Serials both (/layout's pane fields and /index's first are pane serials, + # never reused): a pane closed and another opened in its slot is no loss. before = getattr(c, 'shown', None) + links_before = getattr(c, 'links_seen', 0) c.shown = shown + c.links_seen = len(c.links()) if before is None: return lost = (before - shown) & exist if lost: c.lost = getattr(c, 'lost', 0) + len(lost) - run.breach(f'{c.label}: panes {sorted(lost)} left its view while still open (links {len(c.links())})') + rows = [r.decode(errors='replace') for r in index.splitlines() if r.split() and r.split()[0].isdigit() and int(r.split()[0]) in lost] + why = 'after a rejoin (its view is new: the known gap)' if c.links_seen > links_before else 'with no rejoin since the last quiet point' + run.breach(f'{c.label}: panes {sorted(lost)} left its view while still open, {why}: {rows}') def view_check(run, c): -- cgit v1.3 From 257eec0ae161b28f69b6c6fa8aae03ecc1bf85f0 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 20:00:06 -0300 Subject: test/soak.py: a soak's session gets a workspace tag without Exit, Dump and their like: the monkey's keys land in tags, and minutes of them ran Exit there (the box's session quit cleanly 88 s in, its stats written, no panic) Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index b30254a1..1a62a5f6 100644 --- a/test/soak.py +++ b/test/soak.py @@ -864,6 +864,12 @@ class Run: wait_for('the relay', at.exists) return at + def safe_tags(self): + """The workspace tag without Exit, Dump and the like: the monkey's keys + land in tags too, and an Enter there runs what the line names (a person + does not type Exit by chance; a monkey does, given minutes).""" + self.nine_write('/tag', b'Newcol Joincol Find Grep Help Themes') + def pane_count(self): """The session's panes (a script's view of /index), cached a second: a Del that closed the last one would end the session.""" @@ -924,6 +930,7 @@ class Run: status = self.nine_read('/status').decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) self.pids.append(self.session_pid) + self.safe_tags() # Two shells beside the directory, as the user's had. for _ in range(self.shells): self.nine_write('/pane/1/exec', b'Tty\n') @@ -1016,6 +1023,7 @@ class BoxRun(Run): wait_for('the 9P forward', self.nine_sock.exists, timeout=20) status = wait_for('the session over the 9P forward', lambda: self.nine_read('/status'), timeout=30).decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + self.safe_tags() for _ in range(self.shells): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(1) -- cgit v1.3 From 4de480393606fa1f680a4fe84a2507679016efc1 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 21:04:41 -0300 Subject: test/soak.py: the monkey never asks for Exit: window mode ended with Enter (left open, it took the next action's keys, and a `k` at a column's top went to its tag, where `quebec` typed `SPC q`, Exit, and a session with nothing unsaved quit as asked: the box's 1093rd-input quit), and no word starts with q Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index 1a62a5f6..400b3dc3 100644 --- a/test/soak.py +++ b/test/soak.py @@ -130,7 +130,7 @@ def pct(xs, p): # ---- the fixture: a directory such as a person works in ---- WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa ' - 'quebec romeo sierra tango uniform victor whiskey xray yankee zulu').split() + 'romeo sierra tango uniform victor whiskey xray yankee zulu').split() def fixture(at, rng): @@ -679,7 +679,9 @@ def act_shell(c, rng, k): def act_window(c, rng, k): - k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l'])) + # Enter ends window mode, as a person would: left open, it took the next + # action's keys, and a `k` at a column's top went to its tag, where `SPC q` ran Exit + k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l']) + b'\r') def act_column(c, rng, k): -- cgit v1.3 From c2ddcb39ac6f80e41aaee0bbbe311cd3ef9d6639 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 22:33:55 -0300 Subject: test/soak.py: a quiet point's check is the second new line after its key, as daemon.py's made_live: the first may answer a check the client took before it sent the key, and its /layout read another client's view, still the live one (the "panes left its view with no rejoin" breaches, at 207 ms: a line 104 and 133 ms after the key, where one asked after it comes no sooner than ~500 ms) Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index 400b3dc3..01e29db4 100644 --- a/test/soak.py +++ b/test/soak.py @@ -1114,7 +1114,10 @@ def quiet_point(run, live): t = now() fresh = False while now() - t < 8 + 2 * c.rtt / 1000: - if len(c.check_lines()) > before: + # the second new line, as daemon.py's made_live: the first may answer a + # check the client took before it sent the key (one check in flight), + # read while another client's view was still the live one + if len(c.check_lines()) > before + 1: fresh = True break time.sleep(0.1) -- cgit v1.3 From 793081277cb46143e3ccd80ae1d868febacf3fbd Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Fri, 30 Oct 2026 23:28:06 -0300 Subject: test/soak.py: a box ssh that stalls is no session's death: whether the session lives is asked again (three times), and the harness goes on (a 20-minute box run ended at 14 min on one `kill -0` over ssh that took more than 30 s) Co-Authored-By: Claude Opus 5.5 --- test/soak.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'test/soak.py') diff --git a/test/soak.py b/test/soak.py index 01e29db4..e3ff18c7 100644 --- a/test/soak.py +++ b/test/soak.py @@ -1031,8 +1031,16 @@ class BoxRun(Run): time.sleep(1) def session_alive(self): - got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) - return got.stdout.strip() == b'yes' + # An ssh that stalls (the box's link, 30 s once in a 20-minute run) is no session's + # death: ask again, and only a no is one (a box gone for good fails the next checks) + for _ in range(3): + try: + got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) + except subprocess.TimeoutExpired: + self.note('harness: the box took 30 s to say whether the session lives; asking again') + continue + return got.stdout.strip() == b'yes' + return True def session_said(self): got = self.box(f'cat {self.home}/session.err.*; ls {self.home}/cores/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) -- cgit v1.3