diff options
| author | Gabriel Schneider <[email protected]> | 2026-10-30 15:50:30 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-10-30 19:37:50 -0300 |
| commit | d0b0c6ba0fb06ad4132b395b3bf9b18072638c33 (patch) | |
| tree | 6a8a68472bc7829a8291bffa5c347eac2a836b28 | |
| parent | 1051f99dce57aa5dc389162a5fb34353cde1a718 (diff) | |
| download | pardes-d0b0c6ba0fb06ad4132b395b3bf9b18072638c33.tar.gz pardes-d0b0c6ba0fb06ad4132b395b3bf9b18072638c33.zip | |
test/soak.py instant: does each local action feel local: per action class (type in a file, at a shell prompt, move by keys and by click, scroll a file by keys and the wheel, a terminal's history by keys and the wheel, select, the Panes picker) and client kind, the share applied at once (the replica's trace of stopped inputs, and shown within half a round trip), keystroke to present p50/p99 against a local session's frontend measured in the same run, and the shell's predicted echo; breaches under 99% at once, over two frames past local, under 95% predicted; the soak's blink breach at a visible rollback a minute; clicks as one write; a box restart with no core to keep
Co-Authored-By: Claude Opus 5.5 <[email protected]>
| -rw-r--r-- | test/soak.py | 347 |
1 files changed, 339 insertions, 8 deletions
diff --git a/test/soak.py b/test/soak.py index 73419e92..efe45b53 100644 --- a/test/soak.py +++ b/test/soak.py @@ -7,9 +7,27 @@ measured (docs/next-daemon.md, the replica and GUI replica notes). python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] [--pardes P] [--gui G] [--seed 1] [--trace] [--keep] + python3 -B test/soak.py instant [--rtts 0,30,80,207 | --box] [--clients tty,win] + The gates (`source .scratch/daemon/hunt/env.sh` or any env without PARDES_*): local, ~12 min: python3 -B test/soak.py local --minutes 2.5 box, ~22 min: python3 -B test/soak.py box --minutes 20 + instant, ~5 min a cell: python3 -B test/soak.py instant --rtts 207 + python3 -B test/soak.py instant --box + +`instant` asks whether each local action feels local (mosh's point): per action +class (type in a file, type at a shell prompt, move by keys and by click, scroll +a file by keys and the wheel, scroll a terminal's history by keys and the +wheel, select, type into the Panes picker) and client kind, 25 inputs each in +one client a kind, against a local session's own frontend measured first in the +same run. Its columns: `at once`, the share applied at once (none of them +stopped at a host call, by the replica's own trace, and each shown sooner than +half a round trip); p50/p99 keystroke to present; the local session's p99; +`none`, inputs that showed nothing in 2 s; for type_shell the keys whose echo +was predicted. Breaches: a local-only class under 99% at once, or its p99 more +than two frames over the local one; shell echo predicted under 95%; more than +one input showing nothing. The soak's own breach for blink: visible rollbacks at +1 a minute or more. (the box's binary is deployed first as /root/pardes-test/bin/pardes-hunt, the same build as --pardes/--gui here). Exit 0 only with no breach at all. @@ -235,6 +253,17 @@ class Term: time.sleep(wait) os.write(self.master, part) + def click(self, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + self.inputs += 1 + self.run.note(f'{self.label} < click {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, f'\x1b]777;mouse;down;{button};{px};{py}\x07\x1b]777;mouse;up;{button};{px};{py}\x07'.encode()) + except OSError: + pass + def resize(self, rows, cols): self.rows, self.cols = rows, cols self.px = (cols * 16, rows * 33) @@ -366,6 +395,7 @@ class TtyClient(Base): if not self.held.is_set(): with self.lock: self.no_output += 1 + self.run.note(f'{self.label} no output in 2 s after the key at {t - self.run.t0:.3f}') def mouse(self, kind, button, x, y): # SGR: cells from 1; x, y here are fractions of the screen. @@ -375,6 +405,11 @@ class TtyClient(Base): end = 'm' if kind == 'up' else 'M' self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) + def click(self, button, x, y): + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + self.keys(f'\x1b[<{button - 1};{col};{row}M\x1b[<{button - 1};{col};{row}m'.encode()) + def resize(self, rows, cols): self.term.resize(rows, cols) if self.p: @@ -466,6 +501,17 @@ class WinClient(Base): except OSError: pass + def click(self, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + self.inputs += 1 + self.run.note(f'{self.label} < click {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, f'\x1b]777;mouse;down;{button};{px};{py}\x07\x1b]777;mouse;up;{button};{px};{py}\x07'.encode()) + except OSError: + pass + def resize(self, rows, cols): self.rows, self.cols = rows, cols self.px = (cols * 16, rows * 33) @@ -768,6 +814,10 @@ class Run: self.epochs = 0 self.made = [0] self.upgrade_times = [] + # What the session opens (its work directory, or a file in it) and + # the shells made beside it. + self.opens = '' + self.shells = 2 def reset_session(self): """What a dead session left (its daemon, its shells, its sockets) gone, @@ -865,7 +915,7 @@ class Run: e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) if self.args.trace: e['PARDES_REPLICA_TRACE'] = '1' - self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, + self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work / self.opens)], env=e, cwd=self.work, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / f'session.err.{self.epochs}', 'w'), start_new_session=True) self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' @@ -875,7 +925,7 @@ class Run: self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) self.pids.append(self.session_pid) # Two shells beside the directory, as the user's had. - for _ in range(2): + for _ in range(self.shells): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(0.5) @@ -948,12 +998,12 @@ class BoxRun(Run): fixture(self.work, rng) tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout bin_ = self.args.remote_bin - fresh = f'rm -rf {self.home}; ' if self.epochs <= 1 else f'mkdir -p {self.home}/cores; mv {self.home}/work/*.core {self.home}/cores/ 2>/dev/null; ' + fresh = f'rm -rf {self.home}; ' if self.epochs <= 1 else f'mkdir -p {self.home}/cores; mv {self.home}/work/*.core {self.home}/cores/ 2>/dev/null || true; ' script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' + fresh + f'mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' - f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work </dev/null >{self.home}/session.err.{self.epochs} 2>&1 & ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work/{self.opens} </dev/null >{self.home}/session.err.{self.epochs} 2>&1 & ' f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err.{self.epochs}; exit 4; }}') got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) @@ -966,7 +1016,7 @@ class BoxRun(Run): wait_for('the 9P forward', self.nine_sock.exists, timeout=20) status = wait_for('the session over the 9P forward', lambda: self.nine_read('/status'), timeout=30).decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) - for _ in range(2): + for _ in range(self.shells): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(1) @@ -1331,6 +1381,9 @@ def report_cell(run, rtt): diverged, upgraded = c.rejoins() if diverged: run.breach(f'{c.label}: rejoined {diverged} times after a divergence (links {links}, {upgraded} after an Upgrade)') + mins = ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60 + if mins >= 1 and st['visible'] / mins >= 1: + run.breach(f'{c.label}: {st["visible"]} visible rollbacks in {mins:.1f} min (want under 1 a minute)') if differ: run.breach(f'{c.label}: {len(differ)} checks differ') if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): @@ -1352,7 +1405,7 @@ def report_cell(run, rtt): ROWS.append({ 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged, 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], - 'vis/min': st['visible'] / dur, 'at_once': f"{st['at_once']}/{st['inputs']}", 'echo': f"{st['echoed']}/{st['typed']}", 'echo_missed': st['missed'], 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'vis/min': st['visible'] / dur, 'spec/in': f"{st['at_once']}/{st['inputs']}", 'echo': f"{st['echoed']}/{st['typed']}", 'echo_missed': st['missed'], 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs, }) @@ -1375,7 +1428,7 @@ def report_cell(run, rtt): def table(rows): - cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'at_once', 'visible', 'mispred', 'vis/min', 'echo', 'echo_missed', 'clears/min', + cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'spec/in', 'visible', 'mispred', 'vis/min', 'echo', 'echo_missed', 'clears/min', 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs'] def fmt(v): if v is None: @@ -1400,9 +1453,284 @@ def table(rows): return '\n'.join(out) +# ---- instant: does each kind of local action feel local? ---- +# +# The user: "make sure to use the mosh tricks so that user interaction to +# type/scroll/move the cursor feels instant specially when they don't require +# a request going to the daemon". Per action class and client kind: the share +# of its inputs applied at once (not one of them waiting a round trip), its +# keystroke to present p50/p99 against a local session's measured in the same +# run, and for typing at a shell prompt the share of keys whose echo was +# predicted. A class runs in a link of its own (a client started for it), and +# again with only its setup, so its stats line less the setup's is the class's. + +INSTANT_N = 25 + + +def gen_letters(c, rng, i): + c.keys(bytes([97 + rng.randrange(26)])) + + +def gen_moves(c, rng, i): + c.keys([b'j', b'k', b'w', b'b', b'l', b'h', b'e', b'j'][i % 8]) + + +def gen_scroll_keys(c, rng, i): + c.keys(b'\x04' if i % 2 == 0 else b'\x15') + + +def gen_select(c, rng, i): + c.keys(b'x' if i % 3 else b'w') + + +def gen_term_keys(c, rng, i): + c.keys(b'k' if i % 4 else b'\x15') + + +def gen_click(c, rng, i): + c.click(1, rng.uniform(0.15, 0.9), rng.uniform(0.14, 0.4)) + + +def gen_wheel_file(c, rng, i): + c.mouse('wheel', 1 if i % 3 == 2 else -1, 0.5, 0.3) + + +def gen_wheel_term(c, rng, i): + c.mouse('wheel', 1 if i % 3 != 2 else -1, 0.5, 0.85) + + +# name, pane it is in, setup keys, one measured input, keys after, local only +CLASSES = [ + ('type_shell', 'term', b'', gen_letters, b'\x15', False), + ('type_file', 'file', b'i', gen_letters, ESC, True), + ('move_keys', 'file', b'', gen_moves, b'', True), + ('move_click', 'file', b'', gen_click, b'', True), + ('scroll_keys', 'file', b'', gen_scroll_keys, b'', True), + ('scroll_wheel', 'file', b'', gen_wheel_file, b'', True), + ('term_scroll_keys', 'term', b'\x02', gen_term_keys, b'\x02', True), + ('term_scroll_wheel', 'term', b'', gen_wheel_term, b'', True), + ('select', 'file', b'', gen_select, b';', True), + ('picker', 'file', b' bb', gen_letters, ESC, True), +] + + +def instant_panes(run): + rows = (run.nine_read('/index') or b'').decode().splitlines() + big = next(r.split()[0] for r in rows if r.split()[1] == 'text' and 'big.txt' in r) + term = next(r.split()[0] for r in rows if r.split()[1] == 'term') + return big, term + + +def instant_setup(run, where): + """The pane it starts in: big.txt at line 4000 (normal mode), or the shell + at its prompt with 3000 lines of scrollback above. 9P acts in the view + that typed last: the client's, its join's inputs.""" + big, term = run.instant_panes + run.nine_write('/focus', (big if where == 'file' else term).encode()) + if where == 'file': + run.nine_write(f'/pane/{big}/dot', b'4000') + time.sleep(0.4) + + +def present_ms(c, t0, t1): + """Each measured input's keystroke to present, ms, and how many showed + nothing within 2 s.""" + if isinstance(c, TtyClient): + with c.lock: + return list(c.lat), c.no_output + try: + lines = c.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads = [int(w[1]) for w in (line.split() for line in lines) if len(w) >= 2 and w[0] == 'R' and t0 * 1e9 <= int(w[1]) <= t1 * 1e9] + frames = sorted(int(w[2]) for w in (line.split() for line in lines) if len(w) >= 3 and w[0] == 'F') + out, missed = [], 0 + for r in reads: + f = next((x for x in frames if x >= r), None) + if f is not None and f - r <= 2e9: + out.append((f - r) / 1e6) + else: + missed += 1 + return out, missed + + +def instant_class(run, c, cls): + """One class in client `c` (or the local session's frontend): its setup, + its N inputs a frame apart or more, its keys after. Its present latencies, + the inputs that showed nothing in 2 s, and the replica's own count of the + class's events that stopped at a host call (its trace: PARDES_REPLICA_TRACE).""" + name, where, setup, gen, after, _ = cls + rng = random.Random(sum(name.encode())) + instant_setup(run, where) + if setup: + c.keys(setup) + time.sleep(0.8) + if isinstance(c, TtyClient): + with c.lock: + c.lat.clear() + c.no_output = 0 + said = len(c.said()) + t0 = time.monotonic() + for i in range(INSTANT_N): + gen(c, rng, i) + time.sleep(0.16 + rng.uniform(0, 0.06)) + t1 = time.monotonic() + time.sleep(2.2) + lat, missed = present_ms(c, t0, t1) + stops = c.said()[said:].count('stopped at a host call') + if after: + c.keys(after) + time.sleep(0.5) + return lat, missed, stops + + +def local_session(run, kind): + """A session of this machine's own, the regular path (not detached, no + replica): the baseline, in a terminal or a window, on big.txt and a shell.""" + name = f'local-{kind}' + e = dict(run.env) + big = run.work / 'big.txt' + if kind == 'tty': + c = TtyClient(run, name, None) + c.p = c.term.spawn([run.pardes, f'--9p={name}', str(big)], e, run.work, open(c.err, 'w')) + else: + c = WinClient(run, name, None) + c.p = subprocess.Popen([run.gui, f'--9p={name}', str(big)], env=dict(e, PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(c.cap), + PARDES_TEST_LATENCY=str(c.latency), SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(c.cols), + PARDES_TEST_ROWS=str(c.rows)), cwd=run.work, stdin=c.slave, stdout=subprocess.DEVNULL, + stderr=open(c.err, 'w'), start_new_session=True) + c.started = now() + run.pids.append(c.p.pid) + run.clients.append(c) + run.nine_sock = run.root / f'pardes-9p-{name}.sock' + run.nine = None + wait_for(f'{name}\'s 9P', run.nine_sock.exists, timeout=30) + wait_for(f'{name}\'s first frame', c.ready, timeout=30) + run.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + run.instant_panes = instant_panes(run) + run.nine_write(f'/pane/{run.instant_panes[1]}/body', b'seq 1 3000\n') + time.sleep(1.5) + return c + + +def instant(args): + kinds = args.clients.split(',') + cells = ['box'] if args.where == 'box' else [int(x) for x in args.rtts.split(',')] + results = [] + runs = [] + base = {} + # The baseline: each class in a local session's own frontend. + run = Run(args, 'instant-local') + runs.append(run) + fixture(run.work, random.Random(args.seed)) + try: + for kind in kinds: + c = local_session(run, kind) + for cls in CLASSES: + lat, missed, _ = instant_class(run, c, cls) + base[(kind, cls[0])] = lat + c.hangup() + time.sleep(1) + if c.p.poll() is None: + c.p.kill() + finally: + run.cleanup() + for cell in cells: + run = BoxRun(args, 'instant-box') if cell == 'box' else Run(args, f'instant{cell}') + rtt = 207 if cell == 'box' else cell + runs.append(run) + run.opens = 'big.txt' + run.shells = 1 + # The replica's own words for each input that waited (the client's + # trace alone; the session keeps its one-word digests). + run.env['PARDES_REPLICA_TRACE'] = '1' + try: + run.start_session() + run.instant_panes = instant_panes(run) + run.nine_write(f'/pane/{run.instant_panes[1]}/body', b'seq 1 3000\n') + time.sleep(2) + for kind in kinds: + c = None + for cls in CLASSES: + if not run.session_alive(): + run.breach(f'{kind} {cls[0]}: the session died') + break + if c is None or c.poll() is not None: + run.made[0] += 1 + c = start_client(run, kind, rtt, run.made[0]) + time.sleep(0.6) + lat, missed, stops = instant_class(run, c, cls) + echo = None + if cls[0] == 'type_shell' and c.poll() is None: + # Its link alone: its echo counts are this class's. + c.asked_to_end = True + c.hangup() + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + c.p.kill() + st = c.stats_sum() + echo = (st['echoed'], st['typed']) + c = None + died = c is not None and c.poll() is not None + if died: + run.breach(f'{kind} {cls[0]}: the client died ({c.gone}): {tail(c)[-200:]}') + events = INSTANT_N * (2 if cls[0] == 'move_click' else 1) + # At once: no stop, and shown sooner than a round trip could + # bring it (an input queued behind a stopped one says nothing). + quick = sum(1 for x in lat if x < max(rtt / 2, 10)) if rtt else len(lat) + frac = min(1 - stops / events, quick / INSTANT_N) + results.append((run.cell, kind, cls, frac, stops, quick, pct(lat, 50), pct(lat, 99), missed, + pct(base.get((kind, cls[0]), []), 99), died, echo)) + if c is not None and c.poll() is None: + c.asked_to_end = True + c.hangup() + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + c.p.kill() + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + run.cleanup() + frame = 1000 / 60 + print(f'{"cell":12} {"kind":4} {"class":18} {"at once":>8} {"stops":>5} {"quick":>5} {"p50 ms":>7} {"p99 ms":>7} {"local p99":>9} {"none":>4} {"echo pred":>9}') + for r in results: + cell, kind, cls, frac, stops, quick, p50, p99, missed, lp99, died, echo = r + name, local_only = cls[0], cls[5] + fmt = lambda v: '-' if v is None else f'{v:.1f}' + ep = f'{echo[0]}/{echo[1]}' if echo else '' + print(f'{cell:12} {kind:4} {name:18} {100 * frac:7.0f}% {stops:>5} {quick:>5} {fmt(p50):>7} {fmt(p99):>7} {fmt(lp99):>9} {missed:>4} {ep:>9}') + run = next(x for x in runs if x.cell == cell) + if local_only and frac < 0.99: + run.breach(f'{kind} {name}: {100 * frac:.0f}% applied at once ({stops} stopped, {quick} of {INSTANT_N} shown within half a round trip; want >= 99%)') + if local_only and p99 is not None and lp99 is not None and p99 > lp99 + 2 * frame: + run.breach(f'{kind} {name}: present p99 {p99:.1f} ms against a local session\'s {lp99:.1f} (want within 2 frames)') + if missed > 1: + run.breach(f'{kind} {name}: {missed} of {INSTANT_N} inputs showed nothing within 2 s') + if name == 'type_shell': + if not echo or echo[1] == 0: + run.breach(f'{kind} {name}: no key\'s echo predicted or counted') + elif echo[0] / echo[1] < 0.95: + run.breach(f'{kind} {name}: {echo[0]} of {echo[1]} keys\' echo predicted (want >= 95%)') + breaches = [(x.cell, b) for x in runs for b in x.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for x in runs: + if args.keep or x.breaches: + print(f'kept: {x.root}') + else: + shutil.rmtree(x.root, ignore_errors=True) + print('instant:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + def main(): ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) - ap.add_argument('where', choices=['local', 'box']) + ap.add_argument('where', choices=['local', 'box', 'instant']) + ap.add_argument('--box', action='store_true', help='instant: the session on the box (else local, at each --rtts)') ap.add_argument('--minutes', type=float, default=None) ap.add_argument('--rtts', default='0,30,80,207') ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') @@ -1420,6 +1748,9 @@ def main(): ap.add_argument('--skip', default='', help='actions left out, by name') ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') args = ap.parse_args() + if args.where == 'instant': + args.where = 'box' if args.box else 'local' + return instant(args) kinds = args.clients.split(',') runs = [] if args.where == 'local': |
