diff options
Diffstat (limited to 'test/soak.py')
| -rw-r--r-- | test/soak.py | 1815 |
1 files changed, 1815 insertions, 0 deletions
diff --git a/test/soak.py b/test/soak.py new file mode 100644 index 00000000..e3ff18c7 --- /dev/null +++ b/test/soak.py @@ -0,0 +1,1815 @@ +#!/usr/bin/env python3 +"""A soak of replica clients: what a person does in them, through each link, +measured (docs/next-daemon.md, the replica and GUI replica notes). + + python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10] + [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] [--keep] + python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] + [--pardes P] [--gui G] [--seed 1] [--trace] [--keep] + + python3 -B test/soak.py instant [--rtts 0,30,80,207 | --box] [--clients tty,win] + +The gates (`source .scratch/daemon/hunt/env.sh` or any env without PARDES_*): + local, ~12 min: python3 -B test/soak.py local --minutes 2.5 + box, ~22 min: python3 -B test/soak.py box --minutes 20 + instant, ~5 min a cell: python3 -B test/soak.py instant --rtts 207 + python3 -B test/soak.py instant --box + +`instant` asks whether each local action feels local (mosh's point): per action +class (type in a file, type at a shell prompt, move by keys and by click, scroll +a file by keys and the wheel, scroll a terminal's history by keys and the +wheel, select, type into the Panes picker) and client kind, 25 inputs each in +one client a kind, against a local session's own frontend measured first in the +same run. Its columns: `at once`, the share applied at once (none of them +stopped at a host call, by the replica's own trace, and each shown sooner than +half a round trip); p50/p99 keystroke to present; the local session's p99; +`none`, inputs that showed nothing in 2 s; for type_shell the keys whose echo +was predicted. Breaches: a local-only class under 99% at once, or its p99 more +than two frames over the local one; shell echo predicted under 95%; more than +one input showing nothing. The soak's own breach for blink: visible rollbacks at +1 a minute or more. +(the box's binary is deployed first as /root/pardes-test/bin/pardes-hunt, the +same build as --pardes/--gui here). Exit 0 only with no breach at all. + +Narrowing a breach down: `--trace` (PARDES_REPLICA_TRACE at both ends: per-field +digests, each unit and each divergence described in the clients' and the +session's logs), `--only move,insert` / `--skip click,hover` (the monkey's +actions by name, ACTIONS below), `--events resize,detach` ('' for none), +`--no-upgrade`, `--no-respawn` (a dead client stays dead, a dead session ends +the cell), `--near` (local clients without zstd and blobs by hash), `--keep` +(the run's files, with each terminal client's output and the actions log, a +line an input). A breach's files are kept whatever. + +`local`: a session on this machine (`pardes --detach=soak <a directory>`, the +shape of the user's: a directory pane, then two shells), and at each RTT its +clients through test/delay_relay.py (with jitter), far as through ssh +(PARDES_REPLICA_FAR=1: the blobs by hash, the stream in zstd). One cell an +RTT, `--minutes` each. `box`: the session on the FreeBSD box under +/root/pardes-test (`--remote-bin`, by default bin/pardes-hunt there, run in +run-hunt and home-hunt), the clients here through ssh, as the user's window +was; its 9P through an `ssh -L` of the session's socket. + +The clients: terminals (`pardes --attach`, in a pty here whose screen pyte +keeps, frame requests answered) and windows (`pardes-gui --attach`, SDL's +offscreen driver, keys and the mouse through its test feed on a pty, so it +can be resized). Each runs a seeded monkey of a person's actions: moves of +every kind (keys, clicks, jumps, search), scrolls (keys, the wheel), edits, +undo and save, Look at files and directories, the Panes picker, Find, new +terminals with programs (ls, a long output, less, vi, top, Ctrl-C), panes and +columns opened, moved and closed, the pointer resting and moving, pastes, +idle gaps. Between them, at random: a client resized, a client detached and +another attached, an Upgrade, a 9P script's edit. `--seed` makes a run +again (the timing differs, the actions do not). + +Measured (a row a client, then the session's): + - rejoins: links past the first (PARDES_REPLICA_STATS writes one a link) + and `joined again` said by it; checks (PARDES_REPLICA_CHECK) that differ; + - exits nobody asked for, a signal or a non-zero code, the session's death, + a panic or `crashes` line; + - freezes: a quiet point where a client gives no fresh check within 8 s, + and a terminal client's key with no output within 2 s while it was not + held; the session's worst boundary (its stats at its end); + - the view: at each quiet point, a terminal client's cursor and screen + (pyte) against what the session renders for its view (/screen read right + after its own input made its view the live one), and the panes on it; + - blink: visible rollbacks (`visible`), mispredicted inputs, a terminal + client's full clears, a minute; + - keystroke to output (terminal) and to a frame (window, PARDES_TEST_LATENCY) + p50 / p99. + +Prints a table, then each breach, and exits 1 if there is any. Everything +started is ended by pid (and found by its environment); the run's files are +kept under $TMPDIR/soak-* (`--keep`) or removed. +""" +import argparse +import fcntl +import json +import os +import random +import re +import shutil +import signal +import socket +import statistics +import struct +import subprocess +import sys +import tempfile +import termios +import threading +import time +import tty as ttymod +from pathlib import Path + +import pyte + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from ninep import Client # noqa: E402 +from daemon import processes, alive, wait_for, checks # noqa: E402 + +BOX = '[email protected]' +SSH = ['/usr/bin/ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', + '-o', 'UserKnownHostsFile=/home/goblin/.ssh/known_hosts', '-i', '/home/goblin/.ssh/id_ed25519'] +BOX_T = '/root/pardes-test' +NAME = 'soak' +ESC = b'\x1b' + + +def now(): + return time.monotonic() + + +def pct(xs, p): + if not xs: + return None + xs = sorted(xs) + return xs[min(len(xs) - 1, len(xs) * p // 100)] + + +# ---- the fixture: a directory such as a person works in ---- + +WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa ' + 'romeo sierra tango uniform victor whiskey xray yankee zulu').split() + + +def fixture(at, rng): + at.mkdir(parents=True, exist_ok=True) + (at / 'notes.txt').write_text(''.join(f'line {i} of the notes: {rng.choice(WORDS)} {rng.choice(WORDS)} see code.zig:{i % 90 + 1}\n' + for i in range(300))) + (at / 'code.zig').write_text(''.join(f'pub fn f{i}(x: u32) u32 {{\n return x * {i} + {rng.randrange(1000)}; // {rng.choice(WORDS)}\n}}\n\n' + for i in range(120))) + (at / 'big.txt').write_text(''.join(f'{i:06d} {" ".join(rng.choice(WORDS) for _ in range(9))}\n' for i in range(8000))) + (at / 'README').write_text('A directory to soak in.\nnotes.txt code.zig big.txt sub/\nsub/inner.txt:3\n') + sub = at / 'sub' + sub.mkdir(exist_ok=True) + (sub / 'inner.txt').write_text(''.join(f'inner {i} {rng.choice(WORDS)}\n' for i in range(60))) + (sub / 'more.md').write_text('# More\n\n' + ''.join(f'- item {i} {rng.choice(WORDS)}\n' for i in range(40))) + + +# ---- terminals ---- + +class Term: + """A terminal: a pty whose output is kept, timed and fed to pyte, its + frame requests (CSI 5 n) answered as a terminal emulator does. An Esc is + written alone and nothing follows it for 80 ms, an answer included: the + client's parser takes an Esc and what follows within 50 ms as Alt.""" + + def __init__(self, rows, cols): + self.master, self.slave = os.openpty() + self.rows, self.cols = rows, cols + fcntl.ioctl(self.slave, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.lock = threading.Lock() + self.wlock = threading.Lock() + self.screen = pyte.Screen(cols, rows) + self.stream = pyte.ByteStream(self.screen) + self.out_bytes = 0 + self.last_out = 0.0 + self.chunks = [] # (time, bytes) of the last minute or so, for latencies + self.clears = 0 + self.joined_again = 0 + self.tail = b'' + self.esc_until = 0.0 + self.closing = False + self.log = None + self.reader = threading.Thread(target=self.drain, daemon=True) + self.reader.start() + + def drain(self): + import select + while not self.closing: + try: + if not select.select([self.master], [], [], 0.05)[0]: + continue + got = os.read(self.master, 65536) + except OSError: + return + if not got: + return + t = now() + carry, self.tail = self.tail, (self.tail + got)[-11:] + seen = carry[-3:] + got + with self.lock: + self.out_bytes += len(got) + self.last_out = t + self.chunks.append((t, len(got))) + if len(self.chunks) > 4096: + del self.chunks[:2048] + # The carry is shorter than either word: a match holds a new byte. + self.clears += seen.count(b'\x1b[2J') + self.joined_again += (carry + got).count(b'joined again') + self.feed(got) + # A rejoin's notice on the screen, counted as it comes up. + for word, attr in ((' diverged from the session', 'shown_diverged'), ('is upgrading', 'shown_upgrade')): + up = any(word in line for line in self.screen.display) + if up and not getattr(self, attr + '_on', False): + setattr(self, attr, getattr(self, attr, 0) + 1) + setattr(self, attr + '_on', up) + if self.log: + self.log.write(got) + if b'\x1b[5n' in seen: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + with self.wlock: + try: + os.write(self.master, b'\x1b[0n') + except OSError: + return + + SGR = re.compile(rb'\x1b\[[0-9:;]*m') + + def feed(self, got): + """To pyte, without its colours (pyte reads no colon sub-parameters) + and holding back a sequence a read cut in two.""" + data = getattr(self, 'held_back', b'') + got + cut = data.rfind(b'\x1b') + self.held_back = b'' + if cut >= 0 and not re.match(rb'\x1b(\[[0-9:;?<>=! ]*[@-~]|[^\[\]]|\][^\x07\x1b]*(\x07|\x1b\\))', data[cut:]): + data, self.held_back = data[:cut], data[cut:] + try: + self.stream.feed(self.SGR.sub(b'', data)) + except Exception: + pass + + def write(self, data): + """Keys as typed: each Esc alone, 80 ms clear after it.""" + parts = re.split(b'(\x1b(?![\\[O\\]n0-9a-z]))', data) + for part in parts: + if not part: + continue + if part == ESC: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, ESC) + self.esc_until = now() + 0.08 + time.sleep(0.08) + else: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, part) + + def click(self, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + self.inputs += 1 + self.run.note(f'{self.label} < click {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, f'\x1b]777;mouse;down;{button};{px};{py}\x07\x1b]777;mouse;up;{button};{px};{py}\x07'.encode()) + except OSError: + pass + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + with self.lock: + self.screen.resize(rows, cols) + + def view(self): + with self.lock: + return [line.rstrip() for line in self.screen.display], (self.screen.cursor.y, self.screen.cursor.x), self.screen.cursor.hidden + + def spawn(self, argv, env, cwd, err): + slave = self.slave + return subprocess.Popen(argv, env=env, cwd=cwd, stdin=slave, stdout=slave, stderr=err, start_new_session=True, + preexec_fn=lambda: fcntl.ioctl(0, termios.TIOCSCTTY, 0)) + + def close(self): + self.closing = True + self.reader.join(timeout=2) + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- clients ---- + +class Base: + def __init__(self, run, kind, label, link): + self.run = run + self.kind = kind + self.label = label + self.link = link # where it dials: a socket path, or the box + self.dir = run.root / label + self.dir.mkdir() + self.check = self.dir / 'check' + self.stats = self.dir / 'stats' + self.err = self.dir / 'err' + self.p = None + self.started = None + self.asked_to_end = False + self.gone = None # how it ended, if it did + self.inputs = 0 + self.lat = [] # ms, input to output/frame + self.no_output = 0 + self.held = threading.Event() # the monkey waits while set + self.paused = threading.Event() # set by the monkey once it waits + self.lock = threading.Lock() + + def env(self): + e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) + # Its own machine's state (Recent, the store), as a client elsewhere has. + (self.dir / 'state').mkdir(exist_ok=True) + e['XDG_STATE_HOME'] = str(self.dir / 'state') + if self.run.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + if self.run.far: + e['PARDES_REPLICA_FAR'] = '1' + return e + + def attach_arg(self): + return f'--attach={self.link}:{NAME}' + + def links(self): + return [line for line in checks(self.stats) if line.startswith('inputs ')] + + def stats_sum(self): + keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible', 'missed') + tot = dict.fromkeys(keys, 0) + tot['echoed'] = tot['typed'] = 0 + for line in self.links(): + w = line.split() + d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)} + a, _, b = d.get('echo', '0/0').partition('/') + tot['echoed'] += int(a or 0) + tot['typed'] += int(b or 0) + for k in keys: + try: + # `visible` is the link's (its process's) count so far, the + # rest each replica's own. + tot[k] = max(tot[k], int(d.get(k, 0))) if k == 'visible' else tot[k] + int(d.get(k, 0)) + except ValueError: + pass + return tot + + def check_lines(self): + return checks(self.check) + + def poll(self): + if self.gone is None and self.p is not None and self.p.poll() is not None: + self.gone = self.p.returncode + return self.gone + + +class TtyClient(Base): + def __init__(self, run, label, link, rows=30, cols=100): + super().__init__(run, 'tty', label, link) + self.term = Term(rows, cols) + self.term.log = open(self.dir / 'out', 'wb', buffering=0) if run.args.keep else None + + def start(self): + argv = [self.run.pardes, self.attach_arg()] + # Its stderr a file: a panic's words and the replica's trace. + self.p = self.term.spawn(argv, self.env(), self.run.root, open(self.err, 'w')) + self.started = now() + + def ready(self): + return self.term.out_bytes > 2000 and now() - self.term.last_out > 0.3 + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + before = self.term.out_bytes + t = now() + self.term.write(data) + threading.Thread(target=self._latency, args=(t, before), daemon=True).start() + + def _latency(self, t, before): + deadline = t + 2.0 + while now() < deadline: + with self.term.lock: + got = [c for c in self.term.chunks if c[0] >= t] + if got: + with self.lock: + self.lat.append((got[0][0] - t) * 1000) + return + time.sleep(0.005) + if not self.held.is_set(): + with self.lock: + self.no_output += 1 + self.run.note(f'{self.label} no output in 2 s after the key at {t - self.run.t0:.3f}') + + def mouse(self, kind, button, x, y): + # SGR: cells from 1; x, y here are fractions of the screen. + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + b = {'down': button - 1, 'up': button - 1, 'motion': 35, 'wheel': 64 if button > 0 else 65}[kind] + end = 'm' if kind == 'up' else 'M' + self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) + + def click(self, button, x, y): + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + self.keys(f'\x1b[<{button - 1};{col};{row}M\x1b[<{button - 1};{col};{row}m'.encode()) + + def resize(self, rows, cols): + self.term.resize(rows, cols) + if self.p: + try: + os.killpg(self.p.pid, signal.SIGWINCH) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.term.joined_again + + def rejoins(self): + """Joins again: (after a divergence, after an Upgrade's refusal). A + terminal's notice can be gone before a frame shows it: the Upgrades + made while it ran are counted as its refusals.""" + end = getattr(self, 'ended_at', None) or now() + up = max(getattr(self.term, 'shown_upgrade', 0), sum(1 for t in self.run.upgrade_times if self.started <= t <= end)) + return max(getattr(self.term, 'shown_diverged', 0), len(self.links()) - 1 - up), up + + def hangup(self): + """As its terminal closing: the normal way out (its stats written).""" + try: + os.killpg(self.p.pid, signal.SIGHUP) + except OSError: + pass + + def close(self): + self.term.close() + + +class WinClient(Base): + """A window client in test mode: offscreen, its keys and pointer through + its test feed on a pty (Escape as CSI 27 u: a lone ESC waits for more).""" + + def __init__(self, run, label, link, rows=34, cols=110): + super().__init__(run, 'win', label, link) + self.cap = self.dir / 'cap' + self.cap.mkdir() + self.latency = self.dir / 'latency' + self.master, self.slave = os.openpty() + ttymod.setraw(self.slave) + self.rows, self.cols = rows, cols + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.wlock = threading.Lock() + self.px = (cols * 16, rows * 33) # test mode's 27 px font, about + + def start(self): + e = dict(self.env(), PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(self.cap), PARDES_TEST_LATENCY=str(self.latency), + SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(self.cols), PARDES_TEST_ROWS=str(self.rows)) + self.p = subprocess.Popen([self.run.gui, self.attach_arg()], env=e, cwd=self.run.root, stdin=self.slave, + stdout=subprocess.DEVNULL, stderr=open(self.err, 'w'), start_new_session=True) + self.started = now() + + def ready(self): + # Software presenting (offscreen) captures no ppm: its first frame is + # its latency trace's first F. + try: + return b'F ' in self.latency.read_bytes() and self.p.poll() is None + except FileNotFoundError: + return False + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + with self.wlock: + try: + os.write(self.master, data.replace(ESC + b'[', b'\x00CSI').replace(ESC, b'\x1b[27u').replace(b'\x00CSI', ESC + b'[')) + except OSError: + pass + + def mouse(self, kind, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + if kind == 'wheel': + seq = f'\x1b]777;mouse;wheel;{button};{px};{py}\x07' + else: + seq = f'\x1b]777;mouse;{kind};{button};{px};{py}\x07' + self.inputs += 1 + self.run.note(f'{self.label} < mouse {kind} {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, seq.encode()) + except OSError: + pass + + def click(self, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + self.inputs += 1 + self.run.note(f'{self.label} < click {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, f'\x1b]777;mouse;down;{button};{px};{py}\x07\x1b]777;mouse;up;{button};{px};{py}\x07'.encode()) + except OSError: + pass + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + + def hangup(self): + """Its test feed's end: the normal way out (its stats written).""" + try: + os.close(self.master) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.said().count('joined again') + + def rejoins(self): + said = self.said() + return said.count('joined again: this client diverged'), said.count('joined again: the session is upgrading') + + def frame_size(self): + try: + with open(self.cap / 'latest.ppm', 'rb') as f: + head = f.read(32).split() + self.px = (int(head[1]), int(head[2])) + except (OSError, IndexError, ValueError): + pass + + def latencies(self): + """Each stdin read to the end of the first frame after it, ms; and the + reads no frame followed within 2 s (a frame came after a later one).""" + try: + lines = self.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads, frames = [], [] + for line in lines: + w = line.split() + if len(w) >= 2 and w[0] == 'R': + reads.append(int(w[1])) + elif len(w) >= 3 and w[0] == 'F': + frames.append(int(w[2])) + out, missed, j = [], 0, 0 + for r in reads: + while j < len(frames) and frames[j] < r: + j += 1 + if j < len(frames): + d = (frames[j] - r) / 1e6 + if d <= 2000: + out.append(d) + else: + missed += 1 + return out, missed + + def close(self): + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- what a person does ---- + +def words(rng, n): + return ' '.join(rng.choice(WORDS) for _ in range(n)).encode() + + +def burst(c, rng): + """One action of a person's, in whatever state the client is.""" + k = c.keys + acts = c.run.actions + choice = rng.choices(acts, weights=[a[1] for a in acts])[0] + choice[2](c, rng, k) + return choice[0] + + +def act_move(c, rng, k): + for _ in range(rng.randint(1, 6)): + k(rng.choice([b'j', b'k', b'h', b'l', b'w', b'b', b'e', b'5j', b'3k', b'gg', b'ge', b'\x04', b'\x15', + b'\x06', b'\x02', b'}', b'{', b'%', b'x', b'12G', b'gl', b'gh'])) + time.sleep(rng.uniform(0.02, 0.2)) + + +def act_search(c, rng, k): + k(b'/' + rng.choice(WORDS).encode() + b'\r') + for _ in range(rng.randint(0, 3)): + time.sleep(rng.uniform(0.1, 0.4)) + k(rng.choice([b'n', b'N'])) + + +def act_jump(c, rng, k): + k(rng.choice([b'\x0f', b' jo', b' jj', b'gd', b'\x0f\x0f'])) + + +def act_insert(c, rng, k): + k(rng.choice([b'i', b'a', b'o', b'O', b'A'])) + for w in words(rng, rng.randint(1, 5)).split(): + k(w + b' ') + time.sleep(rng.uniform(0.03, 0.15)) + k(ESC) + + +def act_delete(c, rng, k): + k(rng.choice([b'xd', b'd', b'wd', b'xxd', b'c' + words(rng, 1) + ESC])) + + +def act_undo(c, rng, k): + k(rng.choice([b'u', b'uu', b'U', b'u'])) + + +def act_save(c, rng, k): + k(b' fs') + + +def act_look(c, rng, k): + # Enter is Look: a directory's row opens it, a word finds its next place. + k(rng.choice([b'\r', b'j\r', b'jj\r', b'gg\r'])) + + +def act_panes(c, rng, k): + k(b' bb') + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'\r', ESC, b'jj\r', b'not\r'])) + + +def act_find(c, rng, k): + k(rng.choice([b' ff', b' fr', b' fg'])) + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'notes\r', ESC, b'j\r', b'inner\r'])) + + +def act_shell(c, rng, k): + # A terminal of its own, a program in it, and back out to normal mode. + k(b' nt') + time.sleep(rng.uniform(0.8, 1.5)) + for _ in range(rng.randint(1, 3)): + prog = rng.choice(['ls -la\r', 'seq 1 3000\r', 'less big.txt\r', 'vi notes.txt\r', 'top\r', 'sleep 30\r', + 'echo ' + rng.choice(WORDS) + '\r', 'cat code.zig\r']) + k(prog.encode()) + time.sleep(rng.uniform(0.5, 2.0)) + if prog.startswith('less'): + # Never SPC then q: that is Exit, should the keys reach no less. + k(b'jjjj\x06') + time.sleep(0.3) + k(b'q') + elif prog.startswith('vi'): + k(b'jjjjx') + k(ESC) + k(b':q!\r') + elif prog.startswith(('top', 'sleep', 'seq')): + k(b'\x03' if not prog.startswith('top') else b'q') + time.sleep(rng.uniform(0.2, 0.8)) + k(b'\x02') # Ctrl-b: normal mode + if rng.random() < 0.5 and c.run.pane_count() >= 4: + time.sleep(0.3) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b''])) + + +def act_window(c, rng, k): + # Enter ends window mode, as a person would: left open, it took the next + # action's keys, and a `k` at a column's top went to its tag, where `SPC q` ran Exit + k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l']) + b'\r') + + +def act_column(c, rng, k): + few = c.run.pane_count() < 5 + k(rng.choice([b' cn', b' cn'] if few else [b' cn', b' cn', b' cd', b' cj'])) + + +def act_del(c, rng, k): + if c.run.pane_count() < 4: + return act_move(c, rng, k) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b'', b' d'])) + + +def act_new(c, rng, k): + k(b' fn') + time.sleep(0.4) + k(b'i' + words(rng, 4)) + k(ESC) + + +def act_esc(c, rng, k): + k(rng.choice([ESC, b'\x02', b':', b':'])) + time.sleep(0.3) + k(ESC) + + +def act_click(c, rng, k): + x, y = rng.random(), rng.uniform(0.05, 1.0) + b = rng.choice([1, 1, 1, 3]) + c.mouse('down', b, x, y) + time.sleep(rng.uniform(0.03, 0.12)) + if rng.random() < 0.3: # a drag + c.mouse('motion', b, min(1, x + rng.uniform(0, 0.2)), min(1, y + rng.uniform(0, 0.1))) + c.mouse('up', b, x, y) + + +def act_wheel(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + d = rng.choice([1, -1]) + for _ in range(rng.randint(1, 8)): + c.mouse('wheel', d, x, y) + time.sleep(rng.uniform(0.01, 0.08)) + + +def act_hover(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + for _ in range(rng.randint(2, 12)): + x = min(1, max(0, x + rng.uniform(-0.05, 0.05))) + y = min(1, max(0.05, y + rng.uniform(-0.03, 0.03))) + c.mouse('motion', 0, x, y) + time.sleep(rng.uniform(0.01, 0.06)) + time.sleep(rng.uniform(0.05, 0.6)) # resting: a Look preview + + +def act_paste(c, rng, k): + k(b'i\x1b[200~' + b'\n'.join(words(rng, 6) for _ in range(rng.randint(1, 4))) + b'\x1b[201~') + k(ESC) + + +def act_settings(c, rng, k): + k(rng.choice([b' tw', b' ?', b' hm', b' tt'])) + time.sleep(0.5) + k(ESC) + + +def act_idle(c, rng, k): + time.sleep(rng.choice([0.5, 1, 2, 4, 8])) + + +ACTIONS = [ + ('move', 30, act_move), ('search', 5, act_search), ('jump', 4, act_jump), ('insert', 12, act_insert), + ('delete', 4, act_delete), ('undo', 5, act_undo), ('save', 3, act_save), ('look', 6, act_look), + ('panes', 3, act_panes), ('find', 3, act_find), ('shell', 3, act_shell), ('window', 4, act_window), + ('column', 1, act_column), ('del', 2, act_del), ('new', 2, act_new), ('esc', 4, act_esc), + ('click', 8, act_click), ('wheel', 8, act_wheel), ('hover', 6, act_hover), ('paste', 2, act_paste), + ('settings', 1, act_settings), ('idle', 6, act_idle), +] + + +def monkey(c, rng, stop): + while not stop.is_set(): + if c.held.is_set(): + c.paused.set() + time.sleep(0.05) + continue + c.paused.clear() + if c.poll() is not None: + return + try: + what = burst(c, rng) + c.run.note(f'{c.label} {what}') + except OSError: + return + time.sleep(rng.uniform(0.05, 0.6)) + + +# ---- the run ---- + +class Run: + def __init__(self, args, cell): + self.args = args + self.cell = cell + self.root = Path(tempfile.mkdtemp(prefix=f'soak-{cell}-', dir=os.environ.get('TMPDIR') or '/tmp')) + self.pardes = str(Path(args.pardes).resolve()) + self.gui = str(Path(args.gui).resolve()) + self.far = True + self.pids = [] + self.clients = [] + self.relays = [] + self.events = [] + self.breaches = [] + self.notes = open(self.root / 'actions', 'w') + only = set(args.only.split(',')) if args.only else None + skip = set(args.skip.split(',')) if args.skip else set() + self.actions = [a for a in ACTIONS if (only is None or a[0] in only) and a[0] not in skip] + self.nlock = threading.Lock() + self.t0 = now() + env = {k: v for k, v in os.environ.items() if not k.startswith(('PARDES_', 'WAYLAND', 'DBUS_', 'DISPLAY', 'BROWSER')) + and 'PAGER' not in k and k not in ('winid', 'EDITOR', 'NINE_MOUNT')} + env.update(HOME=str(self.root), XDG_RUNTIME_DIR=str(self.root), XDG_CONFIG_HOME=str(self.root / 'config'), + XDG_CACHE_HOME=str(self.root / 'cache'), XDG_STATE_HOME=str(self.root / 'state'), + TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', PARDES_NOTIME='1') + (self.root / 'config' / 'pardes').mkdir(parents=True) + (self.root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n') + self.env = env + self.work = self.root / 'work' + self.nine = None + self.nine_lock = threading.RLock() + self.session_pid = None + self.epochs = 0 + self.made = [0] + self.upgrade_times = [] + # What the session opens (its work directory, or a file in it) and + # the shells made beside it. + self.opens = '' + self.shells = 2 + + def reset_session(self): + """What a dead session left (its daemon, its shells, its sockets) gone, + for the next to start clean.""" + for pid in processes(self.root): + if pid not in [c.p.pid for c in self.clients if c.p and c.p.poll() is None]: + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + time.sleep(0.5) + for f in self.root.glob('pardes-*'): + if f.is_socket() or f.name.endswith('.lock'): + f.unlink(missing_ok=True) + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + + def note(self, what): + with self.nlock: + self.notes.write(f'{now() - self.t0:9.3f} {what}\n') + self.notes.flush() + + def breach(self, what): + self.note('BREACH ' + what) + self.breaches.append(what) + + def spawn(self, argv, **kw): + p = subprocess.Popen(argv, **kw) + self.pids.append(p.pid) + return p + + def relay(self, rtt): + if not rtt: + return self.root / 'pardes-replica.sock' + at = self.root / f'relay{rtt}.sock' + if not at.exists(): + self.relays.append(self.spawn([sys.executable, '-B', str(HERE / 'delay_relay.py'), str(at), str(self.root / 'pardes-replica.sock'), + '--delay', str(rtt / 2), '--jitter', str(self.args.jitter)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) + wait_for('the relay', at.exists) + return at + + def safe_tags(self): + """The workspace tag without Exit, Dump and the like: the monkey's keys + land in tags too, and an Enter there runs what the line names (a person + does not type Exit by chance; a monkey does, given minutes).""" + self.nine_write('/tag', b'Newcol Joincol Find Grep Help Themes') + + def pane_count(self): + """The session's panes (a script's view of /index), cached a second: + a Del that closed the last one would end the session.""" + t = now() + if t - getattr(self, 'counted_at', 0) > 1: + rows = self.nine_read('/index') or b'' + self.counted, self.counted_at = len(rows.splitlines()), t + return self.counted + + def nine_read(self, path): + with self.nine_lock: + return self._nine_read(path) + + def nine_write(self, path, data): + with self.nine_lock: + return self._nine_write(path, data) + + def _nine_read(self, path): + for _ in range(3): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.read(path) + except (OSError, EOFError): + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + time.sleep(0.5) + return None + + def _nine_write(self, path, data): + for _ in range(2): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.write(path, data) + except EOFError: + self.nine = None + time.sleep(0.5) + except OSError as e: + return e + + # The session: here. + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) + if self.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work / self.opens)], env=e, cwd=self.work, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / f'session.err.{self.epochs}', 'w'), + start_new_session=True) + self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' + wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20) + wait_for('the replica socket', (self.root / 'pardes-replica.sock').exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + self.pids.append(self.session_pid) + self.safe_tags() + # Two shells beside the directory, as the user's had. + for _ in range(self.shells): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(0.5) + + def session_alive(self): + return alive(self.session_pid) if self.session_pid else False + + def end_session(self): + """Exit (twice: unsaved text asks once), then --kill if it stays.""" + for _ in range(3): + self.nine_write('/ctl', b'Exit\n') + try: + wait_for('the session\'s end', lambda: not self.session_alive(), timeout=3) + return True + except TimeoutError: + pass + self.note('the session did not Exit: --kill') + self.kill_session() + return False + + def kill_session(self): + subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30) + + def session_said(self): + return ''.join(f.read_text(errors='replace') for f in sorted(self.root.glob('session.err.*'))) + + def crashes(self): + p = self.root / 'config' / 'pardes' / 'crashes' + return p.read_text() if p.exists() else '' + + def cleanup(self): + for c in self.clients: + if c.p and c.p.poll() is None: + c.p.kill() + c.close() + for pid in self.pids + processes(self.root): + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + + +class BoxRun(Run): + """The session on the FreeBSD box; the clients here through ssh.""" + + def __init__(self, args, cell): + super().__init__(args, cell) + self.far = False # an ssh client is far by itself + self.run_dir = f'{BOX_T}/run-hunt' + self.home = f'{BOX_T}/home-hunt' + self.env['PARDES_SSH'] = ' '.join(SSH) + self.env['PARDES_REMOTE_RUNTIME'] = self.run_dir + self.fwd = None + + def box(self, script, timeout=60): + return subprocess.run(SSH + [BOX, script], capture_output=True, timeout=timeout) + + def benv(self): + return (f'HOME={self.home} XDG_RUNTIME_DIR={self.run_dir} XDG_CONFIG_HOME={self.home}/config TMPDIR={BOX_T}/tmp-hunt ' + f'XDG_CACHE_HOME={self.home}/cache XDG_STATE_HOME={self.home}/state PATH={BOX_T}/bin:/usr/local/bin:/usr/bin:/bin ' + f'SHELL=/usr/local/bin/bash TERM=xterm-256color PARDES_NOTIME=1 PARDES_REPLICA_STATS={self.home}/session.stats' + + (' PARDES_REPLICA_TRACE=1' if self.args.trace else '')) + + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout + bin_ = self.args.remote_bin + fresh = f'rm -rf {self.home}; ' if self.epochs <= 1 else f'mkdir -p {self.home}/cores; mv {self.home}/work/*.core {self.home}/cores/ 2>/dev/null || true; ' + script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' + + fresh + f'mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' + f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' + f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work/{self.opens} </dev/null >{self.home}/session.err.{self.epochs} 2>&1 & ' + f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' + f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err.{self.epochs}; exit 4; }}') + got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) + if got.returncode != 0: + raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}') + self.nine_sock = self.root / 'box9p.sock' + self.nine_sock.unlink(missing_ok=True) + self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + wait_for('the 9P forward', self.nine_sock.exists, timeout=20) + status = wait_for('the session over the 9P forward', lambda: self.nine_read('/status'), timeout=30).decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + self.safe_tags() + for _ in range(self.shells): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + + def session_alive(self): + # An ssh that stalls (the box's link, 30 s once in a 20-minute run) is no session's + # death: ask again, and only a no is one (a box gone for good fails the next checks) + for _ in range(3): + try: + got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) + except subprocess.TimeoutExpired: + self.note('harness: the box took 30 s to say whether the session lives; asking again') + continue + return got.stdout.strip() == b'yes' + return True + + def session_said(self): + got = self.box(f'cat {self.home}/session.err.*; ls {self.home}/cores/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) + return got.stdout.decode(errors='replace') + + def crashes(self): + return '' + + def kill_session(self): + self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) + + def reset_session(self): + self.kill_remote() + if self.fwd and self.fwd.poll() is None: + self.fwd.kill() + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + + def relay(self, rtt): + return BOX + + def cleanup(self): + super().cleanup() + self.kill_remote() + + def kill_remote(self): + # Everything started under run-hunt, by its environment: ps -e prints it. + self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' + f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) + + +def start_client(run, kind, rtt, n): + label = f'{kind}{rtt}-{n}' + link = run.relay(rtt) + c = (TtyClient if kind == 'tty' else WinClient)(run, label, link) + c.rtt = rtt + c.start() + run.pids.append(c.p.pid) + run.clients.append(c) + run.note(f'{label} started (pid {c.p.pid})') + wait_for(f'{label}\'s first frame', c.ready, timeout=60 if isinstance(run, BoxRun) else 30) + if isinstance(c, WinClient): + c.frame_size() + time.sleep(0.3) + return c + + +def hold(run, clients, timeout=30): + for c in clients: + c.held.set() + deadline = now() + timeout + for c in clients: + while not c.paused.is_set() and c.poll() is None and now() < deadline: + time.sleep(0.05) + + +def release(clients): + for c in clients: + c.held.clear() + + +def quiet_point(run, live): + """Every monkey held: each client must give a fresh check within 8 s + (a freeze otherwise), and none may differ; a terminal client's view + against the session's.""" + hold(run, live) + time.sleep(1.5) + for c in live: + if c.poll() is not None: + continue + before = len(c.check_lines()) + # A key that moves nothing a digest sees but the view's cursor, so the + # client's input is the session's last: its view is the live one. + c.keys(b'l') + t = now() + fresh = False + while now() - t < 8 + 2 * c.rtt / 1000: + # the second new line, as daemon.py's made_live: the first may answer a + # check the client took before it sent the key (one check in flight), + # read while another client's view was still the live one + if len(c.check_lines()) > before + 1: + fresh = True + break + time.sleep(0.1) + if not fresh and c.poll() is None: + run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') + if fresh: + panes_check(run, c) + if isinstance(c, TtyClient) and fresh: + view_check(run, c) + release(live) + + +def panes_check(run, c): + """The panes on the client's view (/layout, read while its view is the live + one) against the last quiet point's: one still in the session (/index) and no + longer on its view was lost (a rejoin's new view, a view gone wrong).""" + layout = run.nine_read('/layout') or b'' + index = run.nine_read('/index') or b'' + exist = {int(r.split()[0]) for r in index.splitlines() if r.split() and r.split()[0].isdigit()} + shown = set() + for line in layout.decode(errors='replace').splitlines(): + w = line.split() + if len(w) > 6 and w[0].isdigit(): + shown |= {int(x) for x in w[6:] if x.isdigit()} + # Serials both (/layout's pane fields and /index's first are pane serials, + # never reused): a pane closed and another opened in its slot is no loss. + before = getattr(c, 'shown', None) + links_before = getattr(c, 'links_seen', 0) + c.shown = shown + c.links_seen = len(c.links()) + if before is None: + return + lost = (before - shown) & exist + if lost: + c.lost = getattr(c, 'lost', 0) + len(lost) + rows = [r.decode(errors='replace') for r in index.splitlines() if r.split() and r.split()[0].isdigit() and int(r.split()[0]) in lost] + why = 'after a rejoin (its view is new: the known gap)' if c.links_seen > links_before else 'with no rejoin since the last quiet point' + run.breach(f'{c.label}: panes {sorted(lost)} left its view while still open, {why}: {rows}') + + +def view_check(run, c): + """The terminal client's screen and cursor against what the session renders + for the same view (its input was the last, so 9P reads that view).""" + time.sleep(0.4) + raw = run.nine_read('/screen') + if not raw: + return + try: + s = json.loads(raw) + except ValueError: + return + lines, cursor, hidden = c.term.view() + cols, rows = s.get('cols'), s.get('rows') + if (cols, rows) != (c.term.cols, c.term.rows): + c.view_skips = getattr(c, 'view_skips', 0) + 1 + return + theirs = [] + cells = s.get('cells', []) + for r in range(rows): + row = cells[r * cols:(r + 1) * cols] if len(cells) == rows * cols else (cells[r] if r < len(cells) else []) + theirs.append(''.join(cell[0] if isinstance(cell, list) else str(cell) for cell in row).rstrip()) + sc = s.get('cursor') + differ = [r for r in range(min(rows, len(lines))) if lines[r] != theirs[r]] + c.views = getattr(c, 'views', 0) + 1 + want = None + if isinstance(sc, dict): + want = (sc.get('row', sc.get('y')), sc.get('col', sc.get('x'))) + elif isinstance(sc, list) and len(sc) >= 2: + want = (sc[1], sc[0]) + bad_cursor = want is not None and None not in want and tuple(want) != cursor and not hidden + if differ or bad_cursor: + c.view_bad = getattr(c, 'view_bad', 0) + 1 + d = c.dir / f'view-{c.view_bad}.txt' + d.write_text(f'cursor client {cursor} hidden {hidden} session {sc}\nrows differing {differ[:12]}\n' + + ''.join(f'client {r:2} |{lines[r]}|\nsession {r:2} |{theirs[r]}|\n' for r in differ[:12])) + run.breach(f'{c.label}: its screen is not the session\'s render of its view ({len(differ)} rows' + f'{", cursor " + str(cursor) + " vs " + str(want) if bad_cursor else ""}): {d}') + + +def global_event(run, rng, live, made): + """Something happens beside the typing: a resize, a client detached and + another attached, an Upgrade, a script's edit over 9P.""" + kinds = [e for e in run.args.events.split(',') if e] + if run.args.upgrade and not getattr(run, 'upgraded', False): + kinds.append('upgrade') + if not kinds: + return None + what = rng.choice(kinds) + if what == 'resize': + c = rng.choice(live) + rows, cols = rng.randint(18, 50), rng.randint(60, 160) + run.note(f'{c.label} resized to {cols}x{rows}') + c.resize(rows, cols) + elif what == 'detach': + c = rng.choice(live) + run.note(f'{c.label} detached') + hold(run, [c]) + c.asked_to_end = True + c.keys(ESC) + c.keys(b'\x02') + time.sleep(0.2) + c.keys(ESC) + c.keys(b' sD') + try: + wait_for(f'{c.label} let go', lambda: c.poll() is not None, timeout=15) + except TimeoutError: + # Its keys went to a program or a prompt: it is ended as a window closes. + run.note(f'{c.label} did not detach by keys: hung up') + c.hangup() + wait_for(f'{c.label} hung up', lambda: c.poll() is not None, timeout=15) + live.remove(c) + made[0] += 1 + n = start_client(run, c.kind, c.rtt, made[0]) + live.append(n) + return n + elif what == 'upgrade': + run.note('Upgrade') + run.upgraded = True + run.upgrade_times.append(now()) + run.nine_write('/ctl', b'Upgrade\n') + time.sleep(3) + if run.nine: + try: + run.nine.socket.close() + except Exception: + pass + run.nine = None + elif what == 'script': + rows = (run.nine_read('/index') or b'').splitlines() + texts = [r.split()[0].decode() for r in rows if len(r.split()) > 1 and r.split()[1] == b'text'] + if texts: + n = rng.choice(texts) + run.note(f'9P append to pane {n}') + run.nine_write(f'/pane/{n}/body', b'a line a script wrote\n') + return None + + +def soak(run, kinds, rtt, minutes, seed): + """The cell: its minutes of typing, the session started again (and every + client) whenever it dies, as its user would, so a cell measures to its end.""" + rng = random.Random(seed) + end = now() + minutes * 60 + try: + while True: + run.epochs += 1 + died = soak_body(run, kinds, rtt, end, rng) + if not died or now() > end - 30 or run.epochs >= 20 or not run.args.respawn: + break + run.note(f'the session started again (epoch {run.epochs + 1})') + run.reset_session() + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + report_cell(run, rtt) + run.cleanup() + + +def soak_body(run, kinds, rtt, end, rng): + """One session's life in the cell: True if it died before `end`.""" + stop = threading.Event() + live = [] + threads = [] + made = run.made + run.start_session() + for kind in kinds: + made[0] += 1 + live.append(start_client(run, kind, rtt, made[0])) + for c in list(live): + t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + next_quiet = now() + 45 + next_event = now() + rng.uniform(20, 60) + while now() < end: + time.sleep(0.5) + for c in list(live): + if c.poll() is not None: + run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') + live.remove(c) + c.ended_at = now() + if run.session_alive() and run.args.respawn: + # As its user would: open it again, and go on. + made[0] += 1 + try: + n = start_client(run, c.kind, c.rtt, made[0]) + except TimeoutError as e: + run.breach(f'{c.kind}: a client started again gave no first frame: {e}') + continue + live.append(n) + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + if not run.session_alive(): + run.breach(f'the session died (epoch {run.epochs}, {now() - run.t0:.0f} s into the cell)') + stop.set() + for c in live: + c.asked_to_end = True + for c in live: + try: + wait_for(f'{c.label}\'s end with its session', lambda: c.poll() is not None, timeout=10) + except TimeoutError: + run.breach(f'{c.label}: still running 10 s after its session died') + c.p.kill() + return True + if not live: + break + if now() > next_event: + n = global_event(run, rng, live, made) + if n: + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + next_event = now() + rng.uniform(20, 70) + if now() > next_quiet: + quiet_point(run, live) + next_quiet = now() + rng.uniform(40, 80) + stop.set() + for t in threads: + t.join(timeout=20) + if run.session_alive() and live: + quiet_point(run, live) + # Each client's links written: a hangup ends a client the normal way. + for c in live: + c.asked_to_end = True + if c.p.poll() is None: + c.hangup() + for c in live: + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + run.breach(f'{c.label}: would not end on a hangup') + c.p.kill() + if run.session_alive(): + run.end_session() + return False + + +def tail(c): + if isinstance(c, WinClient): + return c.said()[-300:].replace('\n', ' | ') + lines, _, _ = c.term.view() + return ' | '.join(line for line in lines if line.strip())[-300:] + + +ROWS = [] + + +def cores(run): + """systemd-coredump's cores since the cell began, of the pids it started.""" + if not shutil.which("coredumpctl"): + return [] + since = time.time() - (now() - run.t0) - 5 + got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True) + ours = set(run.pids) | {c.p.pid for c in run.clients if c.p} | ({run.session_pid} if run.session_pid else set()) + out = [] + for line in got.stdout.splitlines(): + w = line.split() + pid = next((int(x) for x in w if x.isdigit() and int(x) in ours), None) + if pid: + out.append(f'pid {pid} {" ".join(x for x in w if x.startswith("SIG"))} {w[-2] if len(w) > 1 else ""}') + return out + + +def report_cell(run, rtt): + for core in cores(run): + run.breach(f'core dumped: {core} (coredumpctl debug <pid>)') + said = run.session_said() + for word in ('panic', 'Segmentation', 'abort', 'reached unreachable', 'core dumped'): + if word in said: + run.breach(f'session said "{word}": {said[said.find(word) - 200:said.find(word) + 300]!r}') + break + if run.crashes(): + run.breach(f'crashes: {run.crashes()[-400:]}') + for c in run.clients: + st = c.stats_sum() + links = len(c.links()) + lines = c.check_lines() + differ = [line for line in lines if line.startswith('differ')] + diverged, upgraded = c.rejoins() + if diverged: + run.breach(f'{c.label}: rejoined {diverged} times after a divergence (links {links}, {upgraded} after an Upgrade)') + mins = ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60 + if mins >= 1 and st['visible'] / mins >= 1: + run.breach(f'{c.label}: {st["visible"]} visible rollbacks in {mins:.1f} min (want under 1 a minute)') + if differ: + run.breach(f'{c.label}: {len(differ)} checks differ') + if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): + run.breach(f'{c.label}: died of signal {-c.gone}') + if c.gone not in (None, 0) and not c.asked_to_end: + pass # said when found + elif c.gone not in (None, 0, 1, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: + run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') + joined = c.said() + for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): + if word in joined: + run.breach(f'{c.label} said "{word}": {joined[joined.find(word) - 200:joined.find(word) + 300]!r}') + break + if isinstance(c, WinClient): + lat, missed = c.latencies() + else: + lat, missed = c.lat, c.no_output + dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) + ROWS.append({ + 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged, + 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], + 'vis/min': st['visible'] / dur, 'spec/in': f"{st['at_once']}/{st['inputs']}", 'echo': f"{st['echoed']}/{st['typed']}", 'echo_missed': st['missed'], 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), + 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs, + }) + st = (run.root / 'session.stats') + text = '' + if isinstance(run, BoxRun): + text = run.box(f'cat {run.home}/session.stats 2>/dev/null', timeout=30).stdout.decode() + elif st.exists(): + text = st.read_text() + for line in text.splitlines(): + if line.startswith('session inputs'): + w = line.split() + d = {w[i]: w[i + 1] for i in range(1, len(w) - 1, 2)} + worst = int(d.get('boundary_us', '0/0/0').split('/')[2]) + ROWS.append({'cell': run.cell, 'client': 'session', 'rtt': rtt, 'boundary_us': d.get('boundary_us'), + 'input_us': d.get('input_us'), 'cut': d.get('cut')}) + if worst > 250_000: + run.breach(f'session: a boundary took {worst / 1000:.0f} ms') + run.note('end of cell') + + +def table(rows): + cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'spec/in', 'visible', 'mispred', 'vis/min', 'echo', 'echo_missed', 'clears/min', + 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs'] + def fmt(v): + if v is None: + return '-' + if isinstance(v, float): + return f'{v:.1f}' + return str(v) + out = [] + widths = {c: len(c) for c in cols} + body = [] + for r in rows: + if r['client'] == 'session': + body.append(f"{r['cell']:<8} session boundary_us p50/p99/max {r['boundary_us']} input_us {r['input_us']} cut {r['cut']}") + continue + vals = [fmt(r.get(c)) for c in cols] + for c, v in zip(cols, vals): + widths[c] = max(widths[c], len(v)) + body.append(vals) + out.append(' '.join(c.ljust(widths[c]) for c in cols)) + for b in body: + out.append(b if isinstance(b, str) else ' '.join(v.ljust(widths[c]) for c, v in zip(cols, b))) + return '\n'.join(out) + + +# ---- instant: does each kind of local action feel local? ---- +# +# The user: "make sure to use the mosh tricks so that user interaction to +# type/scroll/move the cursor feels instant specially when they don't require +# a request going to the daemon". Per action class and client kind: the share +# of its inputs applied at once (not one of them waiting a round trip), its +# keystroke to present p50/p99 against a local session's measured in the same +# run, and for typing at a shell prompt the share of keys whose echo was +# predicted. A class runs in a link of its own (a client started for it), and +# again with only its setup, so its stats line less the setup's is the class's. + +INSTANT_N = 25 + + +def gen_letters(c, rng, i): + c.keys(bytes([97 + rng.randrange(26)])) + + +def gen_moves(c, rng, i): + c.keys([b'j', b'k', b'w', b'b', b'l', b'h', b'e', b'j'][i % 8]) + + +def gen_scroll_keys(c, rng, i): + c.keys(b'\x04' if i % 2 == 0 else b'\x15') + + +def gen_select(c, rng, i): + c.keys(b'x' if i % 3 else b'w') + + +def gen_term_keys(c, rng, i): + c.keys(b'k' if i % 4 else b'\x15') + + +def gen_click(c, rng, i): + c.click(1, rng.uniform(0.15, 0.9), rng.uniform(0.14, 0.4)) + + +def gen_wheel_file(c, rng, i): + c.mouse('wheel', 1 if i % 3 == 2 else -1, 0.5, 0.3) + + +def gen_wheel_term(c, rng, i): + c.mouse('wheel', 1 if i % 3 != 2 else -1, 0.5, 0.85) + + +# name, pane it is in, setup keys, one measured input, keys after, local only +CLASSES = [ + ('type_shell', 'term', b'', gen_letters, b'\x15', False), + ('type_file', 'file', b'i', gen_letters, ESC, True), + ('move_keys', 'file', b'', gen_moves, b'', True), + ('move_click', 'file', b'', gen_click, b'', True), + ('scroll_keys', 'file', b'', gen_scroll_keys, b'', True), + ('scroll_wheel', 'file', b'', gen_wheel_file, b'', True), + ('term_scroll_keys', 'term', b'\x02', gen_term_keys, b'\x02', True), + ('term_scroll_wheel', 'term', b'', gen_wheel_term, b'', True), + ('select', 'file', b'', gen_select, b';', True), + ('picker', 'file', b' bb', gen_letters, ESC, True), +] + + +def instant_panes(run): + rows = (run.nine_read('/index') or b'').decode().splitlines() + big = next(r.split()[0] for r in rows if r.split()[1] == 'text' and 'big.txt' in r) + term = next(r.split()[0] for r in rows if r.split()[1] == 'term') + return big, term + + +def instant_setup(run, where): + """The pane it starts in: big.txt at line 4000 (normal mode), or the shell + at its prompt with 3000 lines of scrollback above. 9P acts in the view + that typed last: the client's, its join's inputs.""" + big, term = run.instant_panes + run.nine_write('/focus', (big if where == 'file' else term).encode()) + if where == 'file': + run.nine_write(f'/pane/{big}/dot', b'4000') + time.sleep(0.4) + + +def present_ms(c, t0, t1): + """Each measured input's keystroke to present, ms, and how many showed + nothing within 2 s.""" + if isinstance(c, TtyClient): + with c.lock: + return list(c.lat), c.no_output + try: + lines = c.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads = [int(w[1]) for w in (line.split() for line in lines) if len(w) >= 2 and w[0] == 'R' and t0 * 1e9 <= int(w[1]) <= t1 * 1e9] + frames = sorted(int(w[2]) for w in (line.split() for line in lines) if len(w) >= 3 and w[0] == 'F') + out, missed = [], 0 + for r in reads: + f = next((x for x in frames if x >= r), None) + if f is not None and f - r <= 2e9: + out.append((f - r) / 1e6) + else: + missed += 1 + return out, missed + + +def instant_class(run, c, cls): + """One class in client `c` (or the local session's frontend): its setup, + its N inputs a frame apart or more, its keys after. Its present latencies, + the inputs that showed nothing in 2 s, and the replica's own count of the + class's events that stopped at a host call (its trace: PARDES_REPLICA_TRACE).""" + name, where, setup, gen, after, _ = cls + rng = random.Random(sum(name.encode())) + instant_setup(run, where) + if setup: + c.keys(setup) + time.sleep(0.8) + if isinstance(c, TtyClient): + with c.lock: + c.lat.clear() + c.no_output = 0 + said = len(c.said()) + t0 = time.monotonic() + for i in range(INSTANT_N): + gen(c, rng, i) + time.sleep(0.16 + rng.uniform(0, 0.06)) + t1 = time.monotonic() + time.sleep(2.2) + lat, missed = present_ms(c, t0, t1) + stops = c.said()[said:].count('stopped at a host call') + if after: + c.keys(after) + time.sleep(0.5) + return lat, missed, stops + + +def local_session(run, kind): + """A session of this machine's own, the regular path (not detached, no + replica): the baseline, in a terminal or a window, on big.txt and a shell.""" + name = f'local-{kind}' + e = dict(run.env) + big = run.work / 'big.txt' + if kind == 'tty': + c = TtyClient(run, name, None) + c.p = c.term.spawn([run.pardes, f'--9p={name}', str(big)], e, run.work, open(c.err, 'w')) + else: + c = WinClient(run, name, None) + c.p = subprocess.Popen([run.gui, f'--9p={name}', str(big)], env=dict(e, PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(c.cap), + PARDES_TEST_LATENCY=str(c.latency), SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(c.cols), + PARDES_TEST_ROWS=str(c.rows)), cwd=run.work, stdin=c.slave, stdout=subprocess.DEVNULL, + stderr=open(c.err, 'w'), start_new_session=True) + c.started = now() + run.pids.append(c.p.pid) + run.clients.append(c) + run.nine_sock = run.root / f'pardes-9p-{name}.sock' + run.nine = None + wait_for(f'{name}\'s 9P', run.nine_sock.exists, timeout=30) + wait_for(f'{name}\'s first frame', c.ready, timeout=30) + run.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + run.instant_panes = instant_panes(run) + run.nine_write(f'/pane/{run.instant_panes[1]}/body', b'seq 1 3000\n') + time.sleep(1.5) + return c + + +def instant(args): + kinds = args.clients.split(',') + cells = ['box'] if args.where == 'box' else [int(x) for x in args.rtts.split(',')] + results = [] + runs = [] + base = {} + # The baseline: each class in a local session's own frontend. + run = Run(args, 'instant-local') + runs.append(run) + fixture(run.work, random.Random(args.seed)) + try: + for kind in kinds: + c = local_session(run, kind) + for cls in CLASSES: + lat, missed, _ = instant_class(run, c, cls) + base[(kind, cls[0])] = lat + c.hangup() + time.sleep(1) + if c.p.poll() is None: + c.p.kill() + finally: + run.cleanup() + for cell in cells: + run = BoxRun(args, 'instant-box') if cell == 'box' else Run(args, f'instant{cell}') + rtt = 207 if cell == 'box' else cell + runs.append(run) + run.opens = 'big.txt' + run.shells = 1 + # The replica's own words for each input that waited (the client's + # trace alone; the session keeps its one-word digests). + run.env['PARDES_REPLICA_TRACE'] = '1' + try: + run.start_session() + run.instant_panes = instant_panes(run) + run.nine_write(f'/pane/{run.instant_panes[1]}/body', b'seq 1 3000\n') + time.sleep(2) + for kind in kinds: + c = None + for cls in CLASSES: + if not run.session_alive(): + run.breach(f'{kind} {cls[0]}: the session died') + break + if c is None or c.poll() is not None: + run.made[0] += 1 + c = start_client(run, kind, rtt, run.made[0]) + time.sleep(0.6) + lat, missed, stops = instant_class(run, c, cls) + echo = None + if cls[0] == 'type_shell' and c.poll() is None: + # Its link alone: its echo counts are this class's. + c.asked_to_end = True + c.hangup() + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + c.p.kill() + st = c.stats_sum() + echo = (st['echoed'], st['typed']) + c = None + died = c is not None and c.poll() is not None + if died: + run.breach(f'{kind} {cls[0]}: the client died ({c.gone}): {tail(c)[-200:]}') + events = INSTANT_N * (2 if cls[0] == 'move_click' else 1) + # At once: no stop, and shown sooner than a round trip could + # bring it (an input queued behind a stopped one says nothing). + # (Half a round trip, or two frames: a window's software present + # alone takes up to a frame or so.) + quick = sum(1 for x in lat if x < max(rtt / 2, 2000 / 60)) if rtt else len(lat) + frac = min(1 - stops / events, quick / INSTANT_N) + results.append((run.cell, kind, cls, frac, stops, quick, pct(lat, 50), pct(lat, 99), missed, + pct(base.get((kind, cls[0]), []), 99), died, echo)) + if c is not None and c.poll() is None: + c.asked_to_end = True + c.hangup() + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + c.p.kill() + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + run.cleanup() + frame = 1000 / 60 + print(f'{"cell":12} {"kind":4} {"class":18} {"at once":>8} {"stops":>5} {"quick":>5} {"p50 ms":>7} {"p99 ms":>7} {"local p99":>9} {"none":>4} {"echo pred":>9}') + for r in results: + cell, kind, cls, frac, stops, quick, p50, p99, missed, lp99, died, echo = r + name, local_only = cls[0], cls[5] + fmt = lambda v: '-' if v is None else f'{v:.1f}' + ep = f'{echo[0]}/{echo[1]}' if echo else '' + print(f'{cell:12} {kind:4} {name:18} {100 * frac:7.0f}% {stops:>5} {quick:>5} {fmt(p50):>7} {fmt(p99):>7} {fmt(lp99):>9} {missed:>4} {ep:>9}') + run = next(x for x in runs if x.cell == cell) + if local_only and frac < 0.99: + run.breach(f'{kind} {name}: {100 * frac:.0f}% applied at once ({stops} stopped, {quick} of {INSTANT_N} shown within half a round trip; want >= 99%)') + if local_only and p99 is not None and lp99 is not None and p99 > lp99 + 2 * frame: + run.breach(f'{kind} {name}: present p99 {p99:.1f} ms against a local session\'s {lp99:.1f} (want within 2 frames)') + if missed > 1: + run.breach(f'{kind} {name}: {missed} of {INSTANT_N} inputs showed nothing within 2 s') + if name == 'type_shell': + if not echo or echo[1] == 0: + run.breach(f'{kind} {name}: no key\'s echo predicted or counted') + elif echo[0] / INSTANT_N < 0.95: + # Of the class's letters (the link's count also has its Ctrl-U, + # which is never predicted). + run.breach(f'{kind} {name}: {echo[0]} of its {INSTANT_N} letters\' echo predicted (want >= 95%)') + breaches = [(x.cell, b) for x in runs for b in x.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for x in runs: + if args.keep or x.breaches: + print(f'kept: {x.root}') + else: + shutil.rmtree(x.root, ignore_errors=True) + print('instant:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('where', choices=['local', 'box', 'instant']) + ap.add_argument('--box', action='store_true', help='instant: the session on the box (else local, at each --rtts)') + ap.add_argument('--minutes', type=float, default=None) + ap.add_argument('--rtts', default='0,30,80,207') + ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') + ap.add_argument('--seed', type=int, default=1) + ap.add_argument('--clients', default='tty,win', help='kinds, one client each: tty,win,win...') + ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) + ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) + ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') + ap.add_argument('--no-respawn', dest='respawn', action='store_false', help='a client that dies stays dead') + ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') + ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') + ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') + ap.add_argument('--keep', action='store_true', help='keep the run\'s files (and the terminals\' output)') + ap.add_argument('--only', default='', help='the monkey\'s actions, by name: move,insert,...') + ap.add_argument('--skip', default='', help='actions left out, by name') + ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') + args = ap.parse_args() + if args.where == 'instant': + args.where = 'box' if args.box else 'local' + return instant(args) + kinds = args.clients.split(',') + runs = [] + if args.where == 'local': + minutes = args.minutes if args.minutes is not None else 2 + for i, rtt in enumerate(int(x) for x in args.rtts.split(',')): + run = Run(args, f'local{rtt}') + run.far = not args.near + runs.append(run) + print(f'soak: {run.cell}: {kinds} for {minutes} min, seed {args.seed + i}, in {run.root}', flush=True) + soak(run, kinds, rtt, minutes, args.seed + i) + else: + minutes = args.minutes if args.minutes is not None else 20 + run = BoxRun(args, 'box') + runs.append(run) + print(f'soak: box: {kinds} for {minutes} min, seed {args.seed}, in {run.root}', flush=True) + soak(run, kinds, 207, minutes, args.seed) + print(table(ROWS)) + breaches = [(r.cell, b) for r in runs for b in r.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for r in runs: + if args.keep or r.breaches: + print(f'kept: {r.root}') + else: + shutil.rmtree(r.root, ignore_errors=True) + print('soak:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + +if __name__ == '__main__': + main() |
