diff options
| author | Gabriel Schneider <[email protected]> | 2026-10-30 14:44:53 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-10-30 19:37:50 -0300 |
| commit | 484baaf3d237c91a6ba5f7e2016c988e8aee00d3 (patch) | |
| tree | 4a8b6a3301b482bbf44da4ce7bd1623b5e117f71 /test/soak.py | |
| parent | 9f84ac9f0d715d7717dc1d732299bdc586422392 (diff) | |
| download | pardes-484baaf3d237c91a6ba5f7e2016c988e8aee00d3.tar.gz pardes-484baaf3d237c91a6ba5f7e2016c988e8aee00d3.zip | |
test/soak.py: a soak of replica clients, what a person does in them through each link, measured: a session on a directory and two shells (here, or on the FreeBSD box over ssh), terminal clients in a pty kept by pyte and offscreen window clients through their test feed, each a seeded monkey of moves, edits, undo, saves, Looks, pickers, shells with programs, panes and columns, clicks, the wheel, the pointer resting, pastes and idle gaps, beside resizes, detaches and attaches, an Upgrade and a script's 9P edits; its table a client (rejoins, checks that differ, exits and cores, freezes at quiet points, a terminal client's screen and cursor against the session's render of its view, visible rollbacks, keystroke to output or frame p50/p99) and the session's worst boundary, exit 1 on any breach; test/delay_relay.py takes --jitter (order kept)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Diffstat (limited to 'test/soak.py')
| -rw-r--r-- | test/soak.py | 1318 |
1 files changed, 1318 insertions, 0 deletions
diff --git a/test/soak.py b/test/soak.py new file mode 100644 index 00000000..f75371ef --- /dev/null +++ b/test/soak.py @@ -0,0 +1,1318 @@ +#!/usr/bin/env python3 +"""A soak of replica clients: what a person does in them, through each link, +measured (docs/next-daemon.md, the replica and GUI replica notes). + + python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10] + [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] + python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] + [--pardes P] [--gui G] [--seed 1] [--trace] + +`local`: a session on this machine (`pardes --detach=soak <a directory>`, the +shape of the user's: a directory pane, then two shells), and at each RTT its +clients through test/delay_relay.py (with jitter), far as through ssh +(PARDES_REPLICA_FAR=1: the blobs by hash, the stream in zstd). One cell an +RTT, `--minutes` each. `box`: the session on the FreeBSD box under +/root/pardes-test (`--remote-bin`, by default bin/pardes-hunt there, run in +run-hunt and home-hunt), the clients here through ssh, as the user's window +was; its 9P through an `ssh -L` of the session's socket. + +The clients: terminals (`pardes --attach`, in a pty here whose screen pyte +keeps, frame requests answered) and windows (`pardes-gui --attach`, SDL's +offscreen driver, keys and the mouse through its test feed on a pty, so it +can be resized). Each runs a seeded monkey of a person's actions: moves of +every kind (keys, clicks, jumps, search), scrolls (keys, the wheel), edits, +undo and save, Look at files and directories, the Panes picker, Find, new +terminals with programs (ls, a long output, less, vi, top, Ctrl-C), panes and +columns opened, moved and closed, the pointer resting and moving, pastes, +idle gaps. Between them, at random: a client resized, a client detached and +another attached, an Upgrade, a 9P script's edit. `--seed` makes a run +again (the timing differs, the actions do not). + +Measured (a row a client, then the session's): + - rejoins: links past the first (PARDES_REPLICA_STATS writes one a link) + and `joined again` said by it; checks (PARDES_REPLICA_CHECK) that differ; + - exits nobody asked for, a signal or a non-zero code, the session's death, + a panic or `crashes` line; + - freezes: a quiet point where a client gives no fresh check within 8 s, + and a terminal client's key with no output within 2 s while it was not + held; the session's worst boundary (its stats at its end); + - the view: at each quiet point, a terminal client's cursor and screen + (pyte) against what the session renders for its view (/screen read right + after its own input made its view the live one), and the panes on it; + - blink: visible rollbacks (`visible`), mispredicted inputs, a terminal + client's full clears, a minute; + - keystroke to output (terminal) and to a frame (window, PARDES_TEST_LATENCY) + p50 / p99. + +Prints a table, then each breach, and exits 1 if there is any. Everything +started is ended by pid (and found by its environment); the run's files are +kept under $TMPDIR/soak-* (`--keep`) or removed. +""" +import argparse +import fcntl +import json +import os +import random +import re +import shutil +import signal +import socket +import statistics +import struct +import subprocess +import sys +import tempfile +import termios +import threading +import time +import tty as ttymod +from pathlib import Path + +import pyte + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from ninep import Client # noqa: E402 +from daemon import processes, alive, wait_for, checks # noqa: E402 + +BOX = '[email protected]' +SSH = ['/usr/bin/ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', + '-o', 'UserKnownHostsFile=/home/goblin/.ssh/known_hosts', '-i', '/home/goblin/.ssh/id_ed25519'] +BOX_T = '/root/pardes-test' +NAME = 'soak' +ESC = b'\x1b' + + +def now(): + return time.monotonic() + + +def pct(xs, p): + if not xs: + return None + xs = sorted(xs) + return xs[min(len(xs) - 1, len(xs) * p // 100)] + + +# ---- the fixture: a directory such as a person works in ---- + +WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa ' + 'quebec romeo sierra tango uniform victor whiskey xray yankee zulu').split() + + +def fixture(at, rng): + at.mkdir(parents=True, exist_ok=True) + (at / 'notes.txt').write_text(''.join(f'line {i} of the notes: {rng.choice(WORDS)} {rng.choice(WORDS)} see code.zig:{i % 90 + 1}\n' + for i in range(300))) + (at / 'code.zig').write_text(''.join(f'pub fn f{i}(x: u32) u32 {{\n return x * {i} + {rng.randrange(1000)}; // {rng.choice(WORDS)}\n}}\n\n' + for i in range(120))) + (at / 'big.txt').write_text(''.join(f'{i:06d} {" ".join(rng.choice(WORDS) for _ in range(9))}\n' for i in range(8000))) + (at / 'README').write_text('A directory to soak in.\nnotes.txt code.zig big.txt sub/\nsub/inner.txt:3\n') + sub = at / 'sub' + sub.mkdir(exist_ok=True) + (sub / 'inner.txt').write_text(''.join(f'inner {i} {rng.choice(WORDS)}\n' for i in range(60))) + (sub / 'more.md').write_text('# More\n\n' + ''.join(f'- item {i} {rng.choice(WORDS)}\n' for i in range(40))) + + +# ---- terminals ---- + +class Term: + """A terminal: a pty whose output is kept, timed and fed to pyte, its + frame requests (CSI 5 n) answered as a terminal emulator does. An Esc is + written alone and nothing follows it for 80 ms, an answer included: the + client's parser takes an Esc and what follows within 50 ms as Alt.""" + + def __init__(self, rows, cols): + self.master, self.slave = os.openpty() + self.rows, self.cols = rows, cols + fcntl.ioctl(self.slave, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.lock = threading.Lock() + self.wlock = threading.Lock() + self.screen = pyte.Screen(cols, rows) + self.stream = pyte.ByteStream(self.screen) + self.out_bytes = 0 + self.last_out = 0.0 + self.chunks = [] # (time, bytes) of the last minute or so, for latencies + self.clears = 0 + self.joined_again = 0 + self.tail = b'' + self.esc_until = 0.0 + self.closing = False + self.log = None + self.reader = threading.Thread(target=self.drain, daemon=True) + self.reader.start() + + def drain(self): + import select + while not self.closing: + try: + if not select.select([self.master], [], [], 0.05)[0]: + continue + got = os.read(self.master, 65536) + except OSError: + return + if not got: + return + t = now() + carry, self.tail = self.tail, (self.tail + got)[-11:] + seen = carry[-3:] + got + with self.lock: + self.out_bytes += len(got) + self.last_out = t + self.chunks.append((t, len(got))) + if len(self.chunks) > 4096: + del self.chunks[:2048] + # The carry is shorter than either word: a match holds a new byte. + self.clears += seen.count(b'\x1b[2J') + self.joined_again += (carry + got).count(b'joined again') + self.feed(got) + if self.log: + self.log.write(got) + if b'\x1b[5n' in seen: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + with self.wlock: + try: + os.write(self.master, b'\x1b[0n') + except OSError: + return + + SGR = re.compile(rb'\x1b\[[0-9:;]*m') + + def feed(self, got): + """To pyte, without its colours (pyte reads no colon sub-parameters) + and holding back a sequence a read cut in two.""" + data = getattr(self, 'held_back', b'') + got + cut = data.rfind(b'\x1b') + self.held_back = b'' + if cut >= 0 and not re.match(rb'\x1b(\[[0-9:;?<>=! ]*[@-~]|[^\[\]]|\][^\x07\x1b]*(\x07|\x1b\\))', data[cut:]): + data, self.held_back = data[:cut], data[cut:] + try: + self.stream.feed(self.SGR.sub(b'', data)) + except Exception: + pass + + def write(self, data): + """Keys as typed: each Esc alone, 80 ms clear after it.""" + parts = re.split(b'(\x1b(?![\\[O\\]n0-9a-z]))', data) + for part in parts: + if not part: + continue + if part == ESC: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, ESC) + self.esc_until = now() + 0.08 + time.sleep(0.08) + else: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, part) + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + with self.lock: + self.screen.resize(rows, cols) + + def view(self): + with self.lock: + return [line.rstrip() for line in self.screen.display], (self.screen.cursor.y, self.screen.cursor.x), self.screen.cursor.hidden + + def spawn(self, argv, env, cwd, err): + slave = self.slave + return subprocess.Popen(argv, env=env, cwd=cwd, stdin=slave, stdout=slave, stderr=err, start_new_session=True, + preexec_fn=lambda: fcntl.ioctl(0, termios.TIOCSCTTY, 0)) + + def close(self): + self.closing = True + self.reader.join(timeout=2) + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- clients ---- + +class Base: + def __init__(self, run, kind, label, link): + self.run = run + self.kind = kind + self.label = label + self.link = link # where it dials: a socket path, or the box + self.dir = run.root / label + self.dir.mkdir() + self.check = self.dir / 'check' + self.stats = self.dir / 'stats' + self.err = self.dir / 'err' + self.p = None + self.started = None + self.asked_to_end = False + self.gone = None # how it ended, if it did + self.inputs = 0 + self.lat = [] # ms, input to output/frame + self.no_output = 0 + self.held = threading.Event() # the monkey waits while set + self.paused = threading.Event() # set by the monkey once it waits + self.lock = threading.Lock() + + def env(self): + e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) + if self.run.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + if self.run.far: + e['PARDES_REPLICA_FAR'] = '1' + return e + + def attach_arg(self): + return f'--attach={self.link}:{NAME}' + + def links(self): + return [line for line in checks(self.stats) if line.startswith('inputs ')] + + def stats_sum(self): + keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible') + tot = dict.fromkeys(keys, 0) + for line in self.links(): + w = line.split() + d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)} + for k in keys: + try: + tot[k] += int(d.get(k, 0)) + except ValueError: + pass + return tot + + def check_lines(self): + return checks(self.check) + + def poll(self): + if self.gone is None and self.p is not None and self.p.poll() is not None: + self.gone = self.p.returncode + return self.gone + + +class TtyClient(Base): + def __init__(self, run, label, link, rows=30, cols=100): + super().__init__(run, 'tty', label, link) + self.term = Term(rows, cols) + self.term.log = open(self.dir / 'out', 'wb', buffering=0) if run.args.keep else None + + def start(self): + argv = [self.run.pardes, self.attach_arg()] + # Its stderr a file: a panic's words and the replica's trace. + self.p = self.term.spawn(argv, self.env(), self.run.root, open(self.err, 'w')) + self.started = now() + + def ready(self): + return self.term.out_bytes > 2000 and now() - self.term.last_out > 0.3 + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + before = self.term.out_bytes + t = now() + self.term.write(data) + threading.Thread(target=self._latency, args=(t, before), daemon=True).start() + + def _latency(self, t, before): + deadline = t + 2.0 + while now() < deadline: + with self.term.lock: + got = [c for c in self.term.chunks if c[0] >= t] + if got: + with self.lock: + self.lat.append((got[0][0] - t) * 1000) + return + time.sleep(0.005) + if not self.held.is_set(): + with self.lock: + self.no_output += 1 + + def mouse(self, kind, button, x, y): + # SGR: cells from 1; x, y here are fractions of the screen. + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + b = {'down': button - 1, 'up': button - 1, 'motion': 35, 'wheel': 64 if button > 0 else 65}[kind] + end = 'm' if kind == 'up' else 'M' + self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) + + def resize(self, rows, cols): + self.term.resize(rows, cols) + if self.p: + try: + os.killpg(self.p.pid, signal.SIGWINCH) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.term.joined_again + + def hangup(self): + """As its terminal closing: the normal way out (its stats written).""" + try: + os.killpg(self.p.pid, signal.SIGHUP) + except OSError: + pass + + def close(self): + self.term.close() + + +class WinClient(Base): + """A window client in test mode: offscreen, its keys and pointer through + its test feed on a pty (Escape as CSI 27 u: a lone ESC waits for more).""" + + def __init__(self, run, label, link, rows=34, cols=110): + super().__init__(run, 'win', label, link) + self.cap = self.dir / 'cap' + self.cap.mkdir() + self.latency = self.dir / 'latency' + self.master, self.slave = os.openpty() + ttymod.setraw(self.slave) + self.rows, self.cols = rows, cols + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.wlock = threading.Lock() + self.px = (cols * 16, rows * 33) # test mode's 27 px font, about + + def start(self): + e = dict(self.env(), PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(self.cap), PARDES_TEST_LATENCY=str(self.latency), + SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(self.cols), PARDES_TEST_ROWS=str(self.rows)) + self.p = subprocess.Popen([self.run.gui, self.attach_arg()], env=e, cwd=self.run.root, stdin=self.slave, + stdout=subprocess.DEVNULL, stderr=open(self.err, 'w'), start_new_session=True) + self.started = now() + + def ready(self): + # Software presenting (offscreen) captures no ppm: its first frame is + # its latency trace's first F. + try: + return b'F ' in self.latency.read_bytes() and self.p.poll() is None + except FileNotFoundError: + return False + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + with self.wlock: + try: + os.write(self.master, data.replace(ESC + b'[', b'\x00CSI').replace(ESC, b'\x1b[27u').replace(b'\x00CSI', ESC + b'[')) + except OSError: + pass + + def mouse(self, kind, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + if kind == 'wheel': + seq = f'\x1b]777;mouse;wheel;{button};{px};{py}\x07' + else: + seq = f'\x1b]777;mouse;{kind};{button};{px};{py}\x07' + self.inputs += 1 + self.run.note(f'{self.label} < mouse {kind} {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, seq.encode()) + except OSError: + pass + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + + def hangup(self): + """Its test feed's end: the normal way out (its stats written).""" + try: + os.close(self.master) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.said().count('joined again') + + def frame_size(self): + try: + with open(self.cap / 'latest.ppm', 'rb') as f: + head = f.read(32).split() + self.px = (int(head[1]), int(head[2])) + except (OSError, IndexError, ValueError): + pass + + def latencies(self): + """Each stdin read to the end of the first frame after it, ms; and the + reads no frame followed within 2 s (a frame came after a later one).""" + try: + lines = self.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads, frames = [], [] + for line in lines: + w = line.split() + if len(w) >= 2 and w[0] == 'R': + reads.append(int(w[1])) + elif len(w) >= 3 and w[0] == 'F': + frames.append(int(w[2])) + out, missed, j = [], 0, 0 + for r in reads: + while j < len(frames) and frames[j] < r: + j += 1 + if j < len(frames): + d = (frames[j] - r) / 1e6 + if d <= 2000: + out.append(d) + else: + missed += 1 + return out, missed + + def close(self): + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- what a person does ---- + +def words(rng, n): + return ' '.join(rng.choice(WORDS) for _ in range(n)).encode() + + +def burst(c, rng): + """One action of a person's, in whatever state the client is.""" + k = c.keys + acts = c.run.actions + choice = rng.choices(acts, weights=[a[1] for a in acts])[0] + choice[2](c, rng, k) + return choice[0] + + +def act_move(c, rng, k): + for _ in range(rng.randint(1, 6)): + k(rng.choice([b'j', b'k', b'h', b'l', b'w', b'b', b'e', b'5j', b'3k', b'gg', b'ge', b'\x04', b'\x15', + b'\x06', b'\x02', b'}', b'{', b'%', b'x', b'12G', b'gl', b'gh'])) + time.sleep(rng.uniform(0.02, 0.2)) + + +def act_search(c, rng, k): + k(b'/' + rng.choice(WORDS).encode() + b'\r') + for _ in range(rng.randint(0, 3)): + time.sleep(rng.uniform(0.1, 0.4)) + k(rng.choice([b'n', b'N'])) + + +def act_jump(c, rng, k): + k(rng.choice([b'\x0f', b' jo', b' jj', b'gd', b'\x0f\x0f'])) + + +def act_insert(c, rng, k): + k(rng.choice([b'i', b'a', b'o', b'O', b'A'])) + for w in words(rng, rng.randint(1, 5)).split(): + k(w + b' ') + time.sleep(rng.uniform(0.03, 0.15)) + k(ESC) + + +def act_delete(c, rng, k): + k(rng.choice([b'xd', b'd', b'wd', b'xxd', b'c' + words(rng, 1) + ESC])) + + +def act_undo(c, rng, k): + k(rng.choice([b'u', b'uu', b'U', b'u'])) + + +def act_save(c, rng, k): + k(b' fs') + + +def act_look(c, rng, k): + # Enter is Look: a directory's row opens it, a word finds its next place. + k(rng.choice([b'\r', b'j\r', b'jj\r', b'gg\r'])) + + +def act_panes(c, rng, k): + k(b' bb') + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'\r', ESC, b'jj\r', b'not\r'])) + + +def act_find(c, rng, k): + k(rng.choice([b' ff', b' fr', b' fg'])) + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'notes\r', ESC, b'j\r', b'inner\r'])) + + +def act_shell(c, rng, k): + # A terminal of its own, a program in it, and back out to normal mode. + k(b' nt') + time.sleep(rng.uniform(0.8, 1.5)) + for _ in range(rng.randint(1, 3)): + prog = rng.choice(['ls -la\r', 'seq 1 3000\r', 'less big.txt\r', 'vi notes.txt\r', 'top\r', 'sleep 30\r', + 'echo ' + rng.choice(WORDS) + '\r', 'cat code.zig\r']) + k(prog.encode()) + time.sleep(rng.uniform(0.5, 2.0)) + if prog.startswith('less'): + # Never SPC then q: that is Exit, should the keys reach no less. + k(b'jjjj\x06') + time.sleep(0.3) + k(b'q') + elif prog.startswith('vi'): + k(b'jjjjx') + k(ESC) + k(b':q!\r') + elif prog.startswith(('top', 'sleep', 'seq')): + k(b'\x03' if not prog.startswith('top') else b'q') + time.sleep(rng.uniform(0.2, 0.8)) + k(b'\x02') # Ctrl-b: normal mode + if rng.random() < 0.5 and c.run.pane_count() >= 4: + time.sleep(0.3) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b''])) + + +def act_window(c, rng, k): + k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l'])) + + +def act_column(c, rng, k): + few = c.run.pane_count() < 5 + k(rng.choice([b' cn', b' cn'] if few else [b' cn', b' cn', b' cd', b' cj'])) + + +def act_del(c, rng, k): + if c.run.pane_count() < 4: + return act_move(c, rng, k) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b'', b' d'])) + + +def act_new(c, rng, k): + k(b' fn') + time.sleep(0.4) + k(b'i' + words(rng, 4)) + k(ESC) + + +def act_esc(c, rng, k): + k(rng.choice([ESC, b'\x02', b':', b':'])) + time.sleep(0.3) + k(ESC) + + +def act_click(c, rng, k): + x, y = rng.random(), rng.uniform(0.05, 1.0) + b = rng.choice([1, 1, 1, 3]) + c.mouse('down', b, x, y) + time.sleep(rng.uniform(0.03, 0.12)) + if rng.random() < 0.3: # a drag + c.mouse('motion', b, min(1, x + rng.uniform(0, 0.2)), min(1, y + rng.uniform(0, 0.1))) + c.mouse('up', b, x, y) + + +def act_wheel(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + d = rng.choice([1, -1]) + for _ in range(rng.randint(1, 8)): + c.mouse('wheel', d, x, y) + time.sleep(rng.uniform(0.01, 0.08)) + + +def act_hover(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + for _ in range(rng.randint(2, 12)): + x = min(1, max(0, x + rng.uniform(-0.05, 0.05))) + y = min(1, max(0.05, y + rng.uniform(-0.03, 0.03))) + c.mouse('motion', 0, x, y) + time.sleep(rng.uniform(0.01, 0.06)) + time.sleep(rng.uniform(0.05, 0.6)) # resting: a Look preview + + +def act_paste(c, rng, k): + k(b'i\x1b[200~' + b'\n'.join(words(rng, 6) for _ in range(rng.randint(1, 4))) + b'\x1b[201~') + k(ESC) + + +def act_settings(c, rng, k): + k(rng.choice([b' tw', b' ?', b' hm', b' tt'])) + time.sleep(0.5) + k(ESC) + + +def act_idle(c, rng, k): + time.sleep(rng.choice([0.5, 1, 2, 4, 8])) + + +ACTIONS = [ + ('move', 30, act_move), ('search', 5, act_search), ('jump', 4, act_jump), ('insert', 12, act_insert), + ('delete', 4, act_delete), ('undo', 5, act_undo), ('save', 3, act_save), ('look', 6, act_look), + ('panes', 3, act_panes), ('find', 3, act_find), ('shell', 3, act_shell), ('window', 4, act_window), + ('column', 1, act_column), ('del', 2, act_del), ('new', 2, act_new), ('esc', 4, act_esc), + ('click', 8, act_click), ('wheel', 8, act_wheel), ('hover', 6, act_hover), ('paste', 2, act_paste), + ('settings', 1, act_settings), ('idle', 6, act_idle), +] + + +def monkey(c, rng, stop): + while not stop.is_set(): + if c.held.is_set(): + c.paused.set() + time.sleep(0.05) + continue + c.paused.clear() + if c.poll() is not None: + return + try: + what = burst(c, rng) + c.run.note(f'{c.label} {what}') + except OSError: + return + time.sleep(rng.uniform(0.05, 0.6)) + + +# ---- the run ---- + +class Run: + def __init__(self, args, cell): + self.args = args + self.cell = cell + self.root = Path(tempfile.mkdtemp(prefix=f'soak-{cell}-', dir=os.environ.get('TMPDIR') or '/tmp')) + self.pardes = str(Path(args.pardes).resolve()) + self.gui = str(Path(args.gui).resolve()) + self.far = True + self.pids = [] + self.clients = [] + self.relays = [] + self.events = [] + self.breaches = [] + self.notes = open(self.root / 'actions', 'w') + only = set(args.only.split(',')) if args.only else None + skip = set(args.skip.split(',')) if args.skip else set() + self.actions = [a for a in ACTIONS if (only is None or a[0] in only) and a[0] not in skip] + self.nlock = threading.Lock() + self.t0 = now() + env = {k: v for k, v in os.environ.items() if not k.startswith(('PARDES_', 'WAYLAND', 'DBUS_', 'DISPLAY', 'BROWSER')) + and 'PAGER' not in k and k not in ('winid', 'EDITOR', 'NINE_MOUNT')} + env.update(HOME=str(self.root), XDG_RUNTIME_DIR=str(self.root), XDG_CONFIG_HOME=str(self.root / 'config'), + XDG_CACHE_HOME=str(self.root / 'cache'), XDG_STATE_HOME=str(self.root / 'state'), + TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', PARDES_NOTIME='1') + (self.root / 'config' / 'pardes').mkdir(parents=True) + (self.root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n') + self.env = env + self.work = self.root / 'work' + self.nine = None + self.nine_lock = threading.RLock() + self.session_pid = None + + def note(self, what): + with self.nlock: + self.notes.write(f'{now() - self.t0:9.3f} {what}\n') + self.notes.flush() + + def breach(self, what): + self.note('BREACH ' + what) + self.breaches.append(what) + + def spawn(self, argv, **kw): + p = subprocess.Popen(argv, **kw) + self.pids.append(p.pid) + return p + + def relay(self, rtt): + if not rtt: + return self.root / 'pardes-replica.sock' + at = self.root / f'relay{rtt}.sock' + if not at.exists(): + self.relays.append(self.spawn([sys.executable, '-B', str(HERE / 'delay_relay.py'), str(at), str(self.root / 'pardes-replica.sock'), + '--delay', str(rtt / 2), '--jitter', str(self.args.jitter)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) + wait_for('the relay', at.exists) + return at + + def pane_count(self): + """The session's panes (a script's view of /index), cached a second: + a Del that closed the last one would end the session.""" + t = now() + if t - getattr(self, 'counted_at', 0) > 1: + rows = self.nine_read('/index') or b'' + self.counted, self.counted_at = len(rows.splitlines()), t + return self.counted + + def nine_read(self, path): + with self.nine_lock: + return self._nine_read(path) + + def nine_write(self, path, data): + with self.nine_lock: + return self._nine_write(path, data) + + def _nine_read(self, path): + for _ in range(3): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.read(path) + except (OSError, EOFError): + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + time.sleep(0.5) + return None + + def _nine_write(self, path, data): + for _ in range(2): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.write(path, data) + except EOFError: + self.nine = None + time.sleep(0.5) + except OSError as e: + return e + + # The session: here. + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) + if self.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / 'session.err', 'w'), + start_new_session=True) + self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' + wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20) + wait_for('the replica socket', (self.root / 'pardes-replica.sock').exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + self.pids.append(self.session_pid) + # Two shells beside the directory, as the user's had. + for _ in range(2): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(0.5) + + def session_alive(self): + return alive(self.session_pid) if self.session_pid else False + + def end_session(self): + """Exit (twice: unsaved text asks once), then --kill if it stays.""" + for _ in range(3): + self.nine_write('/ctl', b'Exit\n') + try: + wait_for('the session\'s end', lambda: not self.session_alive(), timeout=3) + return True + except TimeoutError: + pass + self.note('the session did not Exit: --kill') + self.kill_session() + return False + + def kill_session(self): + subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30) + + def session_said(self): + try: + return (self.root / 'session.err').read_text(errors='replace') + except FileNotFoundError: + return '' + + def crashes(self): + p = self.root / 'config' / 'pardes' / 'crashes' + return p.read_text() if p.exists() else '' + + def cleanup(self): + for c in self.clients: + if c.p and c.p.poll() is None: + c.p.kill() + c.close() + for pid in self.pids + processes(self.root): + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + + +class BoxRun(Run): + """The session on the FreeBSD box; the clients here through ssh.""" + + def __init__(self, args, cell): + super().__init__(args, cell) + self.far = False # an ssh client is far by itself + self.run_dir = f'{BOX_T}/run-hunt' + self.home = f'{BOX_T}/home-hunt' + self.env['PARDES_SSH'] = ' '.join(SSH) + self.env['PARDES_REMOTE_RUNTIME'] = self.run_dir + self.fwd = None + + def box(self, script, timeout=60): + return subprocess.run(SSH + [BOX, script], capture_output=True, timeout=timeout) + + def benv(self): + return (f'HOME={self.home} XDG_RUNTIME_DIR={self.run_dir} XDG_CONFIG_HOME={self.home}/config TMPDIR={BOX_T}/tmp-hunt ' + f'XDG_CACHE_HOME={self.home}/cache XDG_STATE_HOME={self.home}/state PATH={BOX_T}/bin:/usr/local/bin:/usr/bin:/bin ' + f'SHELL=/usr/local/bin/bash TERM=xterm-256color PARDES_NOTIME=1 PARDES_REPLICA_STATS={self.home}/session.stats' + + (' PARDES_REPLICA_TRACE=1' if self.args.trace else '')) + + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout + bin_ = self.args.remote_bin + script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' + f'rm -rf {self.home}; mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' + f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' + f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work </dev/null >{self.home}/session.err 2>&1 & ' + f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' + f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err; exit 4; }}') + got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) + if got.returncode != 0: + raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}') + self.nine_sock = self.root / 'box9p.sock' + self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + wait_for('the 9P forward', self.nine_sock.exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + for _ in range(2): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + + def session_alive(self): + got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) + return got.stdout.strip() == b'yes' + + def session_said(self): + got = self.box(f'cat {self.home}/session.err; ls {self.home}/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) + return got.stdout.decode(errors='replace') + + def crashes(self): + return '' + + def kill_session(self): + self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) + + def relay(self, rtt): + return BOX + + def cleanup(self): + super().cleanup() + # Everything started under run-hunt, by its environment: ps -e prints it. + self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' + f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) + + +def start_client(run, kind, rtt, n): + label = f'{kind}{rtt}-{n}' + link = run.relay(rtt) + c = (TtyClient if kind == 'tty' else WinClient)(run, label, link) + c.rtt = rtt + c.start() + run.pids.append(c.p.pid) + run.clients.append(c) + run.note(f'{label} started (pid {c.p.pid})') + wait_for(f'{label}\'s first frame', c.ready, timeout=60 if isinstance(run, BoxRun) else 30) + if isinstance(c, WinClient): + c.frame_size() + time.sleep(0.3) + return c + + +def hold(run, clients, timeout=30): + for c in clients: + c.held.set() + deadline = now() + timeout + for c in clients: + while not c.paused.is_set() and c.poll() is None and now() < deadline: + time.sleep(0.05) + + +def release(clients): + for c in clients: + c.held.clear() + + +def quiet_point(run, live): + """Every monkey held: each client must give a fresh check within 8 s + (a freeze otherwise), and none may differ; a terminal client's view + against the session's.""" + hold(run, live) + time.sleep(1.5) + for c in live: + if c.poll() is not None: + continue + before = len(c.check_lines()) + # A key that moves nothing a digest sees but the view's cursor, so the + # client's input is the session's last: its view is the live one. + c.keys(b'l') + t = now() + fresh = False + while now() - t < 8 + 2 * c.rtt / 1000: + if len(c.check_lines()) > before: + fresh = True + break + time.sleep(0.1) + if not fresh and c.poll() is None: + run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') + if isinstance(c, TtyClient) and fresh: + view_check(run, c) + release(live) + + +def view_check(run, c): + """The terminal client's screen and cursor against what the session renders + for the same view (its input was the last, so 9P reads that view).""" + time.sleep(0.4) + raw = run.nine_read('/screen') + if not raw: + return + try: + s = json.loads(raw) + except ValueError: + return + lines, cursor, hidden = c.term.view() + cols, rows = s.get('cols'), s.get('rows') + if (cols, rows) != (c.term.cols, c.term.rows): + c.view_skips = getattr(c, 'view_skips', 0) + 1 + return + theirs = [] + cells = s.get('cells', []) + for r in range(rows): + row = cells[r * cols:(r + 1) * cols] if len(cells) == rows * cols else (cells[r] if r < len(cells) else []) + theirs.append(''.join(cell[0] if isinstance(cell, list) else str(cell) for cell in row).rstrip()) + sc = s.get('cursor') + differ = [r for r in range(min(rows, len(lines))) if lines[r] != theirs[r]] + c.views = getattr(c, 'views', 0) + 1 + want = None + if isinstance(sc, dict): + want = (sc.get('row', sc.get('y')), sc.get('col', sc.get('x'))) + elif isinstance(sc, list) and len(sc) >= 2: + want = (sc[1], sc[0]) + bad_cursor = want is not None and None not in want and tuple(want) != cursor and not hidden + if differ or bad_cursor: + c.view_bad = getattr(c, 'view_bad', 0) + 1 + d = c.dir / f'view-{c.view_bad}.txt' + d.write_text(f'cursor client {cursor} hidden {hidden} session {sc}\nrows differing {differ[:12]}\n' + + ''.join(f'client {r:2} |{lines[r]}|\nsession {r:2} |{theirs[r]}|\n' for r in differ[:12])) + run.breach(f'{c.label}: its screen is not the session\'s render of its view ({len(differ)} rows' + f'{", cursor " + str(cursor) + " vs " + str(want) if bad_cursor else ""}): {d}') + + +def global_event(run, rng, live, made): + """Something happens beside the typing: a resize, a client detached and + another attached, an Upgrade, a script's edit over 9P.""" + kinds = [e for e in run.args.events.split(',') if e] + if not kinds: + return None + if run.args.upgrade and not getattr(run, 'upgraded', False): + kinds.append('upgrade') + what = rng.choice(kinds) + if what == 'resize': + c = rng.choice(live) + rows, cols = rng.randint(18, 50), rng.randint(60, 160) + run.note(f'{c.label} resized to {cols}x{rows}') + c.resize(rows, cols) + elif what == 'detach': + c = rng.choice(live) + run.note(f'{c.label} detached') + hold(run, [c]) + c.asked_to_end = True + c.keys(ESC) + c.keys(b'\x02') + time.sleep(0.2) + c.keys(ESC) + c.keys(b' sD') + try: + wait_for(f'{c.label} let go', lambda: c.poll() is not None, timeout=15) + except TimeoutError: + # Its keys went to a program or a prompt: it is ended as a window closes. + run.note(f'{c.label} did not detach by keys: hung up') + c.hangup() + wait_for(f'{c.label} hung up', lambda: c.poll() is not None, timeout=15) + live.remove(c) + made[0] += 1 + n = start_client(run, c.kind, c.rtt, made[0]) + live.append(n) + return n + elif what == 'upgrade': + run.note('Upgrade') + run.upgraded = True + run.nine_write('/ctl', b'Upgrade\n') + time.sleep(3) + if run.nine: + try: + run.nine.socket.close() + except Exception: + pass + run.nine = None + elif what == 'script': + rows = (run.nine_read('/index') or b'').splitlines() + texts = [r.split()[0].decode() for r in rows if len(r.split()) > 1 and r.split()[1] == b'text'] + if texts: + n = rng.choice(texts) + run.note(f'9P append to pane {n}') + run.nine_write(f'/pane/{n}/body', b'a line a script wrote\n') + return None + + +def soak(run, kinds, rtt, minutes, seed): + rng = random.Random(seed) + stop = threading.Event() + live = [] + threads = [] + made = [0] + try: + soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made) + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + stop.set() + report_cell(run, rtt) + run.cleanup() + + +def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): + run.start_session() + for kind in kinds: + made[0] += 1 + live.append(start_client(run, kind, rtt, made[0])) + for c in list(live): + t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + end = now() + minutes * 60 + next_quiet = now() + 45 + next_event = now() + rng.uniform(20, 60) + while now() < end: + time.sleep(0.5) + for c in list(live): + if c.poll() is not None: + run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') + live.remove(c) + if not run.session_alive(): + run.breach('the session died') + break + if not live: + break + if now() > next_event: + n = global_event(run, rng, live, made) + if n: + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + next_event = now() + rng.uniform(20, 70) + if now() > next_quiet: + quiet_point(run, live) + next_quiet = now() + rng.uniform(40, 80) + stop.set() + for t in threads: + t.join(timeout=20) + if run.session_alive() and live: + quiet_point(run, live) + # Each client's links written: a hangup ends a client the normal way. + for c in live: + c.asked_to_end = True + if c.p.poll() is None: + c.hangup() + for c in live: + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + run.breach(f'{c.label}: would not end on a hangup') + c.p.kill() + if run.session_alive(): + run.end_session() + + +def tail(c): + if isinstance(c, WinClient): + return c.said()[-300:].replace('\n', ' | ') + lines, _, _ = c.term.view() + return ' | '.join(line for line in lines if line.strip())[-300:] + + +ROWS = [] + + +def cores(run): + """systemd-coredump's cores since the cell began, of the pids it started.""" + if isinstance(run, BoxRun) or not shutil.which('coredumpctl'): + return [] + since = time.time() - (now() - run.t0) - 5 + got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True) + ours = set(run.pids) | {c.p.pid for c in run.clients if c.p} | ({run.session_pid} if run.session_pid else set()) + out = [] + for line in got.stdout.splitlines(): + w = line.split() + pid = next((int(x) for x in w if x.isdigit() and int(x) in ours), None) + if pid: + out.append(f'pid {pid} {" ".join(x for x in w if x.startswith("SIG"))} {w[-2] if len(w) > 1 else ""}') + return out + + +def report_cell(run, rtt): + for core in cores(run): + run.breach(f'core dumped: {core} (coredumpctl debug <pid>)') + said = run.session_said() + for word in ('panic', 'Segmentation', 'abort', 'reached unreachable', 'core dumped'): + if word in said: + run.breach(f'session said "{word}": {said[said.find(word) - 200:said.find(word) + 300]!r}') + break + if run.crashes(): + run.breach(f'crashes: {run.crashes()[-400:]}') + for c in run.clients: + st = c.stats_sum() + links = len(c.links()) + lines = c.check_lines() + differ = [line for line in lines if line.startswith('differ')] + said = c.said_joined() + if links > 1 or said: + run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})') + if differ: + run.breach(f'{c.label}: {len(differ)} checks differ') + if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): + run.breach(f'{c.label}: died of signal {-c.gone}') + if c.gone not in (None, 0) and not c.asked_to_end: + pass # said when found + elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: + run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') + joined = c.said() + for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): + if word in joined: + run.breach(f'{c.label} said "{word}": {joined[joined.find(word) - 200:joined.find(word) + 300]!r}') + break + if isinstance(c, WinClient): + lat, missed = c.latencies() + else: + lat, missed = c.lat, c.no_output + dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) + ROWS.append({ + 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, + 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], + 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), + 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone, + }) + st = (run.root / 'session.stats') + text = '' + if isinstance(run, BoxRun): + text = run.box(f'cat {run.home}/session.stats 2>/dev/null', timeout=30).stdout.decode() + elif st.exists(): + text = st.read_text() + for line in text.splitlines(): + if line.startswith('session inputs'): + w = line.split() + d = {w[i]: w[i + 1] for i in range(1, len(w) - 1, 2)} + worst = int(d.get('boundary_us', '0/0/0').split('/')[2]) + ROWS.append({'cell': run.cell, 'client': 'session', 'rtt': rtt, 'boundary_us': d.get('boundary_us'), + 'input_us': d.get('input_us'), 'cut': d.get('cut')}) + if worst > 250_000: + run.breach(f'session: a boundary took {worst / 1000:.0f} ms') + run.note('end of cell') + + +def table(rows): + cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', + 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit'] + def fmt(v): + if v is None: + return '-' + if isinstance(v, float): + return f'{v:.1f}' + return str(v) + out = [] + widths = {c: len(c) for c in cols} + body = [] + for r in rows: + if r['client'] == 'session': + body.append(f"{r['cell']:<8} session boundary_us p50/p99/max {r['boundary_us']} input_us {r['input_us']} cut {r['cut']}") + continue + vals = [fmt(r.get(c)) for c in cols] + for c, v in zip(cols, vals): + widths[c] = max(widths[c], len(v)) + body.append(vals) + out.append(' '.join(c.ljust(widths[c]) for c in cols)) + for b in body: + out.append(b if isinstance(b, str) else ' '.join(v.ljust(widths[c]) for c, v in zip(cols, b))) + return '\n'.join(out) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('where', choices=['local', 'box']) + ap.add_argument('--minutes', type=float, default=None) + ap.add_argument('--rtts', default='0,30,80,207') + ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') + ap.add_argument('--seed', type=int, default=1) + ap.add_argument('--clients', default='tty,win', help='kinds, one client each: tty,win,win...') + ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) + ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) + ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') + ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') + ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') + ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') + ap.add_argument('--keep', action='store_true', help='keep the run\'s files (and the terminals\' output)') + ap.add_argument('--only', default='', help='the monkey\'s actions, by name: move,insert,...') + ap.add_argument('--skip', default='', help='actions left out, by name') + ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') + args = ap.parse_args() + kinds = args.clients.split(',') + runs = [] + if args.where == 'local': + minutes = args.minutes if args.minutes is not None else 2 + for i, rtt in enumerate(int(x) for x in args.rtts.split(',')): + run = Run(args, f'local{rtt}') + run.far = not args.near + runs.append(run) + print(f'soak: {run.cell}: {kinds} for {minutes} min, seed {args.seed + i}, in {run.root}', flush=True) + soak(run, kinds, rtt, minutes, args.seed + i) + else: + minutes = args.minutes if args.minutes is not None else 20 + run = BoxRun(args, 'box') + runs.append(run) + print(f'soak: box: {kinds} for {minutes} min, seed {args.seed}, in {run.root}', flush=True) + soak(run, kinds, 207, minutes, args.seed) + print(table(ROWS)) + breaches = [(r.cell, b) for r in runs for b in r.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for r in runs: + if args.keep or r.breaches: + print(f'kept: {r.root}') + else: + shutil.rmtree(r.root, ignore_errors=True) + print('soak:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + +if __name__ == '__main__': + main() |
