#!/usr/bin/env python3 """A soak of replica clients: what a person does in them, through each link, measured (docs/next-daemon.md, the replica and GUI replica notes). python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10] [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] [--pardes P] [--gui G] [--seed 1] [--trace] `local`: a session on this machine (`pardes --detach=soak `, the shape of the user's: a directory pane, then two shells), and at each RTT its clients through test/delay_relay.py (with jitter), far as through ssh (PARDES_REPLICA_FAR=1: the blobs by hash, the stream in zstd). One cell an RTT, `--minutes` each. `box`: the session on the FreeBSD box under /root/pardes-test (`--remote-bin`, by default bin/pardes-hunt there, run in run-hunt and home-hunt), the clients here through ssh, as the user's window was; its 9P through an `ssh -L` of the session's socket. The clients: terminals (`pardes --attach`, in a pty here whose screen pyte keeps, frame requests answered) and windows (`pardes-gui --attach`, SDL's offscreen driver, keys and the mouse through its test feed on a pty, so it can be resized). Each runs a seeded monkey of a person's actions: moves of every kind (keys, clicks, jumps, search), scrolls (keys, the wheel), edits, undo and save, Look at files and directories, the Panes picker, Find, new terminals with programs (ls, a long output, less, vi, top, Ctrl-C), panes and columns opened, moved and closed, the pointer resting and moving, pastes, idle gaps. Between them, at random: a client resized, a client detached and another attached, an Upgrade, a 9P script's edit. `--seed` makes a run again (the timing differs, the actions do not). Measured (a row a client, then the session's): - rejoins: links past the first (PARDES_REPLICA_STATS writes one a link) and `joined again` said by it; checks (PARDES_REPLICA_CHECK) that differ; - exits nobody asked for, a signal or a non-zero code, the session's death, a panic or `crashes` line; - freezes: a quiet point where a client gives no fresh check within 8 s, and a terminal client's key with no output within 2 s while it was not held; the session's worst boundary (its stats at its end); - the view: at each quiet point, a terminal client's cursor and screen (pyte) against what the session renders for its view (/screen read right after its own input made its view the live one), and the panes on it; - blink: visible rollbacks (`visible`), mispredicted inputs, a terminal client's full clears, a minute; - keystroke to output (terminal) and to a frame (window, PARDES_TEST_LATENCY) p50 / p99. Prints a table, then each breach, and exits 1 if there is any. Everything started is ended by pid (and found by its environment); the run's files are kept under $TMPDIR/soak-* (`--keep`) or removed. """ import argparse import fcntl import json import os import random import re import shutil import signal import socket import statistics import struct import subprocess import sys import tempfile import termios import threading import time import tty as ttymod from pathlib import Path import pyte HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) from ninep import Client # noqa: E402 from daemon import processes, alive, wait_for, checks # noqa: E402 BOX = 'root@65.109.81.103' SSH = ['/usr/bin/ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', '-o', 'UserKnownHostsFile=/home/goblin/.ssh/known_hosts', '-i', '/home/goblin/.ssh/id_ed25519'] BOX_T = '/root/pardes-test' NAME = 'soak' ESC = b'\x1b' def now(): return time.monotonic() def pct(xs, p): if not xs: return None xs = sorted(xs) return xs[min(len(xs) - 1, len(xs) * p // 100)] # ---- the fixture: a directory such as a person works in ---- WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa ' 'quebec romeo sierra tango uniform victor whiskey xray yankee zulu').split() def fixture(at, rng): at.mkdir(parents=True, exist_ok=True) (at / 'notes.txt').write_text(''.join(f'line {i} of the notes: {rng.choice(WORDS)} {rng.choice(WORDS)} see code.zig:{i % 90 + 1}\n' for i in range(300))) (at / 'code.zig').write_text(''.join(f'pub fn f{i}(x: u32) u32 {{\n return x * {i} + {rng.randrange(1000)}; // {rng.choice(WORDS)}\n}}\n\n' for i in range(120))) (at / 'big.txt').write_text(''.join(f'{i:06d} {" ".join(rng.choice(WORDS) for _ in range(9))}\n' for i in range(8000))) (at / 'README').write_text('A directory to soak in.\nnotes.txt code.zig big.txt sub/\nsub/inner.txt:3\n') sub = at / 'sub' sub.mkdir(exist_ok=True) (sub / 'inner.txt').write_text(''.join(f'inner {i} {rng.choice(WORDS)}\n' for i in range(60))) (sub / 'more.md').write_text('# More\n\n' + ''.join(f'- item {i} {rng.choice(WORDS)}\n' for i in range(40))) # ---- terminals ---- class Term: """A terminal: a pty whose output is kept, timed and fed to pyte, its frame requests (CSI 5 n) answered as a terminal emulator does. An Esc is written alone and nothing follows it for 80 ms, an answer included: the client's parser takes an Esc and what follows within 50 ms as Alt.""" def __init__(self, rows, cols): self.master, self.slave = os.openpty() self.rows, self.cols = rows, cols fcntl.ioctl(self.slave, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) self.lock = threading.Lock() self.wlock = threading.Lock() self.screen = pyte.Screen(cols, rows) self.stream = pyte.ByteStream(self.screen) self.out_bytes = 0 self.last_out = 0.0 self.chunks = [] # (time, bytes) of the last minute or so, for latencies self.clears = 0 self.joined_again = 0 self.tail = b'' self.esc_until = 0.0 self.closing = False self.log = None self.reader = threading.Thread(target=self.drain, daemon=True) self.reader.start() def drain(self): import select while not self.closing: try: if not select.select([self.master], [], [], 0.05)[0]: continue got = os.read(self.master, 65536) except OSError: return if not got: return t = now() carry, self.tail = self.tail, (self.tail + got)[-11:] seen = carry[-3:] + got with self.lock: self.out_bytes += len(got) self.last_out = t self.chunks.append((t, len(got))) if len(self.chunks) > 4096: del self.chunks[:2048] # The carry is shorter than either word: a match holds a new byte. self.clears += seen.count(b'\x1b[2J') self.joined_again += (carry + got).count(b'joined again') self.feed(got) if self.log: self.log.write(got) if b'\x1b[5n' in seen: wait = self.esc_until - now() if wait > 0: time.sleep(wait) with self.wlock: try: os.write(self.master, b'\x1b[0n') except OSError: return SGR = re.compile(rb'\x1b\[[0-9:;]*m') def feed(self, got): """To pyte, without its colours (pyte reads no colon sub-parameters) and holding back a sequence a read cut in two.""" data = getattr(self, 'held_back', b'') + got cut = data.rfind(b'\x1b') self.held_back = b'' if cut >= 0 and not re.match(rb'\x1b(\[[0-9:;?<>=! ]*[@-~]|[^\[\]]|\][^\x07\x1b]*(\x07|\x1b\\))', data[cut:]): data, self.held_back = data[:cut], data[cut:] try: self.stream.feed(self.SGR.sub(b'', data)) except Exception: pass def write(self, data): """Keys as typed: each Esc alone, 80 ms clear after it.""" parts = re.split(b'(\x1b(?![\\[O\\]n0-9a-z]))', data) for part in parts: if not part: continue if part == ESC: with self.wlock: wait = self.esc_until - now() if wait > 0: time.sleep(wait) os.write(self.master, ESC) self.esc_until = now() + 0.08 time.sleep(0.08) else: with self.wlock: wait = self.esc_until - now() if wait > 0: time.sleep(wait) os.write(self.master, part) def resize(self, rows, cols): self.rows, self.cols = rows, cols self.px = (cols * 16, rows * 33) fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) with self.lock: self.screen.resize(rows, cols) def view(self): with self.lock: return [line.rstrip() for line in self.screen.display], (self.screen.cursor.y, self.screen.cursor.x), self.screen.cursor.hidden def spawn(self, argv, env, cwd, err): slave = self.slave return subprocess.Popen(argv, env=env, cwd=cwd, stdin=slave, stdout=slave, stderr=err, start_new_session=True, preexec_fn=lambda: fcntl.ioctl(0, termios.TIOCSCTTY, 0)) def close(self): self.closing = True self.reader.join(timeout=2) for fd in (self.slave, self.master): try: os.close(fd) except OSError: pass # ---- clients ---- class Base: def __init__(self, run, kind, label, link): self.run = run self.kind = kind self.label = label self.link = link # where it dials: a socket path, or the box self.dir = run.root / label self.dir.mkdir() self.check = self.dir / 'check' self.stats = self.dir / 'stats' self.err = self.dir / 'err' self.p = None self.started = None self.asked_to_end = False self.gone = None # how it ended, if it did self.inputs = 0 self.lat = [] # ms, input to output/frame self.no_output = 0 self.held = threading.Event() # the monkey waits while set self.paused = threading.Event() # set by the monkey once it waits self.lock = threading.Lock() def env(self): e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) if self.run.args.trace: e['PARDES_REPLICA_TRACE'] = '1' if self.run.far: e['PARDES_REPLICA_FAR'] = '1' return e def attach_arg(self): return f'--attach={self.link}:{NAME}' def links(self): return [line for line in checks(self.stats) if line.startswith('inputs ')] def stats_sum(self): keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible') tot = dict.fromkeys(keys, 0) for line in self.links(): w = line.split() d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)} for k in keys: try: tot[k] += int(d.get(k, 0)) except ValueError: pass return tot def check_lines(self): return checks(self.check) def poll(self): if self.gone is None and self.p is not None and self.p.poll() is not None: self.gone = self.p.returncode return self.gone class TtyClient(Base): def __init__(self, run, label, link, rows=30, cols=100): super().__init__(run, 'tty', label, link) self.term = Term(rows, cols) self.term.log = open(self.dir / 'out', 'wb', buffering=0) if run.args.keep else None def start(self): argv = [self.run.pardes, self.attach_arg()] # Its stderr a file: a panic's words and the replica's trace. self.p = self.term.spawn(argv, self.env(), self.run.root, open(self.err, 'w')) self.started = now() def ready(self): return self.term.out_bytes > 2000 and now() - self.term.last_out > 0.3 def keys(self, data): self.inputs += 1 self.run.note(f'{self.label} < {data!r}') before = self.term.out_bytes t = now() self.term.write(data) threading.Thread(target=self._latency, args=(t, before), daemon=True).start() def _latency(self, t, before): deadline = t + 2.0 while now() < deadline: with self.term.lock: got = [c for c in self.term.chunks if c[0] >= t] if got: with self.lock: self.lat.append((got[0][0] - t) * 1000) return time.sleep(0.005) if not self.held.is_set(): with self.lock: self.no_output += 1 def mouse(self, kind, button, x, y): # SGR: cells from 1; x, y here are fractions of the screen. col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) b = {'down': button - 1, 'up': button - 1, 'motion': 35, 'wheel': 64 if button > 0 else 65}[kind] end = 'm' if kind == 'up' else 'M' self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) def resize(self, rows, cols): self.term.resize(rows, cols) if self.p: try: os.killpg(self.p.pid, signal.SIGWINCH) except OSError: pass def said(self): try: return self.err.read_text(errors='replace') except FileNotFoundError: return '' def said_joined(self): return self.term.joined_again def hangup(self): """As its terminal closing: the normal way out (its stats written).""" try: os.killpg(self.p.pid, signal.SIGHUP) except OSError: pass def close(self): self.term.close() class WinClient(Base): """A window client in test mode: offscreen, its keys and pointer through its test feed on a pty (Escape as CSI 27 u: a lone ESC waits for more).""" def __init__(self, run, label, link, rows=34, cols=110): super().__init__(run, 'win', label, link) self.cap = self.dir / 'cap' self.cap.mkdir() self.latency = self.dir / 'latency' self.master, self.slave = os.openpty() ttymod.setraw(self.slave) self.rows, self.cols = rows, cols fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) self.wlock = threading.Lock() self.px = (cols * 16, rows * 33) # test mode's 27 px font, about def start(self): e = dict(self.env(), PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(self.cap), PARDES_TEST_LATENCY=str(self.latency), SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(self.cols), PARDES_TEST_ROWS=str(self.rows)) self.p = subprocess.Popen([self.run.gui, self.attach_arg()], env=e, cwd=self.run.root, stdin=self.slave, stdout=subprocess.DEVNULL, stderr=open(self.err, 'w'), start_new_session=True) self.started = now() def ready(self): # Software presenting (offscreen) captures no ppm: its first frame is # its latency trace's first F. try: return b'F ' in self.latency.read_bytes() and self.p.poll() is None except FileNotFoundError: return False def keys(self, data): self.inputs += 1 self.run.note(f'{self.label} < {data!r}') with self.wlock: try: os.write(self.master, data.replace(ESC + b'[', b'\x00CSI').replace(ESC, b'\x1b[27u').replace(b'\x00CSI', ESC + b'[')) except OSError: pass def mouse(self, kind, button, x, y): w, h = self.px px, py = int(x * w), int(y * h) if kind == 'wheel': seq = f'\x1b]777;mouse;wheel;{button};{px};{py}\x07' else: seq = f'\x1b]777;mouse;{kind};{button};{px};{py}\x07' self.inputs += 1 self.run.note(f'{self.label} < mouse {kind} {button} {px} {py}') with self.wlock: try: os.write(self.master, seq.encode()) except OSError: pass def resize(self, rows, cols): self.rows, self.cols = rows, cols self.px = (cols * 16, rows * 33) fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) def hangup(self): """Its test feed's end: the normal way out (its stats written).""" try: os.close(self.master) except OSError: pass def said(self): try: return self.err.read_text(errors='replace') except FileNotFoundError: return '' def said_joined(self): return self.said().count('joined again') def frame_size(self): try: with open(self.cap / 'latest.ppm', 'rb') as f: head = f.read(32).split() self.px = (int(head[1]), int(head[2])) except (OSError, IndexError, ValueError): pass def latencies(self): """Each stdin read to the end of the first frame after it, ms; and the reads no frame followed within 2 s (a frame came after a later one).""" try: lines = self.latency.read_text().split('\n') except FileNotFoundError: return [], 0 reads, frames = [], [] for line in lines: w = line.split() if len(w) >= 2 and w[0] == 'R': reads.append(int(w[1])) elif len(w) >= 3 and w[0] == 'F': frames.append(int(w[2])) out, missed, j = [], 0, 0 for r in reads: while j < len(frames) and frames[j] < r: j += 1 if j < len(frames): d = (frames[j] - r) / 1e6 if d <= 2000: out.append(d) else: missed += 1 return out, missed def close(self): for fd in (self.slave, self.master): try: os.close(fd) except OSError: pass # ---- what a person does ---- def words(rng, n): return ' '.join(rng.choice(WORDS) for _ in range(n)).encode() def burst(c, rng): """One action of a person's, in whatever state the client is.""" k = c.keys acts = c.run.actions choice = rng.choices(acts, weights=[a[1] for a in acts])[0] choice[2](c, rng, k) return choice[0] def act_move(c, rng, k): for _ in range(rng.randint(1, 6)): k(rng.choice([b'j', b'k', b'h', b'l', b'w', b'b', b'e', b'5j', b'3k', b'gg', b'ge', b'\x04', b'\x15', b'\x06', b'\x02', b'}', b'{', b'%', b'x', b'12G', b'gl', b'gh'])) time.sleep(rng.uniform(0.02, 0.2)) def act_search(c, rng, k): k(b'/' + rng.choice(WORDS).encode() + b'\r') for _ in range(rng.randint(0, 3)): time.sleep(rng.uniform(0.1, 0.4)) k(rng.choice([b'n', b'N'])) def act_jump(c, rng, k): k(rng.choice([b'\x0f', b' jo', b' jj', b'gd', b'\x0f\x0f'])) def act_insert(c, rng, k): k(rng.choice([b'i', b'a', b'o', b'O', b'A'])) for w in words(rng, rng.randint(1, 5)).split(): k(w + b' ') time.sleep(rng.uniform(0.03, 0.15)) k(ESC) def act_delete(c, rng, k): k(rng.choice([b'xd', b'd', b'wd', b'xxd', b'c' + words(rng, 1) + ESC])) def act_undo(c, rng, k): k(rng.choice([b'u', b'uu', b'U', b'u'])) def act_save(c, rng, k): k(b' fs') def act_look(c, rng, k): # Enter is Look: a directory's row opens it, a word finds its next place. k(rng.choice([b'\r', b'j\r', b'jj\r', b'gg\r'])) def act_panes(c, rng, k): k(b' bb') time.sleep(rng.uniform(0.3, 1.0)) k(rng.choice([b'\r', ESC, b'jj\r', b'not\r'])) def act_find(c, rng, k): k(rng.choice([b' ff', b' fr', b' fg'])) time.sleep(rng.uniform(0.3, 1.0)) k(rng.choice([b'notes\r', ESC, b'j\r', b'inner\r'])) def act_shell(c, rng, k): # A terminal of its own, a program in it, and back out to normal mode. k(b' nt') time.sleep(rng.uniform(0.8, 1.5)) for _ in range(rng.randint(1, 3)): prog = rng.choice(['ls -la\r', 'seq 1 3000\r', 'less big.txt\r', 'vi notes.txt\r', 'top\r', 'sleep 30\r', 'echo ' + rng.choice(WORDS) + '\r', 'cat code.zig\r']) k(prog.encode()) time.sleep(rng.uniform(0.5, 2.0)) if prog.startswith('less'): # Never SPC then q: that is Exit, should the keys reach no less. k(b'jjjj\x06') time.sleep(0.3) k(b'q') elif prog.startswith('vi'): k(b'jjjjx') k(ESC) k(b':q!\r') elif prog.startswith(('top', 'sleep', 'seq')): k(b'\x03' if not prog.startswith('top') else b'q') time.sleep(rng.uniform(0.2, 0.8)) k(b'\x02') # Ctrl-b: normal mode if rng.random() < 0.5 and c.run.pane_count() >= 4: time.sleep(0.3) k(b' d') time.sleep(0.6) k(rng.choice([b'k', b'j', b''])) def act_window(c, rng, k): k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l'])) def act_column(c, rng, k): few = c.run.pane_count() < 5 k(rng.choice([b' cn', b' cn'] if few else [b' cn', b' cn', b' cd', b' cj'])) def act_del(c, rng, k): if c.run.pane_count() < 4: return act_move(c, rng, k) k(b' d') time.sleep(0.6) k(rng.choice([b'k', b'j', b'', b' d'])) def act_new(c, rng, k): k(b' fn') time.sleep(0.4) k(b'i' + words(rng, 4)) k(ESC) def act_esc(c, rng, k): k(rng.choice([ESC, b'\x02', b':', b':'])) time.sleep(0.3) k(ESC) def act_click(c, rng, k): x, y = rng.random(), rng.uniform(0.05, 1.0) b = rng.choice([1, 1, 1, 3]) c.mouse('down', b, x, y) time.sleep(rng.uniform(0.03, 0.12)) if rng.random() < 0.3: # a drag c.mouse('motion', b, min(1, x + rng.uniform(0, 0.2)), min(1, y + rng.uniform(0, 0.1))) c.mouse('up', b, x, y) def act_wheel(c, rng, k): x, y = rng.random(), rng.uniform(0.1, 1.0) d = rng.choice([1, -1]) for _ in range(rng.randint(1, 8)): c.mouse('wheel', d, x, y) time.sleep(rng.uniform(0.01, 0.08)) def act_hover(c, rng, k): x, y = rng.random(), rng.uniform(0.1, 1.0) for _ in range(rng.randint(2, 12)): x = min(1, max(0, x + rng.uniform(-0.05, 0.05))) y = min(1, max(0.05, y + rng.uniform(-0.03, 0.03))) c.mouse('motion', 0, x, y) time.sleep(rng.uniform(0.01, 0.06)) time.sleep(rng.uniform(0.05, 0.6)) # resting: a Look preview def act_paste(c, rng, k): k(b'i\x1b[200~' + b'\n'.join(words(rng, 6) for _ in range(rng.randint(1, 4))) + b'\x1b[201~') k(ESC) def act_settings(c, rng, k): k(rng.choice([b' tw', b' ?', b' hm', b' tt'])) time.sleep(0.5) k(ESC) def act_idle(c, rng, k): time.sleep(rng.choice([0.5, 1, 2, 4, 8])) ACTIONS = [ ('move', 30, act_move), ('search', 5, act_search), ('jump', 4, act_jump), ('insert', 12, act_insert), ('delete', 4, act_delete), ('undo', 5, act_undo), ('save', 3, act_save), ('look', 6, act_look), ('panes', 3, act_panes), ('find', 3, act_find), ('shell', 3, act_shell), ('window', 4, act_window), ('column', 1, act_column), ('del', 2, act_del), ('new', 2, act_new), ('esc', 4, act_esc), ('click', 8, act_click), ('wheel', 8, act_wheel), ('hover', 6, act_hover), ('paste', 2, act_paste), ('settings', 1, act_settings), ('idle', 6, act_idle), ] def monkey(c, rng, stop): while not stop.is_set(): if c.held.is_set(): c.paused.set() time.sleep(0.05) continue c.paused.clear() if c.poll() is not None: return try: what = burst(c, rng) c.run.note(f'{c.label} {what}') except OSError: return time.sleep(rng.uniform(0.05, 0.6)) # ---- the run ---- class Run: def __init__(self, args, cell): self.args = args self.cell = cell self.root = Path(tempfile.mkdtemp(prefix=f'soak-{cell}-', dir=os.environ.get('TMPDIR') or '/tmp')) self.pardes = str(Path(args.pardes).resolve()) self.gui = str(Path(args.gui).resolve()) self.far = True self.pids = [] self.clients = [] self.relays = [] self.events = [] self.breaches = [] self.notes = open(self.root / 'actions', 'w') only = set(args.only.split(',')) if args.only else None skip = set(args.skip.split(',')) if args.skip else set() self.actions = [a for a in ACTIONS if (only is None or a[0] in only) and a[0] not in skip] self.nlock = threading.Lock() self.t0 = now() env = {k: v for k, v in os.environ.items() if not k.startswith(('PARDES_', 'WAYLAND', 'DBUS_', 'DISPLAY', 'BROWSER')) and 'PAGER' not in k and k not in ('winid', 'EDITOR', 'NINE_MOUNT')} env.update(HOME=str(self.root), XDG_RUNTIME_DIR=str(self.root), XDG_CONFIG_HOME=str(self.root / 'config'), XDG_CACHE_HOME=str(self.root / 'cache'), XDG_STATE_HOME=str(self.root / 'state'), TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', PARDES_NOTIME='1') (self.root / 'config' / 'pardes').mkdir(parents=True) (self.root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n') self.env = env self.work = self.root / 'work' self.nine = None self.nine_lock = threading.RLock() self.session_pid = None def note(self, what): with self.nlock: self.notes.write(f'{now() - self.t0:9.3f} {what}\n') self.notes.flush() def breach(self, what): self.note('BREACH ' + what) self.breaches.append(what) def spawn(self, argv, **kw): p = subprocess.Popen(argv, **kw) self.pids.append(p.pid) return p def relay(self, rtt): if not rtt: return self.root / 'pardes-replica.sock' at = self.root / f'relay{rtt}.sock' if not at.exists(): self.relays.append(self.spawn([sys.executable, '-B', str(HERE / 'delay_relay.py'), str(at), str(self.root / 'pardes-replica.sock'), '--delay', str(rtt / 2), '--jitter', str(self.args.jitter)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) wait_for('the relay', at.exists) return at def pane_count(self): """The session's panes (a script's view of /index), cached a second: a Del that closed the last one would end the session.""" t = now() if t - getattr(self, 'counted_at', 0) > 1: rows = self.nine_read('/index') or b'' self.counted, self.counted_at = len(rows.splitlines()), t return self.counted def nine_read(self, path): with self.nine_lock: return self._nine_read(path) def nine_write(self, path, data): with self.nine_lock: return self._nine_write(path, data) def _nine_read(self, path): for _ in range(3): try: if self.nine is None: self.nine = Client(self.nine_sock, timeout=10) return self.nine.read(path) except (OSError, EOFError): try: self.nine.socket.close() except Exception: pass self.nine = None time.sleep(0.5) return None def _nine_write(self, path, data): for _ in range(2): try: if self.nine is None: self.nine = Client(self.nine_sock, timeout=10) return self.nine.write(path, data) except EOFError: self.nine = None time.sleep(0.5) except OSError as e: return e # The session: here. def start_session(self): rng = random.Random(self.args.seed) fixture(self.work, rng) e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) if self.args.trace: e['PARDES_REPLICA_TRACE'] = '1' self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / 'session.err', 'w'), start_new_session=True) self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20) wait_for('the replica socket', (self.root / 'pardes-replica.sock').exists, timeout=20) status = self.nine_read('/status').decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) self.pids.append(self.session_pid) # Two shells beside the directory, as the user's had. for _ in range(2): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(0.5) def session_alive(self): return alive(self.session_pid) if self.session_pid else False def end_session(self): """Exit (twice: unsaved text asks once), then --kill if it stays.""" for _ in range(3): self.nine_write('/ctl', b'Exit\n') try: wait_for('the session\'s end', lambda: not self.session_alive(), timeout=3) return True except TimeoutError: pass self.note('the session did not Exit: --kill') self.kill_session() return False def kill_session(self): subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30) def session_said(self): try: return (self.root / 'session.err').read_text(errors='replace') except FileNotFoundError: return '' def crashes(self): p = self.root / 'config' / 'pardes' / 'crashes' return p.read_text() if p.exists() else '' def cleanup(self): for c in self.clients: if c.p and c.p.poll() is None: c.p.kill() c.close() for pid in self.pids + processes(self.root): try: os.kill(pid, signal.SIGKILL) except OSError: pass if self.nine: try: self.nine.socket.close() except Exception: pass class BoxRun(Run): """The session on the FreeBSD box; the clients here through ssh.""" def __init__(self, args, cell): super().__init__(args, cell) self.far = False # an ssh client is far by itself self.run_dir = f'{BOX_T}/run-hunt' self.home = f'{BOX_T}/home-hunt' self.env['PARDES_SSH'] = ' '.join(SSH) self.env['PARDES_REMOTE_RUNTIME'] = self.run_dir self.fwd = None def box(self, script, timeout=60): return subprocess.run(SSH + [BOX, script], capture_output=True, timeout=timeout) def benv(self): return (f'HOME={self.home} XDG_RUNTIME_DIR={self.run_dir} XDG_CONFIG_HOME={self.home}/config TMPDIR={BOX_T}/tmp-hunt ' f'XDG_CACHE_HOME={self.home}/cache XDG_STATE_HOME={self.home}/state PATH={BOX_T}/bin:/usr/local/bin:/usr/bin:/bin ' f'SHELL=/usr/local/bin/bash TERM=xterm-256color PARDES_NOTIME=1 PARDES_REPLICA_STATS={self.home}/session.stats' + (' PARDES_REPLICA_TRACE=1' if self.args.trace else '')) def start_session(self): rng = random.Random(self.args.seed) fixture(self.work, rng) tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout bin_ = self.args.remote_bin script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' f'rm -rf {self.home}; mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work {self.home}/session.err 2>&1 & ' f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err; exit 4; }}') got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) if got.returncode != 0: raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}') self.nine_sock = self.root / 'box9p.sock' self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) wait_for('the 9P forward', self.nine_sock.exists, timeout=20) status = self.nine_read('/status').decode() self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) for _ in range(2): self.nine_write('/pane/1/exec', b'Tty\n') time.sleep(1) def session_alive(self): got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) return got.stdout.strip() == b'yes' def session_said(self): got = self.box(f'cat {self.home}/session.err; ls {self.home}/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) return got.stdout.decode(errors='replace') def crashes(self): return '' def kill_session(self): self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) def relay(self, rtt): return BOX def cleanup(self): super().cleanup() # Everything started under run-hunt, by its environment: ps -e prints it. self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) def start_client(run, kind, rtt, n): label = f'{kind}{rtt}-{n}' link = run.relay(rtt) c = (TtyClient if kind == 'tty' else WinClient)(run, label, link) c.rtt = rtt c.start() run.pids.append(c.p.pid) run.clients.append(c) run.note(f'{label} started (pid {c.p.pid})') wait_for(f'{label}\'s first frame', c.ready, timeout=60 if isinstance(run, BoxRun) else 30) if isinstance(c, WinClient): c.frame_size() time.sleep(0.3) return c def hold(run, clients, timeout=30): for c in clients: c.held.set() deadline = now() + timeout for c in clients: while not c.paused.is_set() and c.poll() is None and now() < deadline: time.sleep(0.05) def release(clients): for c in clients: c.held.clear() def quiet_point(run, live): """Every monkey held: each client must give a fresh check within 8 s (a freeze otherwise), and none may differ; a terminal client's view against the session's.""" hold(run, live) time.sleep(1.5) for c in live: if c.poll() is not None: continue before = len(c.check_lines()) # A key that moves nothing a digest sees but the view's cursor, so the # client's input is the session's last: its view is the live one. c.keys(b'l') t = now() fresh = False while now() - t < 8 + 2 * c.rtt / 1000: if len(c.check_lines()) > before: fresh = True break time.sleep(0.1) if not fresh and c.poll() is None: run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') if isinstance(c, TtyClient) and fresh: view_check(run, c) release(live) def view_check(run, c): """The terminal client's screen and cursor against what the session renders for the same view (its input was the last, so 9P reads that view).""" time.sleep(0.4) raw = run.nine_read('/screen') if not raw: return try: s = json.loads(raw) except ValueError: return lines, cursor, hidden = c.term.view() cols, rows = s.get('cols'), s.get('rows') if (cols, rows) != (c.term.cols, c.term.rows): c.view_skips = getattr(c, 'view_skips', 0) + 1 return theirs = [] cells = s.get('cells', []) for r in range(rows): row = cells[r * cols:(r + 1) * cols] if len(cells) == rows * cols else (cells[r] if r < len(cells) else []) theirs.append(''.join(cell[0] if isinstance(cell, list) else str(cell) for cell in row).rstrip()) sc = s.get('cursor') differ = [r for r in range(min(rows, len(lines))) if lines[r] != theirs[r]] c.views = getattr(c, 'views', 0) + 1 want = None if isinstance(sc, dict): want = (sc.get('row', sc.get('y')), sc.get('col', sc.get('x'))) elif isinstance(sc, list) and len(sc) >= 2: want = (sc[1], sc[0]) bad_cursor = want is not None and None not in want and tuple(want) != cursor and not hidden if differ or bad_cursor: c.view_bad = getattr(c, 'view_bad', 0) + 1 d = c.dir / f'view-{c.view_bad}.txt' d.write_text(f'cursor client {cursor} hidden {hidden} session {sc}\nrows differing {differ[:12]}\n' + ''.join(f'client {r:2} |{lines[r]}|\nsession {r:2} |{theirs[r]}|\n' for r in differ[:12])) run.breach(f'{c.label}: its screen is not the session\'s render of its view ({len(differ)} rows' f'{", cursor " + str(cursor) + " vs " + str(want) if bad_cursor else ""}): {d}') def global_event(run, rng, live, made): """Something happens beside the typing: a resize, a client detached and another attached, an Upgrade, a script's edit over 9P.""" kinds = [e for e in run.args.events.split(',') if e] if not kinds: return None if run.args.upgrade and not getattr(run, 'upgraded', False): kinds.append('upgrade') what = rng.choice(kinds) if what == 'resize': c = rng.choice(live) rows, cols = rng.randint(18, 50), rng.randint(60, 160) run.note(f'{c.label} resized to {cols}x{rows}') c.resize(rows, cols) elif what == 'detach': c = rng.choice(live) run.note(f'{c.label} detached') hold(run, [c]) c.asked_to_end = True c.keys(ESC) c.keys(b'\x02') time.sleep(0.2) c.keys(ESC) c.keys(b' sD') try: wait_for(f'{c.label} let go', lambda: c.poll() is not None, timeout=15) except TimeoutError: # Its keys went to a program or a prompt: it is ended as a window closes. run.note(f'{c.label} did not detach by keys: hung up') c.hangup() wait_for(f'{c.label} hung up', lambda: c.poll() is not None, timeout=15) live.remove(c) made[0] += 1 n = start_client(run, c.kind, c.rtt, made[0]) live.append(n) return n elif what == 'upgrade': run.note('Upgrade') run.upgraded = True run.nine_write('/ctl', b'Upgrade\n') time.sleep(3) if run.nine: try: run.nine.socket.close() except Exception: pass run.nine = None elif what == 'script': rows = (run.nine_read('/index') or b'').splitlines() texts = [r.split()[0].decode() for r in rows if len(r.split()) > 1 and r.split()[1] == b'text'] if texts: n = rng.choice(texts) run.note(f'9P append to pane {n}') run.nine_write(f'/pane/{n}/body', b'a line a script wrote\n') return None def soak(run, kinds, rtt, minutes, seed): rng = random.Random(seed) stop = threading.Event() live = [] threads = [] made = [0] try: soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made) except Exception: import traceback run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) finally: stop.set() report_cell(run, rtt) run.cleanup() def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): run.start_session() for kind in kinds: made[0] += 1 live.append(start_client(run, kind, rtt, made[0])) for c in list(live): t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) t.start() threads.append(t) end = now() + minutes * 60 next_quiet = now() + 45 next_event = now() + rng.uniform(20, 60) while now() < end: time.sleep(0.5) for c in list(live): if c.poll() is not None: run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') live.remove(c) if not run.session_alive(): run.breach('the session died') break if not live: break if now() > next_event: n = global_event(run, rng, live, made) if n: t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) t.start() threads.append(t) next_event = now() + rng.uniform(20, 70) if now() > next_quiet: quiet_point(run, live) next_quiet = now() + rng.uniform(40, 80) stop.set() for t in threads: t.join(timeout=20) if run.session_alive() and live: quiet_point(run, live) # Each client's links written: a hangup ends a client the normal way. for c in live: c.asked_to_end = True if c.p.poll() is None: c.hangup() for c in live: try: wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) except TimeoutError: run.breach(f'{c.label}: would not end on a hangup') c.p.kill() if run.session_alive(): run.end_session() def tail(c): if isinstance(c, WinClient): return c.said()[-300:].replace('\n', ' | ') lines, _, _ = c.term.view() return ' | '.join(line for line in lines if line.strip())[-300:] ROWS = [] def cores(run): """systemd-coredump's cores since the cell began, of the pids it started.""" if isinstance(run, BoxRun) or not shutil.which('coredumpctl'): return [] since = time.time() - (now() - run.t0) - 5 got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True) ours = set(run.pids) | {c.p.pid for c in run.clients if c.p} | ({run.session_pid} if run.session_pid else set()) out = [] for line in got.stdout.splitlines(): w = line.split() pid = next((int(x) for x in w if x.isdigit() and int(x) in ours), None) if pid: out.append(f'pid {pid} {" ".join(x for x in w if x.startswith("SIG"))} {w[-2] if len(w) > 1 else ""}') return out def report_cell(run, rtt): for core in cores(run): run.breach(f'core dumped: {core} (coredumpctl debug )') said = run.session_said() for word in ('panic', 'Segmentation', 'abort', 'reached unreachable', 'core dumped'): if word in said: run.breach(f'session said "{word}": {said[said.find(word) - 200:said.find(word) + 300]!r}') break if run.crashes(): run.breach(f'crashes: {run.crashes()[-400:]}') for c in run.clients: st = c.stats_sum() links = len(c.links()) lines = c.check_lines() differ = [line for line in lines if line.startswith('differ')] said = c.said_joined() if links > 1 or said: run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})') if differ: run.breach(f'{c.label}: {len(differ)} checks differ') if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): run.breach(f'{c.label}: died of signal {-c.gone}') if c.gone not in (None, 0) and not c.asked_to_end: pass # said when found elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') joined = c.said() for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): if word in joined: run.breach(f'{c.label} said "{word}": {joined[joined.find(word) - 200:joined.find(word) + 300]!r}') break if isinstance(c, WinClient): lat, missed = c.latencies() else: lat, missed = c.lat, c.no_output dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) ROWS.append({ 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone, }) st = (run.root / 'session.stats') text = '' if isinstance(run, BoxRun): text = run.box(f'cat {run.home}/session.stats 2>/dev/null', timeout=30).stdout.decode() elif st.exists(): text = st.read_text() for line in text.splitlines(): if line.startswith('session inputs'): w = line.split() d = {w[i]: w[i + 1] for i in range(1, len(w) - 1, 2)} worst = int(d.get('boundary_us', '0/0/0').split('/')[2]) ROWS.append({'cell': run.cell, 'client': 'session', 'rtt': rtt, 'boundary_us': d.get('boundary_us'), 'input_us': d.get('input_us'), 'cut': d.get('cut')}) if worst > 250_000: run.breach(f'session: a boundary took {worst / 1000:.0f} ms') run.note('end of cell') def table(rows): cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit'] def fmt(v): if v is None: return '-' if isinstance(v, float): return f'{v:.1f}' return str(v) out = [] widths = {c: len(c) for c in cols} body = [] for r in rows: if r['client'] == 'session': body.append(f"{r['cell']:<8} session boundary_us p50/p99/max {r['boundary_us']} input_us {r['input_us']} cut {r['cut']}") continue vals = [fmt(r.get(c)) for c in cols] for c, v in zip(cols, vals): widths[c] = max(widths[c], len(v)) body.append(vals) out.append(' '.join(c.ljust(widths[c]) for c in cols)) for b in body: out.append(b if isinstance(b, str) else ' '.join(v.ljust(widths[c]) for c, v in zip(cols, b))) return '\n'.join(out) def main(): ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) ap.add_argument('where', choices=['local', 'box']) ap.add_argument('--minutes', type=float, default=None) ap.add_argument('--rtts', default='0,30,80,207') ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') ap.add_argument('--seed', type=int, default=1) ap.add_argument('--clients', default='tty,win', help='kinds, one client each: tty,win,win...') ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') ap.add_argument('--keep', action='store_true', help='keep the run\'s files (and the terminals\' output)') ap.add_argument('--only', default='', help='the monkey\'s actions, by name: move,insert,...') ap.add_argument('--skip', default='', help='actions left out, by name') ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') args = ap.parse_args() kinds = args.clients.split(',') runs = [] if args.where == 'local': minutes = args.minutes if args.minutes is not None else 2 for i, rtt in enumerate(int(x) for x in args.rtts.split(',')): run = Run(args, f'local{rtt}') run.far = not args.near runs.append(run) print(f'soak: {run.cell}: {kinds} for {minutes} min, seed {args.seed + i}, in {run.root}', flush=True) soak(run, kinds, rtt, minutes, args.seed + i) else: minutes = args.minutes if args.minutes is not None else 20 run = BoxRun(args, 'box') runs.append(run) print(f'soak: box: {kinds} for {minutes} min, seed {args.seed}, in {run.root}', flush=True) soak(run, kinds, 207, minutes, args.seed) print(table(ROWS)) breaches = [(r.cell, b) for r in runs for b in r.breaches] for cell, b in breaches: print(f'BREACH {cell}: {b}') for r in runs: if args.keep or r.breaches: print(f'kept: {r.root}') else: shutil.rmtree(r.root, ignore_errors=True) print('soak:', 'FAIL' if breaches else 'pass') sys.exit(1 if breaches else 0) if __name__ == '__main__': main()